genml-kit 0.2.1__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {genml_kit-0.2.1 → genml_kit-0.3.0}/PKG-INFO +1975 -63
- {genml_kit-0.2.1 → genml_kit-0.3.0}/README.md +1956 -61
- genml_kit-0.3.0/genml_kit/cli/__init__.py +12 -0
- {genml_kit-0.2.1/genml_kit/training → genml_kit-0.3.0/genml_kit/cli}/infer.py +43 -79
- genml_kit-0.3.0/genml_kit/cli/llm.py +53 -0
- genml_kit-0.3.0/genml_kit/cli/tokenizer.py +146 -0
- genml_kit-0.3.0/genml_kit/cli/train.py +526 -0
- genml_kit-0.3.0/genml_kit/cli/tune.py +303 -0
- genml_kit-0.3.0/genml_kit/datasets/__init__.py +49 -0
- genml_kit-0.3.0/genml_kit/datasets/base.py +156 -0
- genml_kit-0.3.0/genml_kit/datasets/batch_policy.py +139 -0
- genml_kit-0.3.0/genml_kit/datasets/capped.py +125 -0
- genml_kit-0.3.0/genml_kit/datasets/counting.py +784 -0
- genml_kit-0.3.0/genml_kit/datasets/det_folder.py +285 -0
- genml_kit-0.3.0/genml_kit/datasets/det_transforms.py +198 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/ensemble.py +216 -28
- genml_kit-0.3.0/genml_kit/datasets/factory.py +85 -0
- genml_kit-0.3.0/genml_kit/datasets/field_selector.py +124 -0
- genml_kit-0.3.0/genml_kit/datasets/hf_proxy.py +489 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/image_folder.py +3 -3
- genml_kit-0.3.0/genml_kit/datasets/iterable.py +282 -0
- genml_kit-0.3.0/genml_kit/datasets/length_bucket_sampler.py +92 -0
- genml_kit-0.3.0/genml_kit/datasets/packing_sampler.py +108 -0
- genml_kit-0.3.0/genml_kit/datasets/preference_proxy.py +138 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/replay_buffer.py +11 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/retry.py +10 -3
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/rollout_buffer.py +71 -14
- genml_kit-0.3.0/genml_kit/datasets/seg_folder.py +206 -0
- genml_kit-0.3.0/genml_kit/datasets/split.py +213 -0
- genml_kit-0.3.0/genml_kit/datasets/stream_batching.py +212 -0
- genml_kit-0.3.0/genml_kit/datasets/text_proxy.py +267 -0
- genml_kit-0.3.0/genml_kit/datasets/transforms.py +223 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/vo_pairs.py +19 -11
- genml_kit-0.3.0/genml_kit/datasets/webdataset_source.py +180 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/geometry/similarity.py +5 -2
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/io/checkpointing.py +119 -33
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/io/storage_utils.py +33 -32
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/__init__.py +28 -0
- genml_kit-0.3.0/genml_kit/losses/alignment.py +104 -0
- genml_kit-0.3.0/genml_kit/losses/builtin.py +208 -0
- genml_kit-0.3.0/genml_kit/losses/detection.py +342 -0
- genml_kit-0.3.0/genml_kit/losses/distillation.py +241 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/rl.py +9 -3
- genml_kit-0.3.0/genml_kit/losses/segmentation.py +131 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/__init__.py +4 -5
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/base.py +169 -49
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/byol.py +29 -24
- genml_kit-0.3.0/genml_kit/methods/causal_lm.py +370 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/classification.py +141 -31
- genml_kit-0.3.0/genml_kit/methods/detection.py +326 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/dino.py +46 -27
- genml_kit-0.3.0/genml_kit/methods/distillation.py +577 -0
- genml_kit-0.3.0/genml_kit/methods/dpo.py +183 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/ijepa.py +138 -44
- genml_kit-0.3.0/genml_kit/methods/reward_model.py +187 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/rl_dqn.py +27 -25
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/rl_ppo.py +31 -39
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/rl_sac.py +83 -58
- genml_kit-0.3.0/genml_kit/methods/rl_utils.py +130 -0
- genml_kit-0.3.0/genml_kit/methods/segmentation.py +358 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/simmim.py +13 -7
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/supcon.py +24 -8
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/vo_pair.py +22 -16
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/__init__.py +12 -5
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/byol.py +4 -5
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/loader.py +6 -4
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/model.py +20 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/convvit/loader.py +25 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/convvit/masked_encoder.py +16 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/convvit/model.py +44 -7
- genml_kit-0.3.0/genml_kit/models/decoders/__init__.py +109 -0
- genml_kit-0.3.0/genml_kit/models/decoders/base.py +133 -0
- genml_kit-0.3.0/genml_kit/models/decoders/fcos_head.py +170 -0
- genml_kit-0.3.0/genml_kit/models/decoders/fpn.py +90 -0
- genml_kit-0.3.0/genml_kit/models/decoders/linear.py +60 -0
- genml_kit-0.3.0/genml_kit/models/decoders/unet.py +105 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/__init__.py +7 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/loader.py +147 -0
- genml_kit-0.3.0/genml_kit/models/det_model_wrapper/model.py +122 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/dino.py +3 -4
- genml_kit-0.3.0/genml_kit/models/ema.py +276 -0
- genml_kit-0.3.0/genml_kit/models/encoder_utils.py +157 -0
- genml_kit-0.3.0/genml_kit/models/generative.py +91 -0
- genml_kit-0.3.0/genml_kit/models/layer_match.py +203 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/registry.py +54 -12
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/actor_critic.py +13 -2
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/qnetwork.py +11 -3
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/sac_critic.py +7 -7
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/spaces.py +5 -2
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/__init__.py +7 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/encoder.py +119 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/loader.py +184 -0
- genml_kit-0.3.0/genml_kit/models/seg_model_wrapper/model.py +118 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/timm/loader.py +3 -2
- genml_kit-0.3.0/genml_kit/models/tokenizer.py +291 -0
- genml_kit-0.3.0/genml_kit/models/tokenizer_train.py +346 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/uvito/loader.py +7 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/uvito/model.py +14 -10
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/vo/loader.py +16 -3
- genml_kit-0.3.0/genml_kit/pipelines/__init__.py +34 -0
- genml_kit-0.3.0/genml_kit/pipelines/base.py +282 -0
- genml_kit-0.3.0/genml_kit/pipelines/contracts.py +91 -0
- genml_kit-0.3.0/genml_kit/pipelines/detection.py +225 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/pipelines/images.py +187 -86
- genml_kit-0.3.0/genml_kit/pipelines/preference.py +197 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/pipelines/rl.py +84 -137
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/pipelines/rl_normalize.py +3 -2
- genml_kit-0.3.0/genml_kit/pipelines/segmentation.py +421 -0
- genml_kit-0.3.0/genml_kit/pipelines/text.py +515 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/pipelines/vo_pair.py +23 -15
- genml_kit-0.3.0/genml_kit/search/__init__.py +35 -0
- genml_kit-0.3.0/genml_kit/search/errors.py +17 -0
- genml_kit-0.3.0/genml_kit/search/pruners.py +242 -0
- genml_kit-0.3.0/genml_kit/search/runner.py +307 -0
- genml_kit-0.3.0/genml_kit/search/samplers.py +293 -0
- genml_kit-0.3.0/genml_kit/search/space.py +306 -0
- genml_kit-0.3.0/genml_kit/search/store.py +101 -0
- genml_kit-0.3.0/genml_kit/search/study.py +139 -0
- genml_kit-0.3.0/genml_kit/search/trial.py +120 -0
- genml_kit-0.3.0/genml_kit/training/budget.py +208 -0
- genml_kit-0.3.0/genml_kit/training/calibration.py +123 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/classifiers/__init__.py +15 -26
- genml_kit-0.3.0/genml_kit/training/det_metrics.py +224 -0
- genml_kit-0.3.0/genml_kit/training/generate.py +231 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/grad_monitor.py +9 -6
- genml_kit-0.3.0/genml_kit/training/infer.py +71 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/labels.py +11 -5
- genml_kit-0.3.0/genml_kit/training/llm_eval.py +366 -0
- genml_kit-0.3.0/genml_kit/training/llm_model.py +148 -0
- genml_kit-0.3.0/genml_kit/training/metrics_sinks.py +364 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/model_utils.py +158 -11
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/optim_factory.py +138 -10
- genml_kit-0.3.0/genml_kit/training/packed_attention.py +199 -0
- genml_kit-0.3.0/genml_kit/training/preference_collator.py +112 -0
- genml_kit-0.3.0/genml_kit/training/rl_trainer.py +712 -0
- genml_kit-0.3.0/genml_kit/training/run_manifest.py +125 -0
- genml_kit-0.3.0/genml_kit/training/seg_metrics.py +126 -0
- genml_kit-0.3.0/genml_kit/training/sequence_mode.py +103 -0
- genml_kit-0.3.0/genml_kit/training/text_collator.py +390 -0
- genml_kit-0.3.0/genml_kit/training/train.py +292 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/train_compat.py +58 -13
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/train_reporting.py +252 -93
- genml_kit-0.3.0/genml_kit/training/trainer.py +747 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/video_utils.py +1 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/vo/eval_vo.py +8 -4
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/vo/train_vo.py +19 -9
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/xgb_pipeline.py +16 -12
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/xgb_utils.py +7 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/args.py +71 -61
- genml_kit-0.3.0/genml_kit/utils/attr.py +113 -0
- genml_kit-0.3.0/genml_kit/utils/cli.py +237 -0
- genml_kit-0.3.0/genml_kit/utils/distributed.py +1056 -0
- genml_kit-0.3.0/genml_kit/utils/generation.py +449 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/gpu.py +18 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/logging.py +7 -2
- genml_kit-0.3.0/genml_kit/utils/script.py +213 -0
- genml_kit-0.3.0/genml_kit/utils/seg.py +236 -0
- genml_kit-0.3.0/genml_kit/utils/shared_args.py +537 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/signal.py +6 -4
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit.egg-info/PKG-INFO +1975 -63
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit.egg-info/SOURCES.txt +166 -1
- genml_kit-0.3.0/genml_kit.egg-info/entry_points.txt +6 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit.egg-info/requires.txt +24 -1
- genml_kit-0.3.0/pyproject.toml +97 -0
- genml_kit-0.3.0/tests/test_alignment_losses.py +101 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_args_frozen.py +1 -27
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_attr_utils.py +26 -7
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_balanced_sampler.py +4 -4
- genml_kit-0.3.0/tests/test_batch_policy.py +241 -0
- genml_kit-0.3.0/tests/test_budget.py +135 -0
- genml_kit-0.3.0/tests/test_budget_distributed.py +157 -0
- genml_kit-0.3.0/tests/test_cache_dir.py +92 -0
- genml_kit-0.3.0/tests/test_calibration.py +164 -0
- genml_kit-0.3.0/tests/test_capped_dataset.py +111 -0
- genml_kit-0.3.0/tests/test_causal_lm.py +280 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_checkpointing.py +150 -16
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_classifiers.py +4 -3
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_cli_help.py +1 -3
- genml_kit-0.3.0/tests/test_cli_utils.py +262 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_contrastive_models.py +22 -0
- genml_kit-0.3.0/tests/test_counting.py +543 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_custom_models.py +71 -9
- genml_kit-0.3.0/tests/test_dataset_arg.py +90 -0
- genml_kit-0.3.0/tests/test_det_e2e.py +213 -0
- genml_kit-0.3.0/tests/test_det_folder.py +176 -0
- genml_kit-0.3.0/tests/test_det_head.py +129 -0
- genml_kit-0.3.0/tests/test_det_loss.py +116 -0
- genml_kit-0.3.0/tests/test_det_metrics.py +114 -0
- genml_kit-0.3.0/tests/test_det_transforms.py +123 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_dino.py +2 -1
- genml_kit-0.3.0/tests/test_distillation_losses.py +154 -0
- genml_kit-0.3.0/tests/test_distillation_method.py +306 -0
- genml_kit-0.3.0/tests/test_distributed.py +832 -0
- genml_kit-0.3.0/tests/test_distributed_e2e.py +186 -0
- genml_kit-0.3.0/tests/test_dpo.py +186 -0
- genml_kit-0.3.0/tests/test_ema.py +147 -0
- genml_kit-0.3.0/tests/test_ema_weights.py +262 -0
- genml_kit-0.3.0/tests/test_ema_weights_trainer.py +154 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_encoder_utils.py +58 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_ensemble_collate.py +5 -5
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_ensemble_labels.py +9 -8
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_ensemble_mixed_labels.py +15 -4
- genml_kit-0.3.0/tests/test_generation.py +273 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_grad_monitor.py +97 -102
- genml_kit-0.3.0/tests/test_hyperband.py +97 -0
- genml_kit-0.3.0/tests/test_ijepa.py +145 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_init_orthogonal.py +12 -6
- genml_kit-0.3.0/tests/test_iterable_base.py +260 -0
- genml_kit-0.3.0/tests/test_iterable_ddp.py +343 -0
- genml_kit-0.3.0/tests/test_iterable_ensemble.py +195 -0
- genml_kit-0.3.0/tests/test_iterable_flush.py +129 -0
- genml_kit-0.3.0/tests/test_iterable_hf_proxy.py +123 -0
- genml_kit-0.3.0/tests/test_iterable_resume.py +175 -0
- genml_kit-0.3.0/tests/test_iterable_split.py +189 -0
- genml_kit-0.3.0/tests/test_iterable_webdataset.py +412 -0
- genml_kit-0.3.0/tests/test_iterable_workers.py +240 -0
- genml_kit-0.3.0/tests/test_layer_match.py +183 -0
- genml_kit-0.3.0/tests/test_length_bucket_sampler.py +49 -0
- genml_kit-0.3.0/tests/test_llm_cli.py +195 -0
- genml_kit-0.3.0/tests/test_llm_e2e.py +108 -0
- genml_kit-0.3.0/tests/test_llm_eval.py +119 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_log_format.py +35 -33
- genml_kit-0.3.0/tests/test_logging_helpers.py +49 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_lora.py +81 -1
- genml_kit-0.3.0/tests/test_lora_targets.py +171 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_loss_and_multipliers.py +1 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_loss_registry.py +15 -3
- genml_kit-0.3.0/tests/test_make_tiny_coco.py +83 -0
- genml_kit-0.3.0/tests/test_md_latex.py +322 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_method_lifecycle.py +14 -7
- genml_kit-0.3.0/tests/test_metrics_sinks.py +242 -0
- genml_kit-0.3.0/tests/test_midepoch_resume.py +275 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_optim_factory.py +11 -11
- genml_kit-0.3.0/tests/test_owner_arg_defaults.py +221 -0
- genml_kit-0.3.0/tests/test_packed_attention.py +130 -0
- genml_kit-0.3.0/tests/test_packing_sampler.py +108 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_pipeline.py +56 -15
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_pipeline_method_registry.py +106 -41
- genml_kit-0.3.0/tests/test_preference_collator.py +109 -0
- genml_kit-0.3.0/tests/test_preference_e2e.py +90 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_pretrain_methods.py +53 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_pretrain_smoke.py +2 -2
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_pretrain_vis.py +14 -14
- genml_kit-0.3.0/tests/test_readme_cli.py +50 -0
- genml_kit-0.3.0/tests/test_readme_cli_coverage.py +163 -0
- genml_kit-0.3.0/tests/test_readme_cli_harness.py +394 -0
- genml_kit-0.3.0/tests/test_rl_dist_e2e.py +107 -0
- genml_kit-0.3.0/tests/test_rl_distributed.py +540 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_learning.py +18 -42
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_method.py +5 -22
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_model_selector.py +7 -47
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_normalize.py +8 -7
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_pipeline.py +23 -10
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_ppo.py +7 -24
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_sac.py +5 -21
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_trainer.py +260 -198
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_video.py +67 -23
- genml_kit-0.3.0/tests/test_run_manifest.py +98 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_s3r2util.py +1 -1
- genml_kit-0.3.0/tests/test_safe_len.py +285 -0
- genml_kit-0.3.0/tests/test_script_utils.py +229 -0
- genml_kit-0.3.0/tests/test_search_runner.py +244 -0
- genml_kit-0.3.0/tests/test_search_samplers.py +113 -0
- genml_kit-0.3.0/tests/test_search_space.py +123 -0
- genml_kit-0.3.0/tests/test_search_store_trial.py +125 -0
- genml_kit-0.3.0/tests/test_search_study_pruners.py +119 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_seed_utils.py +9 -9
- genml_kit-0.3.0/tests/test_seg_class_map.py +254 -0
- genml_kit-0.3.0/tests/test_seg_decoders.py +137 -0
- genml_kit-0.3.0/tests/test_seg_e2e.py +188 -0
- genml_kit-0.3.0/tests/test_seg_folder.py +199 -0
- genml_kit-0.3.0/tests/test_seg_hf_proxy.py +69 -0
- genml_kit-0.3.0/tests/test_seg_transforms.py +239 -0
- genml_kit-0.3.0/tests/test_seg_vis.py +174 -0
- genml_kit-0.3.0/tests/test_sequence_mode.py +71 -0
- genml_kit-0.3.0/tests/test_shared_args.py +104 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_signal_utils.py +4 -4
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_simmim.py +41 -0
- genml_kit-0.3.0/tests/test_ssl_ddp.py +203 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_storage_utils.py +4 -5
- genml_kit-0.3.0/tests/test_streaming_classification.py +238 -0
- genml_kit-0.3.0/tests/test_streaming_text_pipeline.py +206 -0
- genml_kit-0.3.0/tests/test_text_collator.py +222 -0
- genml_kit-0.3.0/tests/test_text_pipeline.py +243 -0
- genml_kit-0.3.0/tests/test_tokenizer.py +117 -0
- genml_kit-0.3.0/tests/test_tokenizer_arg.py +95 -0
- genml_kit-0.3.0/tests/test_tokenizer_train.py +259 -0
- genml_kit-0.3.0/tests/test_train_cli_integration.py +82 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_train_reporting.py +166 -102
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_train_smoke.py +2 -2
- genml_kit-0.3.0/tests/test_trainer.py +654 -0
- genml_kit-0.3.0/tests/test_trainer_ddp.py +224 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_transformer_encoder_init.py +0 -1
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_vo_training.py +6 -6
- genml_kit-0.3.0/tests/test_wandb_sink.py +160 -0
- genml_kit-0.3.0/tests/test_warmup_schedulers.py +216 -0
- genml_kit-0.3.0/tests/test_webdataset_pipeline.py +94 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_xgb_pipeline.py +4 -2
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_xgb_utils.py +7 -6
- genml_kit-0.2.1/genml_kit/datasets/__init__.py +0 -23
- genml_kit-0.2.1/genml_kit/datasets/factory.py +0 -58
- genml_kit-0.2.1/genml_kit/datasets/field_dataset.py +0 -40
- genml_kit-0.2.1/genml_kit/datasets/hf_proxy.py +0 -239
- genml_kit-0.2.1/genml_kit/datasets/transforms.py +0 -60
- genml_kit-0.2.1/genml_kit/losses/builtin.py +0 -106
- genml_kit-0.2.1/genml_kit/methods/rl_utils.py +0 -69
- genml_kit-0.2.1/genml_kit/models/encoder_utils.py +0 -89
- genml_kit-0.2.1/genml_kit/pipelines/__init__.py +0 -25
- genml_kit-0.2.1/genml_kit/pipelines/base.py +0 -62
- genml_kit-0.2.1/genml_kit/pipelines/contracts.py +0 -21
- genml_kit-0.2.1/genml_kit/training/rl_trainer.py +0 -496
- genml_kit-0.2.1/genml_kit/training/train.py +0 -499
- genml_kit-0.2.1/genml_kit/training/trainer.py +0 -337
- genml_kit-0.2.1/genml_kit/utils/attr.py +0 -141
- genml_kit-0.2.1/genml_kit/utils/cli.py +0 -110
- genml_kit-0.2.1/genml_kit/utils/script.py +0 -92
- genml_kit-0.2.1/genml_kit.egg-info/entry_points.txt +0 -3
- genml_kit-0.2.1/pyproject.toml +0 -69
- genml_kit-0.2.1/tests/test_cli_utils.py +0 -146
- genml_kit-0.2.1/tests/test_script_utils.py +0 -67
- genml_kit-0.2.1/tests/test_trainer.py +0 -320
- {genml_kit-0.2.1 → genml_kit-0.3.0}/LICENSE +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/augmentations/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/augmentations/dual_view.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/augmentations/multicrop.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/balanced_sampler.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/datasets/weighted_sampler.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/geometry/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/io/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/byol.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/contrastive.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/dino.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/focal.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/losses/registry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/methods/registry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/attention_pooling.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/cls_model_wrapper/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/contrastive.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/convvit/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/convvit/processor.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/init.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/masked_encoder.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/processors/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/processors/base.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/rl/sac_model.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/simmim.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/timm/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/timm/model.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/timm/processor.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/transformer_utils.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/uvito/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/uvito/processor.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/vo/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/vo/processor.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/models/vo/vo_similar.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/pipelines/registry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/classifiers/base.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/classifiers/cls_attention.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/classifiers/mlp.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/eval.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/metrics.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/param_align.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/tta.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/training/vo/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/__init__.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/env.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/image_dump.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/label.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/registry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/seed.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/table.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit/utils/transformer.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit.egg-info/dependency_links.txt +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/genml_kit.egg-info/top_level.txt +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/setup.cfg +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_actor_critic.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_attention_pooling.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_byol.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_datasets_ensemble.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_ensemble_ensure_label_space.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_env.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_evaluation_metrics.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_extract_features.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_geometry_similarity.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_headless_models.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_image_dump.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_infer_topk.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_logging_utils.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_metrics.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_model_script_loading.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_param_align.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_replay_buffer.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_retry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_cli.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_losses.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_losses_phase2.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_models.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rl_spaces.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_rollout_buffer.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_set_train_mode.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_supcon_loss.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_supcon_method.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_table_utils.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_timm_models.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_tta.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_utils_registry.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_uvito.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_vo_e2e.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_vo_pairs.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_vo_similar.py +0 -0
- {genml_kit-0.2.1 → genml_kit-0.3.0}/tests/test_weighted_sampler.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: genml_kit
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: General-purpose ML toolkit: image-classification, visual-odometry and self-supervised pre-training through one unified harness.
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
Project-URL: Homepage, https://github.com/davidel/genml_kit
|
|
@@ -14,6 +14,7 @@ Requires-Dist: torch
|
|
|
14
14
|
Requires-Dist: torchvision
|
|
15
15
|
Requires-Dist: transformers
|
|
16
16
|
Requires-Dist: datasets
|
|
17
|
+
Requires-Dist: webdataset
|
|
17
18
|
Requires-Dist: xgboost>=2.0
|
|
18
19
|
Requires-Dist: scikit-learn
|
|
19
20
|
Requires-Dist: pillow
|
|
@@ -24,6 +25,10 @@ Provides-Extra: s3
|
|
|
24
25
|
Requires-Dist: boto3>=1.28; extra == "s3"
|
|
25
26
|
Provides-Extra: lora
|
|
26
27
|
Requires-Dist: peft>=0.7.0; extra == "lora"
|
|
28
|
+
Provides-Extra: llm
|
|
29
|
+
Requires-Dist: peft>=0.7.0; extra == "llm"
|
|
30
|
+
Provides-Extra: llm-spm
|
|
31
|
+
Requires-Dist: sentencepiece>=0.1.99; extra == "llm-spm"
|
|
27
32
|
Provides-Extra: timm
|
|
28
33
|
Requires-Dist: timm>=1.0; extra == "timm"
|
|
29
34
|
Provides-Extra: uvito
|
|
@@ -32,21 +37,37 @@ Provides-Extra: rl
|
|
|
32
37
|
Requires-Dist: gymnasium>=0.29; extra == "rl"
|
|
33
38
|
Requires-Dist: pygame>=2.1; extra == "rl"
|
|
34
39
|
Requires-Dist: imageio[ffmpeg]>=2.31; extra == "rl"
|
|
40
|
+
Requires-Dist: box2d>=2.3.10; extra == "rl"
|
|
41
|
+
Provides-Extra: tune
|
|
42
|
+
Requires-Dist: scipy; extra == "tune"
|
|
43
|
+
Provides-Extra: gpu
|
|
44
|
+
Requires-Dist: nvidia-ml-py; extra == "gpu"
|
|
45
|
+
Provides-Extra: flash
|
|
46
|
+
Requires-Dist: flash-attn; extra == "flash"
|
|
47
|
+
Provides-Extra: wandb
|
|
48
|
+
Requires-Dist: wandb>=0.16; extra == "wandb"
|
|
35
49
|
Provides-Extra: all
|
|
36
|
-
Requires-Dist: genml_kit[gcs,lora,rl,s3,timm,uvito]; extra == "all"
|
|
50
|
+
Requires-Dist: genml_kit[gcs,gpu,llm,llm-spm,lora,rl,s3,timm,tune,uvito,wandb]; extra == "all"
|
|
37
51
|
Provides-Extra: dev
|
|
38
52
|
Requires-Dist: pytest; extra == "dev"
|
|
53
|
+
Requires-Dist: pytest-timeout; extra == "dev"
|
|
54
|
+
Requires-Dist: pytest-split; extra == "dev"
|
|
39
55
|
Requires-Dist: ruff; extra == "dev"
|
|
40
56
|
Requires-Dist: yapf; extra == "dev"
|
|
57
|
+
Requires-Dist: pyyaml; extra == "dev"
|
|
41
58
|
Dynamic: license-file
|
|
42
59
|
|
|
43
60
|
# genml_kit
|
|
44
61
|
|
|
45
62
|
[](https://github.com/davidel/genml_kit/actions/workflows/ci.yml)
|
|
63
|
+
[](https://pypi.org/project/genml_kit/)
|
|
64
|
+
[](https://pypi.org/project/genml_kit/)
|
|
65
|
+
[](LICENSE)
|
|
46
66
|
|
|
47
|
-
A general-purpose toolkit for image classification
|
|
48
|
-
pre-training, supervised fine-tuning,
|
|
49
|
-
|
|
67
|
+
A general-purpose toolkit for image classification and decoder-only language
|
|
68
|
+
models. Supports self-supervised pre-training, supervised fine-tuning, LoRA,
|
|
69
|
+
causal-LM training / generation, preference alignment (DPO, reward models),
|
|
70
|
+
and XGBoost ensemble inference — all from the command line.
|
|
50
71
|
|
|
51
72
|
`genml_kit` is domain-agnostic: point it at any HuggingFace dataset, local
|
|
52
73
|
`ImageFolder` tree, or timm/HuggingFace backbone. It grew out of a
|
|
@@ -70,6 +91,14 @@ in [`vo/README.md`](vo/README.md).
|
|
|
70
91
|
- [Gradient Monitor](#gradient-monitor)
|
|
71
92
|
- [Custom Losses](#custom-losses)
|
|
72
93
|
- [Custom Models](#custom-models)
|
|
94
|
+
- [Semantic Segmentation](#semantic-segmentation)
|
|
95
|
+
- [Object Detection](#object-detection)
|
|
96
|
+
- [Visual Odometry](#visual-odometry)
|
|
97
|
+
- [Knowledge Distillation](#knowledge-distillation)
|
|
98
|
+
- [Large Language Models (causal LM)](#large-language-models-causal-lm)
|
|
99
|
+
- [Reinforcement Learning](#reinforcement-learning)
|
|
100
|
+
- [Multi-GPU Training](#multi-gpu-training)
|
|
101
|
+
- [Hyperparameter Search](#hyperparameter-search)
|
|
73
102
|
- [References And Further Reading](#references-and-further-reading)
|
|
74
103
|
- [Development](#development)
|
|
75
104
|
- [License](#license)
|
|
@@ -191,15 +220,17 @@ reuse the same hook.
|
|
|
191
220
|
|
|
192
221
|
| Module | Contents |
|
|
193
222
|
|---|---|
|
|
194
|
-
| `genml_kit.training` | `train.py` and `infer.py`
|
|
195
|
-
| `genml_kit.methods` | objective-side registry (classification, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon) |
|
|
196
|
-
| `genml_kit.losses` | shared loss library (focal, NT-Xent contrastive, DINO self-distillation, BYOL) |
|
|
223
|
+
| `genml_kit.training` | `train.py` (dataset loading: `load_and_split_dataset`, plus legacy compat re-exports) and `infer.py` (inference helpers), `optim_factory.py` (optimizers, LLRD, schedulers), `model_utils.py` (loading, freezing, feature extraction), `param_align.py`, `eval.py` + `metrics.py`, `grad_monitor.py`, `train_reporting.py`, `tta.py`, `xgb_utils.py` + `xgb_pipeline.py`, `text_collator.py` + `preference_collator.py` (text / preference batches), `llm_model.py` + `generate.py` + `llm_eval.py` (LLM inference, generation, eval), `classifiers/` (pluggable heads) |
|
|
224
|
+
| `genml_kit.methods` | objective-side registry (classification, segmentation, detection, vo_pair, SimMIM, I-JEPA, DINO, BYOL, SupCon, distillation, `causal_lm`, `dpo`, `reward_model`) |
|
|
225
|
+
| `genml_kit.losses` | shared loss library (focal, Dice+CE segmentation, NT-Xent contrastive, DINO self-distillation, BYOL, KD distillation, alignment: `sequence_logps` / `dpo_loss` / `bradley_terry_loss`) |
|
|
197
226
|
| `genml_kit.augmentations` | objective-level transforms (dual-view, multi-crop) |
|
|
198
|
-
| `genml_kit.models` | model/processor registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/` backends; `processors/base.py` |
|
|
199
|
-
| `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
|
|
227
|
+
| `genml_kit.models` | model/processor/tokenizer registry; `timm/`, `convvit/`, `uvito/`, `cls_model_wrapper/`, `seg_model_wrapper/` backends; `decoders/` (pluggable segmentation decoders); `generative.py` (`GenerativeLM` contract), `tokenizer.py` (`TOKENIZERS`), `processors/base.py` |
|
|
228
|
+
| `genml_kit.datasets` | `hf_proxy.py` (HuggingFace → PyTorch bridge), `image_folder.py`, `seg_folder.py` (paired image/mask), `det_folder.py` (COCO detection), `text_proxy.py` (HF text → items), `preference_proxy.py` (chosen/rejected pairs), `ensemble.py`, `field_dataset.py`, `balanced_sampler.py`, `weighted_sampler.py`, `retry.py`, `vo_pairs.py` (synthetic VO pairs) |
|
|
200
229
|
| `genml_kit.geometry` | `similarity.py` (2-D similarity algebra, batched differentiable Umeyama, backward-map warp) — the math behind the visual-odometry front-end lives in [`vo/README.md`](vo/README.md) |
|
|
201
230
|
| `genml_kit.io` | `checkpointing.py` (atomic saves, LoRA state, remote fetch), `storage_utils.py` (S3 / GCS / R2) |
|
|
202
|
-
| `genml_kit.utils` | glog-style logging, CLI arg groups, seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers |
|
|
231
|
+
| `genml_kit.utils` | glog-style logging, CLI arg groups + `shared_args.py` (globally shared `ARG_*` CLI flags), seeding, signal handling, GPU info, tables, external `.py` script loading, `image_dump`, transformer init helpers, `generation.py` (samplers + batched `generate`) |
|
|
232
|
+
| `genml_kit.cli` | console-script entry points: `train.py` (`genml-kit-train`), `infer.py` (`genml-kit-infer`), `llm.py` (`genml-kit-llm`, `generate` / `eval` subcommands), `tune.py` (`genml-kit-tune`, `run` / `report` subcommands) |
|
|
233
|
+
| `genml_kit.search` | hyperparameter search: `space.py` (typed dimensions + argv encoding), `study.py` (declarative study spec), `samplers.py` (`SEARCHERS`: grid/random/tpe/gpei), `pruners.py` (`PRUNERS`: none/median/asha/hyperband), `runner.py` (subprocess-isolated trials), `store.py` (resumable ledger), `trial.py` |
|
|
203
234
|
|
|
204
235
|
## Installation
|
|
205
236
|
|
|
@@ -218,10 +249,30 @@ pip install "genml_kit[s3]"
|
|
|
218
249
|
# With LoRA fine-tuning:
|
|
219
250
|
pip install "genml_kit[lora]"
|
|
220
251
|
|
|
252
|
+
# With causal-LM fine-tuning (LoRA + transformers):
|
|
253
|
+
pip install "genml_kit[llm]"
|
|
254
|
+
|
|
255
|
+
# With a bare SentencePiece tokenizer.model (the 'spm' tokenizer entry):
|
|
256
|
+
pip install "genml_kit[llm-spm]"
|
|
257
|
+
|
|
221
258
|
# With UVito model support:
|
|
222
259
|
pip install "genml_kit[uvito]"
|
|
223
260
|
|
|
224
|
-
#
|
|
261
|
+
# With the scipy-backed hyperparameter sampler (genml-kit-tune gpei):
|
|
262
|
+
pip install "genml_kit[tune]"
|
|
263
|
+
|
|
264
|
+
# With NVML GPU utilization in the training logs (the 'util=...%' field):
|
|
265
|
+
pip install "genml_kit[gpu]"
|
|
266
|
+
|
|
267
|
+
# With Weights & Biases metric streaming (the 'wandb' metrics sink):
|
|
268
|
+
pip install "genml_kit[wandb]"
|
|
269
|
+
|
|
270
|
+
# With flash-attn packing support (requires the CUDA toolkit with CUDA_HOME
|
|
271
|
+
# set to an nvcc, plus torch already installed):
|
|
272
|
+
pip install "genml_kit[flash]"
|
|
273
|
+
|
|
274
|
+
# Everything above in one shot (gcs, s3, lora, llm, llm-spm, timm, uvito, rl, tune, gpu, wandb):
|
|
275
|
+
# (flash is intentionally excluded: it needs a CUDA toolkit to build)
|
|
225
276
|
pip install "genml_kit[all]"
|
|
226
277
|
```
|
|
227
278
|
|
|
@@ -232,10 +283,19 @@ NumPy, scikit-learn ≥ 1.3, XGBoost ≥ 2.0, Pillow, tensorboard.
|
|
|
232
283
|
|
|
233
284
|
The fastest way to get started:
|
|
234
285
|
|
|
286
|
+
<!-- genml-cli-test
|
|
287
|
+
name: quick-start-cifar10
|
|
288
|
+
network: true
|
|
289
|
+
override:
|
|
290
|
+
- --dataset {tmp}/img
|
|
291
|
+
- -.--image_column
|
|
292
|
+
extra: --epochs 1 --max_items 16
|
|
293
|
+
-->
|
|
235
294
|
```bash
|
|
236
295
|
# Fine-tune a ViT on an image-classification dataset (5 epochs, ~2 minutes on GPU)
|
|
237
296
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
238
|
-
--dataset cifar10 \
|
|
297
|
+
--dataset uoft-cs/cifar10 \
|
|
298
|
+
--image_column img \
|
|
239
299
|
--label_column label \
|
|
240
300
|
--epochs 5 \
|
|
241
301
|
--batch_size 32 \
|
|
@@ -296,12 +356,13 @@ SimMIM minimizes the mean squared reconstruction error over masked patches:
|
|
|
296
356
|
|
|
297
357
|
$$
|
|
298
358
|
\large
|
|
299
|
-
\mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left
|
|
359
|
+
\mathcal{L}_{\text{MIM}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| \hat{x}_i - x_i \right\\|_2^2
|
|
300
360
|
$$
|
|
301
361
|
|
|
302
362
|
where:
|
|
303
363
|
|
|
304
|
-
- $`x_i`$ — the pixels of the $`i`$-th original patch
|
|
364
|
+
- $`x_i`$ — the pixels of the $`i`$-th original patch ($`i = 1, \dots, N`$;
|
|
365
|
+
$`N`$ = total number of patches);
|
|
305
366
|
- $`\hat{x}_i`$ — the decoder's reconstruction of that patch;
|
|
306
367
|
- $`\mathcal{M}`$ — the set of masked patch indices, and $`|\mathcal{M}|`$ its
|
|
307
368
|
size (how many patches are masked);
|
|
@@ -328,7 +389,7 @@ masked region. The objective is regression in representation space:
|
|
|
328
389
|
|
|
329
390
|
$$
|
|
330
391
|
\large
|
|
331
|
-
\mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left
|
|
392
|
+
\mathcal{L}_{\text{IJEPA}} = \frac{1}{|\mathcal{M}|} \sum_{i \in \mathcal{M}} \left\\| q_\theta(f_\theta(\text{context}))_i - \mathop{\mathrm{stopgrad}}\left(f_\xi(x_i)\right) \right\\|_2^2
|
|
332
393
|
$$
|
|
333
394
|
|
|
334
395
|
where:
|
|
@@ -374,6 +435,28 @@ Three mechanism details complete the picture:
|
|
|
374
435
|
matures, the teacher freezes progressively, stabilizing the targets late in
|
|
375
436
|
training.
|
|
376
437
|
|
|
438
|
+
Run it like any other image pre-training configuration — here on a small
|
|
439
|
+
unlabeled collection:
|
|
440
|
+
|
|
441
|
+
<!-- genml-cli-test
|
|
442
|
+
name: pretrain-ijepa
|
|
443
|
+
network: true
|
|
444
|
+
gated: true
|
|
445
|
+
override:
|
|
446
|
+
- --datasets {tmp}/img
|
|
447
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --predictor_heads 8 --predictor_depth 2 --ijepa_weight 0.5
|
|
448
|
+
-->
|
|
449
|
+
```bash
|
|
450
|
+
genml-kit-train --pipeline images --method ijepa \
|
|
451
|
+
--model convvit \
|
|
452
|
+
--datasets uoft-cs/cifar10 \
|
|
453
|
+
--image_size 32 \
|
|
454
|
+
--batch_size 32 \
|
|
455
|
+
--epochs 100 \
|
|
456
|
+
--predictor_heads 8 \
|
|
457
|
+
--lr 1e-4
|
|
458
|
+
```
|
|
459
|
+
|
|
377
460
|
**SupCon** (Supervised Contrastive Learning): Uses labels to define "positive"
|
|
378
461
|
pairs (same class) and "negative" pairs (different classes). The loss pulls
|
|
379
462
|
features of same-class images together and pushes different-class features
|
|
@@ -409,7 +492,7 @@ Fix one example $`i`$, called the *anchor*. Its *positive set* is
|
|
|
409
492
|
|
|
410
493
|
$$
|
|
411
494
|
\large
|
|
412
|
-
\mathcal{P}(i) =
|
|
495
|
+
\mathcal{P}(i) = \\{\\, j : j \neq i \text{ and } y_j = y_i \\,\\}
|
|
413
496
|
$$
|
|
414
497
|
|
|
415
498
|
i.e. all other examples in the batch that share the anchor's class label
|
|
@@ -466,31 +549,64 @@ they are meant to be confident) and a larger student temperature (softer
|
|
|
466
549
|
[Emerging Properties in Self-Supervised Vision Transformers](https://arxiv.org/abs/2104.14294), ICCV 2021.
|
|
467
550
|
One backbone is instantiated twice. The **student** $`f_\theta`$ is trained by
|
|
468
551
|
gradient descent; the **teacher** $`f_\xi`$ is the EMA copy of the student
|
|
469
|
-
($`\xi \leftarrow m
|
|
552
|
+
($`\xi \leftarrow m\,\xi + (1-m)\,\theta`$, as in I-JEPA above, ramped from
|
|
470
553
|
`--dino_momentum` 0.996 to `--dino_final_momentum` 1.0). Each image is cut
|
|
471
554
|
into 2 **global** crops plus `--dino_local_num` (8) small **local** crops:
|
|
472
555
|
the teacher sees only global views, the student sees all of them, so the
|
|
473
556
|
student must produce globally consistent features from partial views.
|
|
474
557
|
|
|
475
|
-
Both towers end in an MLP projection head
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
$`c
|
|
558
|
+
Both towers end in an MLP projection head that maps each backbone feature to a
|
|
559
|
+
$`D`$-dimensional vector (`--dino_proj_dim`, 256); these vectors are turned
|
|
560
|
+
into probability distributions over the $`D`$ projection dimensions. The
|
|
561
|
+
teacher's logits are *centered* (a running-mean vector $`c`$ is subtracted) and
|
|
562
|
+
*sharpened* (divided by a small temperature $`\tau_\xi`$) before the softmax,
|
|
563
|
+
while the student's logits are divided only by its larger temperature
|
|
564
|
+
$`\tau_\theta`$. Writing $`f_\theta(x)`$ and $`f_\xi(x)`$ for the student and
|
|
565
|
+
teacher projections of an image $`x`$, the center is updated once per step,
|
|
479
566
|
|
|
480
567
|
$$
|
|
481
568
|
\large
|
|
482
|
-
c \leftarrow m_c \\, c + (1 - m_c) \cdot \frac{1}{
|
|
569
|
+
c \leftarrow m_c \\, c + (1 - m_c) \cdot \frac{1}{B_g}\sum_{i=1}^{B_g} f_\xi(x_i)
|
|
483
570
|
$$
|
|
484
571
|
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
572
|
+
and the teacher target and student prediction are
|
|
573
|
+
|
|
574
|
+
$$
|
|
575
|
+
\large
|
|
576
|
+
p_\xi(x) = \mathrm{softmax}\left(\frac{f_\xi(x) - c}{\tau_\xi}\right),
|
|
577
|
+
\qquad
|
|
578
|
+
p_\theta(x) = \mathrm{softmax}\left(\frac{f_\theta(x)}{\tau_\theta}\right)
|
|
579
|
+
$$
|
|
488
580
|
|
|
489
581
|
$$
|
|
490
582
|
\large
|
|
491
|
-
\mathcal{L} = - \sum_{
|
|
583
|
+
\mathcal{L} = - \frac{1}{N}\sum_{n=1}^{N} \sum_{d=1}^{D} p_\xi(x^{(n)})_d \\, \log p_\theta(x^{(n)})_d
|
|
492
584
|
$$
|
|
493
585
|
|
|
586
|
+
where:
|
|
587
|
+
|
|
588
|
+
- $`f_\theta(x), f_\xi(x) \in \mathbb{R}^{D}`$ — the student and teacher
|
|
589
|
+
projection-head outputs for image $`x`$ ($`D`$ = projection dimension,
|
|
590
|
+
`--dino_proj_dim`);
|
|
591
|
+
- $`c \in \mathbb{R}^{D}`$ — the running center: a per-dimension mean of the
|
|
592
|
+
teacher's outputs, updated once per step with momentum $`m_c`$
|
|
593
|
+
(`--dino_center_momentum`, 0.9) over the $`B_g`$ teacher *global-crop*
|
|
594
|
+
outputs in the batch (two global views per image, so $`B_g = 2B`$). It is
|
|
595
|
+
subtracted from the teacher logits so that no single dimension can dominate
|
|
596
|
+
the target;
|
|
597
|
+
- $`\tau_\xi`$ — the teacher temperature (`--dino_teacher_temp`, 0.04); small,
|
|
598
|
+
so the teacher distribution is sharp (confident);
|
|
599
|
+
- $`\tau_\theta`$ — the student temperature (`--dino_student_temp`, 0.1);
|
|
600
|
+
larger, so the student distribution is softer and its gradients stay alive;
|
|
601
|
+
- $`p_\xi(x)`$ — the teacher's target distribution over the $`D`$ dimensions
|
|
602
|
+
(centered, sharpened, softmaxed, and detached from the gradient graph);
|
|
603
|
+
- $`p_\theta(x)`$ — the student's predicted distribution over the same
|
|
604
|
+
$`D`$ dimensions;
|
|
605
|
+
- $`x^{(n)}`$ — the $`n`$-th (student crop, teacher view) pairing: each global
|
|
606
|
+
view is paired with the *other* global view of the same image, and each local
|
|
607
|
+
crop with the teacher's second global view. The loss averages the per-pairing
|
|
608
|
+
cross-entropy over all $`N`$ pairings.
|
|
609
|
+
|
|
494
610
|
Centering and sharpening are the two anti-collapse mechanisms: centering
|
|
495
611
|
cancels any component the teacher produces for *every* input (a constant
|
|
496
612
|
output carries no information and would otherwise still lower the loss), and
|
|
@@ -499,27 +615,72 @@ its argmax rather than drifting toward a flat, uninformative distribution.
|
|
|
499
615
|
Because the targets come from the teacher and the teacher follows the student,
|
|
500
616
|
this is self-distillation — there are no external labels anywhere.
|
|
501
617
|
|
|
618
|
+
A DINO run adds the multi-crop knobs on top of the standard configuration:
|
|
619
|
+
|
|
620
|
+
<!-- genml-cli-test
|
|
621
|
+
name: pretrain-dino
|
|
622
|
+
network: true
|
|
623
|
+
gated: true
|
|
624
|
+
override:
|
|
625
|
+
- --datasets {tmp}/img
|
|
626
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --dino_local_num 2 --dino_local_size 16 --dino_global_size 32
|
|
627
|
+
-->
|
|
628
|
+
```bash
|
|
629
|
+
genml-kit-train --pipeline images --method dino \
|
|
630
|
+
--model convvit \
|
|
631
|
+
--datasets uoft-cs/cifar10 \
|
|
632
|
+
--image_size 32 \
|
|
633
|
+
--batch_size 32 \
|
|
634
|
+
--epochs 100 \
|
|
635
|
+
--lr 1e-4
|
|
636
|
+
```
|
|
637
|
+
|
|
502
638
|
**BYOL** (**B**ootstrap **Y**our **O**wn **L**atent): Grill et al.,
|
|
503
639
|
[Bootstrap Your Own Latent](https://arxiv.org/abs/2006.07733), NeurIPS 2020.
|
|
504
640
|
The same two-tower idea as DINO but with two augmented views and no
|
|
505
641
|
multi-crop. The **online** network (encoder + projector + predictor MLP) and
|
|
506
642
|
the **target** network (encoder + projector, EMA copy) process the two views;
|
|
507
|
-
only the online tower receives gradients.
|
|
508
|
-
|
|
509
|
-
those projections, the loss is the symmetric cosine regression
|
|
643
|
+
only the online tower receives gradients. The loss is the symmetric cosine
|
|
644
|
+
regression
|
|
510
645
|
|
|
511
646
|
$$
|
|
512
647
|
\large
|
|
513
648
|
\mathcal{L} = 2 - \cos(p_1, z_2) - \cos(p_2, z_1)
|
|
514
649
|
$$
|
|
515
650
|
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
651
|
+
where:
|
|
652
|
+
|
|
653
|
+
- $`z_1, z_2`$ — the target network's projections of the two views
|
|
654
|
+
(stop-gradient, so no gradients flow into the target tower);
|
|
655
|
+
- $`p_1, p_2`$ — the online network's *predictions* of those projections;
|
|
656
|
+
- $`\cos(u, v) = u^\top v / (\lVert u \rVert_2 \lVert v \rVert_2)`$ — the
|
|
657
|
+
cosine of the angle between the two vectors.
|
|
658
|
+
|
|
659
|
+
The loss is 0 when predictions perfectly match the target projections and
|
|
660
|
+
reaches 4 when they point in opposite directions. Two asymmetries keep this
|
|
661
|
+
from collapsing to a constant: the predictor exists only on the online side
|
|
662
|
+
(predicting a target is easier than being one), and the targets move slowly
|
|
663
|
+
under the EMA — exactly the teacher-follows-student trick noted under I-JEPA.
|
|
664
|
+
|
|
665
|
+
The BYOL configuration differs only in the method name:
|
|
666
|
+
|
|
667
|
+
<!-- genml-cli-test
|
|
668
|
+
name: pretrain-byol
|
|
669
|
+
network: true
|
|
670
|
+
gated: true
|
|
671
|
+
override:
|
|
672
|
+
- --datasets {tmp}/img
|
|
673
|
+
extra: --epochs 1 --batch_size 2 --max_items 16 --byol_proj_hidden 256 --byol_predictor_hidden 256 --byol_final_momentum 1.0
|
|
674
|
+
-->
|
|
675
|
+
```bash
|
|
676
|
+
genml-kit-train --pipeline images --method byol \
|
|
677
|
+
--model convvit \
|
|
678
|
+
--datasets uoft-cs/cifar10 \
|
|
679
|
+
--image_size 32 \
|
|
680
|
+
--batch_size 32 \
|
|
681
|
+
--epochs 100 \
|
|
682
|
+
--lr 1e-4
|
|
683
|
+
```
|
|
523
684
|
|
|
524
685
|
### Typical Hyperparameters
|
|
525
686
|
|
|
@@ -549,6 +710,9 @@ size and GPU memory:
|
|
|
549
710
|
- DINO and BYOL train on smaller crops (224) with a larger LR because their
|
|
550
711
|
loss acts on a small projection space rather than every pixel; the defaults
|
|
551
712
|
above match the method-specific `--dino_*` / `--byol_*` flags.
|
|
713
|
+
- The DINO/BYOL loss lives in that projection space (`--dino_proj_dim` /
|
|
714
|
+
`--byol_proj_dim`, both 256), not in the backbone feature space: the
|
|
715
|
+
objective compares 256-dimensional distributions or vectors, not pixels.
|
|
552
716
|
- `--samples_per_class 16` with `--batch_size 64` gives 4 classes per batch
|
|
553
717
|
on a 7-class dataset. batch_size need not be divisible by
|
|
554
718
|
samples_per_class; the remainder is spread evenly across the per-class
|
|
@@ -558,6 +722,16 @@ size and GPU memory:
|
|
|
558
722
|
|
|
559
723
|
### Example: Full Pre-Training Pipeline
|
|
560
724
|
|
|
725
|
+
<!-- genml-cli-test
|
|
726
|
+
name: pretrain-full-pipeline
|
|
727
|
+
network: true
|
|
728
|
+
override:
|
|
729
|
+
- --datasets {tmp}/img
|
|
730
|
+
- --dataset {tmp}/img
|
|
731
|
+
- --label_column label
|
|
732
|
+
- -.--source_checkpoint
|
|
733
|
+
extra: --epochs 1
|
|
734
|
+
-->
|
|
561
735
|
```bash
|
|
562
736
|
# Step 1: Pre-train with SimMIM on two large image collections.
|
|
563
737
|
# v4.2: pre-training is a *configuration* of the unified genml-kit-train
|
|
@@ -587,6 +761,14 @@ genml-kit-train --model convvit \
|
|
|
587
761
|
|
|
588
762
|
### Example: Supervised Contrastive Pre-Training
|
|
589
763
|
|
|
764
|
+
<!-- genml-cli-test
|
|
765
|
+
name: pretrain-supcon
|
|
766
|
+
network: true
|
|
767
|
+
override:
|
|
768
|
+
- --datasets {tmp}/img
|
|
769
|
+
- --label_column label
|
|
770
|
+
extra: --epochs 1
|
|
771
|
+
-->
|
|
590
772
|
```bash
|
|
591
773
|
genml-kit-train --pipeline images --method supcon \
|
|
592
774
|
--model convvit \
|
|
@@ -614,7 +796,6 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
614
796
|
| `--datasets` | (required) | Space-separated dataset names or local paths. |
|
|
615
797
|
| `--cache_dir` | `None` | HuggingFace cache directory for downloads. |
|
|
616
798
|
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
617
|
-
| `--hf_token` | `None` | HuggingFace token for gated datasets (or set `HF_TOKEN` env var). |
|
|
618
799
|
| `--image_column` | auto-detected | Explicit HF image column name. |
|
|
619
800
|
| `--label_column` | `None` | Explicit HF label column name. Required by `--method supcon` if non-standard. |
|
|
620
801
|
| `STRICT_DATASETS` (env) | `False` | Abort on first dataset-loading failure instead of skipping. |
|
|
@@ -622,6 +803,8 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
622
803
|
| `--batch_size` | `32` | Per-GPU batch size. |
|
|
623
804
|
| `--seed` | `None` | Explicit RNG seed: pins the shuffling and enables full determinism (cuDNN deterministic kernels, benchmark off). Omit (default) to keep runs fast — RNG streams are still seeded from `$GENML_KIT_SEED` (default 42). See [Reproducibility](#reproducibility). |
|
|
624
805
|
| `--epochs` | `5` | Number of epochs. The recipes below override this for pre-training. |
|
|
806
|
+
| `--max_batches` | unset | Cap the micro-batches **per epoch**; the next epoch still runs (up to `--epochs`). Unset drains the loader each epoch. |
|
|
807
|
+
| `--max_train_time` | unset | Wall-clock budget **in seconds for the whole run**. Stops mid-epoch once exceeded (DDP-synchronized); unset means no time limit. |
|
|
625
808
|
| `--lr` | `3e-5` | Peak learning rate (base). The recipes below override this. |
|
|
626
809
|
| `--amp_dtype` | `None` | Mixed precision: `float16` or `bfloat16`. Omit to disable. |
|
|
627
810
|
| `--num_workers` | `4` | DataLoader worker processes. |
|
|
@@ -636,6 +819,10 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
636
819
|
| `--norm_history` | `0` | Keep last N norm snapshots per parameter for trend analysis. |
|
|
637
820
|
| `--trend_top_n` | `10` | Show top N params in trend table by abs change %. `0` = show all. |
|
|
638
821
|
| `--grad_clip` | `1.0` | Maximum gradient norm for clipping. `0` disables. |
|
|
822
|
+
| `--ema_decay` | `0.0` | Weights-EMA decay for validation and the best checkpoint. `0` disables the shadow (supported methods only -- see [Weights EMA](#weights-ema)). |
|
|
823
|
+
| `--ema_final_decay` | `None` | Optional ramp target: the effective decay moves linearly to this value over the run. |
|
|
824
|
+
| `--ema_warmup_steps` | `0` | Linear warmup of the effective decay from `0` up to `--ema_decay` over this many optimizer steps. Ignored when `--ema_final_decay` is set. |
|
|
825
|
+
| `--ema_eval` | `False` | Evaluate and best-save the EMA weights instead of the raw trajectory (`best.pt` carries EMA weights; `latest.pt` always carries the raw trajectory plus the shadow). |
|
|
639
826
|
| `--lr_group` | `None` | Per-parameter-group learning rates (repeatable). Format: `"REGEX=LR"`. |
|
|
640
827
|
| `--llrd_decay` | `None` | Layer-wise learning rate decay factor. |
|
|
641
828
|
| `--vis_every` | `0` | Log reconstruction visualisation every N steps (SimMIM only). |
|
|
@@ -644,8 +831,9 @@ genml-kit-train --pipeline images --method supcon \
|
|
|
644
831
|
| `--proc_arg` | `{}` | Override processor configuration (repeatable). |
|
|
645
832
|
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
646
833
|
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
647
|
-
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path
|
|
834
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
|
|
648
835
|
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
836
|
+
| `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
|
|
649
837
|
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
650
838
|
| `--param_rename` | `None` | Regex-based key rename patterns (`SEARCH;REPLACE`). |
|
|
651
839
|
| `--grad_checkpoint` | `False` | Enable gradient checkpointing. Reduces activation memory by ~40-50% at the cost of ~25-35% more compute per step. Enables larger batch sizes. See [Gradient Checkpointing](#gradient-checkpointing). |
|
|
@@ -672,8 +860,8 @@ single dataset is large enough for effective pre-training on its own.
|
|
|
672
860
|
|
|
673
861
|
Supported dataset types:
|
|
674
862
|
- **HuggingFace datasets** — any HF dataset ID that returns decoded image
|
|
675
|
-
data (e.g. `cifar10`, `food101`). Gated datasets require
|
|
676
|
-
|
|
863
|
+
data (e.g. `cifar10`, `food101`). Gated datasets require the `HF_TOKEN`
|
|
864
|
+
environment variable.
|
|
677
865
|
- **Local image directories** — pass a path to a folder of images
|
|
678
866
|
(ImageFolder format).
|
|
679
867
|
|
|
@@ -695,6 +883,167 @@ Labels are automatically remapped to a shared global label space across all
|
|
|
695
883
|
datasets, so mixing datasets with overlapping but differently-named classes
|
|
696
884
|
works transparently.
|
|
697
885
|
|
|
886
|
+
#### Streaming (Iterable) Datasets
|
|
887
|
+
|
|
888
|
+
A dataset does not have to fit on disk or in RAM. When a source can only be
|
|
889
|
+
*iterated* (no index, no length), the pipeline streams it: items are read once,
|
|
890
|
+
converted on the fly, and never materialised into an index space.
|
|
891
|
+
|
|
892
|
+
Streaming sources can be used in two places:
|
|
893
|
+
|
|
894
|
+
- **`--dataset` with `--pipeline text`** — pass `streaming=True` through
|
|
895
|
+
`--dataset_arg` to have the HuggingFace loader return a streaming source
|
|
896
|
+
instead of a materialised split:
|
|
897
|
+
|
|
898
|
+
<!-- genml-cli-test-exclude: illustrative streaming run against a user corpus; covered by tests/test_streaming_text_pipeline.py -->
|
|
899
|
+
```bash
|
|
900
|
+
genml-kit-train --pipeline text --model gpt2 \
|
|
901
|
+
--dataset my-org/huge-corpus \
|
|
902
|
+
--text_column text \
|
|
903
|
+
--dataset_arg streaming=True \
|
|
904
|
+
--epochs 1
|
|
905
|
+
```
|
|
906
|
+
|
|
907
|
+
- **`--datasets` (image pre-training ensemble)** — use a `webdataset:` spec
|
|
908
|
+
(see below). If every dataset in an ensemble streams, the ensemble streams
|
|
909
|
+
too, in declaration order.
|
|
910
|
+
|
|
911
|
+
- **`--dataset` with `--pipeline images` (classification)** — pass
|
|
912
|
+
`streaming=True` through `--dataset_arg` to stream an image *classification*
|
|
913
|
+
source instead of materialising it. The train/val split becomes a positional
|
|
914
|
+
one (`IterableDatasetFilter`), and the label column must be a typed
|
|
915
|
+
`ClassLabel` so the class vocabulary is read from the dataset's `features`
|
|
916
|
+
without scanning the corpus. Class weighting (`--sampler weighted` /
|
|
917
|
+
`balanced`, inverse-frequency weights) needs the whole label column, so it is
|
|
918
|
+
refused for streams with a clear error.
|
|
919
|
+
|
|
920
|
+
**What works, and what does not.** A stream has no index space, so anything
|
|
921
|
+
that needs one is refused with a clear error rather than silently misbehaving:
|
|
922
|
+
|
|
923
|
+
| Feature | Random-access | Streaming |
|
|
924
|
+
| --- | --- | --- |
|
|
925
|
+
| Plain batching | yes | yes |
|
|
926
|
+
| `--length_bucket` | yes | yes (over a bounded buffer) |
|
|
927
|
+
| `--packing wrapped` / `bfd` | yes | yes (over a bounded buffer) |
|
|
928
|
+
| `--val_split` | index slice | positional split (`IterableDatasetFilter`) |
|
|
929
|
+
| `--sampler weighted` / `balanced` | yes | not available (needs labels across the whole corpus) |
|
|
930
|
+
| `--max_items` | yes | yes (caps the stream) |
|
|
931
|
+
| Classification (`--dataset`) | yes | yes, with a typed `ClassLabel` label column |
|
|
932
|
+
|
|
933
|
+
Because a stream cannot be shuffled by index, `shuffle` is disabled for it and
|
|
934
|
+
its order is the source order. Under DDP, `--shuffle_buffer_k` supplies
|
|
935
|
+
batch-level shuffling instead — see below.
|
|
936
|
+
|
|
937
|
+
### `--shuffle_buffer_k` (Streaming Shuffle)
|
|
938
|
+
|
|
939
|
+
A streaming source cannot be shuffled by sampler: there is no index space to
|
|
940
|
+
permute. Instead the *batch* stream is shuffled, with a bounded round-robin
|
|
941
|
+
buffer.
|
|
942
|
+
|
|
943
|
+
`--shuffle_buffer_k K` sets the size of that buffer in **batches per rank**, so
|
|
944
|
+
the buffer holds `world_size * K` batches. It only has an effect on a streaming
|
|
945
|
+
loader; random-access datasets shuffle by sampler and ignore it.
|
|
946
|
+
|
|
947
|
+
- **`K = 10`** (default) — a reasonable trade-off between shuffle quality and
|
|
948
|
+
the number of batches buffered in memory.
|
|
949
|
+
- **`K = 1`** — the minimal round-robin: each rank takes one batch per round,
|
|
950
|
+
with no permuting.
|
|
951
|
+
- **`K = 0`** — round-robin without permuting.
|
|
952
|
+
|
|
953
|
+
Memory cost scales with `K`: each buffered batch is a full batch of decoded
|
|
954
|
+
samples, so raise `K` only as far as your RAM allows.
|
|
955
|
+
|
|
956
|
+
<!-- genml-cli-test-exclude: illustrative DDP invocation over local shards; covered by tests/test_iterable_ddp.py -->
|
|
957
|
+
```bash
|
|
958
|
+
torchrun --nproc_per_node=4 -m genml_kit.cli.train --model convvit \
|
|
959
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" \
|
|
960
|
+
--shuffle_buffer_k 20 \
|
|
961
|
+
--batch_size 64
|
|
962
|
+
```
|
|
963
|
+
|
|
964
|
+
### WebDataset Tar Shards (`webdataset:`)
|
|
965
|
+
|
|
966
|
+
For corpora already stored as webdataset tar shards, pass a spec of the form
|
|
967
|
+
`webdataset:<shard-list>` inside `--datasets`. The shard list is passed to
|
|
968
|
+
webdataset verbatim, so its brace expansion applies:
|
|
969
|
+
|
|
970
|
+
<!-- genml-cli-test-exclude: illustrative webdataset pre-training run; covered by tests/test_webdataset_pipeline.py -->
|
|
971
|
+
```bash
|
|
972
|
+
genml-kit-train --pipeline images --model convvit --method simmim \
|
|
973
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" \
|
|
974
|
+
--batch_size 64 \
|
|
975
|
+
--epochs 100
|
|
976
|
+
```
|
|
977
|
+
|
|
978
|
+
A webdataset "sample" is a group of members named `<key>.<extension>` inside the
|
|
979
|
+
tar (`s0001.png`, `s0001.cls`, `s0001.txt`). Members are decoded by extension:
|
|
980
|
+
`png`/`jpg` become images, `txt` a string, `cls` an integer, `json` a dict, `npy`
|
|
981
|
+
an ndarray, `pth` a tensor; anything else stays raw bytes.
|
|
982
|
+
|
|
983
|
+
The image member is detected from the first sample (its name is the extension,
|
|
984
|
+
e.g. `png`). If a shard carries more than one image member, name it explicitly:
|
|
985
|
+
|
|
986
|
+
```bash
|
|
987
|
+
--datasets "webdataset:/data/train-{0000..0999}.tar" --image_column png
|
|
988
|
+
```
|
|
989
|
+
|
|
990
|
+
The `webdataset` package ships with the core install (it is a hard dependency,
|
|
991
|
+
like `datasets` and `transformers`), so no extra is needed.
|
|
992
|
+
|
|
993
|
+
A webdataset member is keyed by extension, so these sources are unlabeled by
|
|
994
|
+
default (there is no dataset-level class vocabulary). Use a method that does not
|
|
995
|
+
need labels (`simmim`, `dino`, `byol`, `ijepa`), or select the label member
|
|
996
|
+
explicitly in code with `IterableFieldSelectorDataset`.
|
|
997
|
+
|
|
998
|
+
### HuggingFace Datasets
|
|
999
|
+
|
|
1000
|
+
Any HF dataset ID whose rows carry decoded image data works directly as
|
|
1001
|
+
`--dataset` (classification) or inside `--datasets` (ensemble). Public
|
|
1002
|
+
datasets need no credentials:
|
|
1003
|
+
|
|
1004
|
+
<!-- genml-cli-test
|
|
1005
|
+
name: hf-dataset-public
|
|
1006
|
+
network: true
|
|
1007
|
+
service: hf
|
|
1008
|
+
extra: --epochs 1 --max_items 64 --batch_size 8
|
|
1009
|
+
-->
|
|
1010
|
+
```bash
|
|
1011
|
+
genml-kit-train --model convvit \
|
|
1012
|
+
--dataset uoft-cs/cifar10 \
|
|
1013
|
+
--image_column img \
|
|
1014
|
+
--label_column label \
|
|
1015
|
+
--image_size 224 \
|
|
1016
|
+
--epochs 5
|
|
1017
|
+
```
|
|
1018
|
+
|
|
1019
|
+
`--max_items` caps each split (applied *after* the train/val split), which
|
|
1020
|
+
keeps these examples fast (the full `cifar10` train split is 50k images).
|
|
1021
|
+
|
|
1022
|
+
### Gated / Private Datasets
|
|
1023
|
+
|
|
1024
|
+
Gated datasets require a HuggingFace token whose owner has been granted access.
|
|
1025
|
+
Set the `HF_TOKEN` environment variable:
|
|
1026
|
+
|
|
1027
|
+
<!-- genml-cli-test
|
|
1028
|
+
name: hf-dataset-gated
|
|
1029
|
+
network: true
|
|
1030
|
+
service: hf
|
|
1031
|
+
gated: true
|
|
1032
|
+
extra: --epochs 1 --batch_size 2
|
|
1033
|
+
-->
|
|
1034
|
+
```bash
|
|
1035
|
+
genml-kit-train --model convvit \
|
|
1036
|
+
--dataset sirtolkien/autotrain-data-test \
|
|
1037
|
+
--image_column image \
|
|
1038
|
+
--label_column target \
|
|
1039
|
+
--image_size 224 \
|
|
1040
|
+
--epochs 5
|
|
1041
|
+
```
|
|
1042
|
+
|
|
1043
|
+
Because the gated dataset cannot be fetched without a grant, the README test
|
|
1044
|
+
harness remaps `--dataset` to a synthesized local fixture and runs the
|
|
1045
|
+
example offline.
|
|
1046
|
+
|
|
698
1047
|
### Preparing Datasets
|
|
699
1048
|
|
|
700
1049
|
Some datasets store images inside zip archives or need custom preprocessing
|
|
@@ -707,10 +1056,18 @@ python my_prepare_script.py --output_dir ./prepared_images
|
|
|
707
1056
|
|
|
708
1057
|
Then use the extracted directory as a local dataset:
|
|
709
1058
|
|
|
1059
|
+
<!-- genml-cli-test
|
|
1060
|
+
name: pretrain-prepared-images
|
|
1061
|
+
network: true
|
|
1062
|
+
override:
|
|
1063
|
+
- --datasets {tmp}/img
|
|
1064
|
+
extra: --epochs 1
|
|
1065
|
+
-->
|
|
710
1066
|
```bash
|
|
711
1067
|
genml-kit-train --pipeline images --method simmim \
|
|
1068
|
+
--model convvit \
|
|
712
1069
|
--datasets ./prepared_images ./other-images \
|
|
713
|
-
--image_size 448 --batch_size 32
|
|
1070
|
+
--image_size 448 --batch_size 32 --epochs 20
|
|
714
1071
|
```
|
|
715
1072
|
|
|
716
1073
|
For a concrete worked example of such a preparation script, see
|
|
@@ -757,7 +1114,7 @@ true class survives and the loss reduces to the familiar $`-\log p(y \mid x)`$.
|
|
|
757
1114
|
genml_kit generalizes this in three directions via `CombinedFocalLoss`:
|
|
758
1115
|
|
|
759
1116
|
- **Soft targets.** With Mixup (below) $`q`$ is the blend
|
|
760
|
-
$`\lambda
|
|
1117
|
+
$`\lambda\, y_i + (1-\lambda)\, y_j`$, and the sum keeps both classes'
|
|
761
1118
|
terms weighted by $`\lambda`$ and $`1-\lambda`$.
|
|
762
1119
|
- **Label smoothing** (via `--loss_args '{"label_smoothing": ...}'`):
|
|
763
1120
|
the one-hot target is replaced by
|
|
@@ -826,6 +1183,37 @@ The confusion matrix uses rows for true classes and columns for predicted
|
|
|
826
1183
|
classes. Diagonal entries are correct predictions. The compact confusion
|
|
827
1184
|
summary reports each class's recall and its largest confusion destinations.
|
|
828
1185
|
|
|
1186
|
+
Two ranking/calibration metrics are also reported when the validation
|
|
1187
|
+
predictions carry logits (the default for model-based methods):
|
|
1188
|
+
|
|
1189
|
+
- **AUROC (macro, one-vs-rest):** how well the model's scores rank a random
|
|
1190
|
+
positive of a class above a random negative, averaged over classes. It is
|
|
1191
|
+
threshold-free and therefore the right summary for imbalanced clinical
|
|
1192
|
+
workloads where the operating point is chosen later.
|
|
1193
|
+
- **ECE (expected calibration error):** the average absolute gap between the
|
|
1194
|
+
model's confidence and its observed accuracy over 15 confidence bins. A
|
|
1195
|
+
model can be accurate and still overconfident; ECE quantifies that gap.
|
|
1196
|
+
|
|
1197
|
+
Overconfidence is fixed post hoc with `--fit_temperature`: after training, a
|
|
1198
|
+
single scalar $`T`$ is fitted on the validation set (minimising negative
|
|
1199
|
+
log-likelihood of `logits / T`), written to `temperature.json` next to the
|
|
1200
|
+
checkpoint, and applied at inference via `genml-kit-infer --temperature T`:
|
|
1201
|
+
|
|
1202
|
+
<!-- genml-cli-test
|
|
1203
|
+
name: fit-temperature
|
|
1204
|
+
override:
|
|
1205
|
+
- --dataset {tmp}/img
|
|
1206
|
+
- --label_column label
|
|
1207
|
+
extra: --epochs 1 --max_items 16 --image_size 32
|
|
1208
|
+
-->
|
|
1209
|
+
```bash
|
|
1210
|
+
genml-kit-train --model convvit --method classification --fit_temperature \
|
|
1211
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1212
|
+
--epochs 5 --batch_size 32
|
|
1213
|
+
genml-kit-infer --model convvit --checkpoint genml_kit_best.pt \
|
|
1214
|
+
--temperature 1.31 --top_k 3 path/to/lesion.jpg
|
|
1215
|
+
```
|
|
1216
|
+
|
|
829
1217
|
|
|
830
1218
|
When TTA is enabled, validation loss is computed from the original image only,
|
|
831
1219
|
while Top-1 accuracy, balanced accuracy, macro F1, weighted F1, and per-class
|
|
@@ -839,6 +1227,14 @@ the same sampling and preprocessing scheme.
|
|
|
839
1227
|
|
|
840
1228
|
### Basic Fine-Tuning
|
|
841
1229
|
|
|
1230
|
+
<!-- genml-cli-test
|
|
1231
|
+
name: fine-tune-basic
|
|
1232
|
+
network: true
|
|
1233
|
+
override:
|
|
1234
|
+
- --dataset {tmp}/img
|
|
1235
|
+
- --label_column label
|
|
1236
|
+
extra: --epochs 1
|
|
1237
|
+
-->
|
|
842
1238
|
```bash
|
|
843
1239
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
844
1240
|
--dataset my-org/my-labeled-images \
|
|
@@ -854,11 +1250,20 @@ genml-kit-train --model google/vit-base-patch16-224 \
|
|
|
854
1250
|
Replace the default linear head with a custom MLP or attention-based
|
|
855
1251
|
classifier:
|
|
856
1252
|
|
|
1253
|
+
<!-- genml-cli-test
|
|
1254
|
+
name: fine-tune-custom-head
|
|
1255
|
+
network: true
|
|
1256
|
+
override:
|
|
1257
|
+
- --dataset {tmp}/img
|
|
1258
|
+
- --label_column label
|
|
1259
|
+
extra: --epochs 1
|
|
1260
|
+
-->
|
|
857
1261
|
```bash
|
|
858
1262
|
# Freeze backbone, train only the custom head
|
|
859
1263
|
genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
|
|
860
1264
|
--dataset my-org/my-labeled-images \
|
|
861
1265
|
--label_column category \
|
|
1266
|
+
--image_size 224 \
|
|
862
1267
|
--model_arg classifier=mlp:hidden=512,dropout=0.3 \
|
|
863
1268
|
--freeze ".*\.(head|pool)"
|
|
864
1269
|
```
|
|
@@ -868,11 +1273,20 @@ genml-kit-train --model cls_model_wrapper:google/vit-base-patch16-224 \
|
|
|
868
1273
|
Freeze the entire backbone and train only small low-rank adapter matrices.
|
|
869
1274
|
Reduces trainable parameters by ~97% while often matching full fine-tuning:
|
|
870
1275
|
|
|
1276
|
+
<!-- genml-cli-test
|
|
1277
|
+
name: fine-tune-lora
|
|
1278
|
+
network: true
|
|
1279
|
+
extras: lora
|
|
1280
|
+
override:
|
|
1281
|
+
- --dataset {tmp}/img
|
|
1282
|
+
- --label_column label
|
|
1283
|
+
extra: --epochs 1
|
|
1284
|
+
-->
|
|
871
1285
|
```bash
|
|
872
1286
|
genml-kit-train \
|
|
873
1287
|
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
1288
|
+
--model_arg classifier=mlp:hidden=512 \
|
|
874
1289
|
--lora --lora_r 16 --lora_alpha 32 \
|
|
875
|
-
--lora_target_modules "query,key,value" \
|
|
876
1290
|
--freeze "classifier\.(head|pool|encoder)" \
|
|
877
1291
|
--lr 3e-5 \
|
|
878
1292
|
--dataset my-org/my-labeled-images \
|
|
@@ -884,6 +1298,14 @@ genml-kit-train \
|
|
|
884
1298
|
|
|
885
1299
|
Load encoder weights from a pre-training run (SimMIM, I-JEPA, or SupCon):
|
|
886
1300
|
|
|
1301
|
+
<!-- genml-cli-test
|
|
1302
|
+
name: fine-tune-from-checkpoint
|
|
1303
|
+
override:
|
|
1304
|
+
- --dataset {tmp}/img
|
|
1305
|
+
- --label_column label
|
|
1306
|
+
- -.--source_checkpoint
|
|
1307
|
+
extra: --epochs 1
|
|
1308
|
+
-->
|
|
887
1309
|
```bash
|
|
888
1310
|
genml-kit-train --model convvit \
|
|
889
1311
|
--dataset my-org/my-labeled-images \
|
|
@@ -939,6 +1361,88 @@ the model settle into a minimum instead of bouncing around it. Late-training
|
|
|
939
1361
|
low learning rates are also what make the EMA teacher of DINO/BYOL
|
|
940
1362
|
effective: the teacher tracks a student that changes gently.
|
|
941
1363
|
|
|
1364
|
+
### Warmup
|
|
1365
|
+
|
|
1366
|
+
Transformer fine-tuning is unstable at the step-off: the randomly initialised
|
|
1367
|
+
head meets a pre-trained backbone with full force. The three warmup
|
|
1368
|
+
schedulers ramp the learning rate up linearly from zero over `--warmup_epochs`
|
|
1369
|
+
epochs (an integer or fraction) before handing over to the chosen decay
|
|
1370
|
+
shape:
|
|
1371
|
+
|
|
1372
|
+
<!-- genml-cli-test
|
|
1373
|
+
name: warmup-scheduler
|
|
1374
|
+
override:
|
|
1375
|
+
- --dataset {tmp}/img
|
|
1376
|
+
- --label_column label
|
|
1377
|
+
extra: --epochs 2 --max_items 16
|
|
1378
|
+
-->
|
|
1379
|
+
```bash
|
|
1380
|
+
genml-kit-train --model convvit --method classification \
|
|
1381
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1382
|
+
--scheduler warmup_cosine --warmup_epochs 2.5 \
|
|
1383
|
+
--epochs 20 --batch_size 32
|
|
1384
|
+
```
|
|
1385
|
+
|
|
1386
|
+
`--warmup_epochs 0` (the default) disables the ramp and every `warmup_*`
|
|
1387
|
+
name then behaves exactly like its plain shape. Available shapes:
|
|
1388
|
+
|
|
1389
|
+
| Name | Shape after warmup |
|
|
1390
|
+
|---|---|
|
|
1391
|
+
| `warmup_cosine` | half-cosine decay to `eta_min` (0), as above |
|
|
1392
|
+
| `warmup_linear` | linear decay to 0 over `--epochs` |
|
|
1393
|
+
| `warmup_rsqrt` | inverse square-root decay |
|
|
1394
|
+
|
|
1395
|
+
A `--sched_arg warmup_epochs=...` override wins over the generic flag, and
|
|
1396
|
+
`--sched_arg T_max=...` overrides the cosine total (the epoch of the first
|
|
1397
|
+
minimum). Plain torch names (`CosineAnnealingLR`, ...) ignore the warmup
|
|
1398
|
+
flag entirely, so existing invocations are unchanged.
|
|
1399
|
+
|
|
1400
|
+
### Weights EMA
|
|
1401
|
+
|
|
1402
|
+
Alongside the *loss-coupled* EMA teachers (DINO/BYOL/I-JEPA, and
|
|
1403
|
+
`--teacher_kind ema` in distillation), the trainer can maintain an
|
|
1404
|
+
*eval-time* shadow of the trained weights: after every optimizer step the
|
|
1405
|
+
shadow blends toward the live weights (`shadow <- decay * shadow +
|
|
1406
|
+
(1 - decay) * model`), damping the step-to-step noise of the optimisation
|
|
1407
|
+
trajectory. Validation and the best checkpoint then run on the shadow
|
|
1408
|
+
instead of the raw weights -- the classic timm / MMDetection weights-EMA,
|
|
1409
|
+
which typically buys a fraction of a point on noisy runs.
|
|
1410
|
+
|
|
1411
|
+
The feature is **opt-in per method**: only the method knows whether a
|
|
1412
|
+
shadow of "the model" is meaningful for its topology (DPO owns a frozen
|
|
1413
|
+
reference tower, distillation owns teacher towers -- neither is EMA
|
|
1414
|
+
material). Methods opt in by inheriting `WeightsEMAMixin`
|
|
1415
|
+
(`ClassificationMethod` does); methods without the mixin never see the
|
|
1416
|
+
feature, and setting `--ema_decay` on one logs a warning instead of
|
|
1417
|
+
silently doing nothing. The RL trainer shares the loop and simply never
|
|
1418
|
+
wires a shadow. The `--ema_*` flags are declared by the mixin's
|
|
1419
|
+
`add_args`, which adopting methods chain into their own `--help` group,
|
|
1420
|
+
so the flags render inside "classification method".
|
|
1421
|
+
|
|
1422
|
+
<!-- genml-cli-test
|
|
1423
|
+
name: weights-ema
|
|
1424
|
+
override:
|
|
1425
|
+
- --model convvit
|
|
1426
|
+
- --dataset {tmp}/img
|
|
1427
|
+
- --label_column label
|
|
1428
|
+
extra: --epochs 1 --max_items 16
|
|
1429
|
+
-->
|
|
1430
|
+
```bash
|
|
1431
|
+
genml-kit-train --model google/vit-base-patch16-224 --method classification \
|
|
1432
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1433
|
+
--ema_decay 0.9995 --ema_eval \
|
|
1434
|
+
--epochs 5 --batch_size 32
|
|
1435
|
+
```
|
|
1436
|
+
|
|
1437
|
+
`--ema_decay 0` (the default) keeps the run exactly as before. With
|
|
1438
|
+
`--ema_eval`, validation and `best.pt` carry the EMA weights while
|
|
1439
|
+
`latest.pt` keeps the raw trajectory plus the shadow in `ema_state`, so an
|
|
1440
|
+
interrupted run resumes exactly. `--ema_final_decay` ramps the effective
|
|
1441
|
+
decay linearly over the run (the DINO-style momentum schedule) and
|
|
1442
|
+
`--ema_warmup_steps` ramps it up from zero; when both are absent the decay
|
|
1443
|
+
is constant. `run.json` records the decay, so provenance of "which weights
|
|
1444
|
+
the best checkpoint holds" is unambiguous.
|
|
1445
|
+
|
|
942
1446
|
### Mixup, Label Smoothing, And Imbalance
|
|
943
1447
|
|
|
944
1448
|
Mixup forms a virtual example from two training examples.
|
|
@@ -998,7 +1502,7 @@ $$
|
|
|
998
1502
|
|
|
999
1503
|
Two details matter for the reader combining these flags:
|
|
1000
1504
|
|
|
1001
|
-
- **Unified $`p_t
|
|
1505
|
+
- **Unified $`p_t`$.** The modulating factor uses the *expected* probability
|
|
1002
1506
|
under the target distribution, $`p_t = \sum_c q_c p_c`$, rather than the
|
|
1003
1507
|
probability of a single class. For a one-hot target $`q`$ this reduces
|
|
1004
1508
|
exactly to the basic form above ($`p_t`$ becomes the correct-class
|
|
@@ -1014,6 +1518,91 @@ Two details matter for the reader combining these flags:
|
|
|
1014
1518
|
Use these tools deliberately; combining every regularizer is not
|
|
1015
1519
|
automatically better.
|
|
1016
1520
|
|
|
1521
|
+
### Data Augmentation
|
|
1522
|
+
|
|
1523
|
+
By default the `classification` method augments the training set with the
|
|
1524
|
+
chain built by `build_transforms` in
|
|
1525
|
+
`genml_kit/training/train_compat.py`:
|
|
1526
|
+
|
|
1527
|
+
```python
|
|
1528
|
+
import torch
|
|
1529
|
+
from torchvision.transforms import v2
|
|
1530
|
+
|
|
1531
|
+
train_augmentations = v2.Compose([
|
|
1532
|
+
# Randomly crop 20-100% of the image, then resize to image_size.
|
|
1533
|
+
v2.RandomResizedCrop(size=(image_size, image_size),
|
|
1534
|
+
scale=(0.2, 1.0),
|
|
1535
|
+
antialias=True),
|
|
1536
|
+
# Mirror left/right, applied half the time.
|
|
1537
|
+
v2.RandomHorizontalFlip(p=0.5),
|
|
1538
|
+
# Mirror top/bottom, applied half the time.
|
|
1539
|
+
v2.RandomVerticalFlip(p=0.5),
|
|
1540
|
+
# Rotate by an arbitrary angle.
|
|
1541
|
+
v2.RandomRotation(degrees=360),
|
|
1542
|
+
# Photometric jitter on brightness, contrast, saturation and hue.
|
|
1543
|
+
v2.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.15, hue=0.05),
|
|
1544
|
+
# Local elastic warping.
|
|
1545
|
+
v2.ElasticTransform(alpha=50.0, sigma=5.0),
|
|
1546
|
+
# Fixed tail, always appended automatically:
|
|
1547
|
+
# PIL image -> tensor.
|
|
1548
|
+
v2.ToImage(),
|
|
1549
|
+
# uint8 -> float in [0, 1].
|
|
1550
|
+
v2.ToDtype(torch.float32, scale=True),
|
|
1551
|
+
# Model-specific normalization.
|
|
1552
|
+
v2.Normalize(mean=processor.image_mean, std=processor.image_std),
|
|
1553
|
+
])
|
|
1554
|
+
```
|
|
1555
|
+
|
|
1556
|
+
The validation chain is never augmented: it is always `Resize` →
|
|
1557
|
+
`CenterCrop` → `Normalize`.
|
|
1558
|
+
|
|
1559
|
+
#### Custom Training Augmentation
|
|
1560
|
+
|
|
1561
|
+
To override the training chain, pass `--train_augmentation_script <path>.py`
|
|
1562
|
+
(a local path or an HTTP/HTTPS URL). The script is imported lazily, so a
|
|
1563
|
+
plain run never loads it. It must define
|
|
1564
|
+
|
|
1565
|
+
```python
|
|
1566
|
+
def create_train_transform(image_size: int):
|
|
1567
|
+
...
|
|
1568
|
+
```
|
|
1569
|
+
|
|
1570
|
+
for example:
|
|
1571
|
+
|
|
1572
|
+
```python
|
|
1573
|
+
# my_aug.py
|
|
1574
|
+
from torchvision.transforms import v2
|
|
1575
|
+
|
|
1576
|
+
def create_train_transform(image_size):
|
|
1577
|
+
return [
|
|
1578
|
+
v2.RandomResizedCrop(image_size, scale=(0.4, 1.0), antialias=True),
|
|
1579
|
+
v2.RandomHorizontalFlip(p=0.5),
|
|
1580
|
+
v2.ColorJitter(brightness=0.3, contrast=0.3),
|
|
1581
|
+
]
|
|
1582
|
+
```
|
|
1583
|
+
|
|
1584
|
+
The contract is:
|
|
1585
|
+
|
|
1586
|
+
- The function is called once as `create_train_transform(image_size)` and
|
|
1587
|
+
**must return a `list`** of transforms. Any other return type raises a
|
|
1588
|
+
`TypeError` (`create_train_transform() must return a list of transforms,
|
|
1589
|
+
got <type>`).
|
|
1590
|
+
- The returned list is composed as `v2.Compose(user_transforms + tail)`,
|
|
1591
|
+
where the fixed tail above (`ToImage` → `ToDtype` → `Normalize`) is
|
|
1592
|
+
**always appended automatically**. Your transforms therefore run on the
|
|
1593
|
+
**raw PIL image**, and you must **not** add `ToImage`, `ToDtype`, or
|
|
1594
|
+
`Normalize` yourself.
|
|
1595
|
+
- This differs from the segmentation method's `create_seg_transform`, which
|
|
1596
|
+
owns the whole image + mask chain *including* normalization. For
|
|
1597
|
+
classification, normalization stays under the processor's control.
|
|
1598
|
+
- Only the **train** chain is replaced; validation is unaffected. Use
|
|
1599
|
+
`--tta` for test-time augmentation (see
|
|
1600
|
+
[XGBoost And Test-Time Augmentation](#xgboost-and-test-time-augmentation)).
|
|
1601
|
+
|
|
1602
|
+
Modules placed next to the script (or importable via `import` from the same
|
|
1603
|
+
directory) resolve by plain name, so a custom transform can live in a
|
|
1604
|
+
sibling file.
|
|
1605
|
+
|
|
1017
1606
|
### Hyperparameter Guidance
|
|
1018
1607
|
|
|
1019
1608
|
| Scenario | `--lr` | `--epochs` | `--batch_size` | Notes |
|
|
@@ -1050,8 +1639,17 @@ Two levels of determinism:
|
|
|
1050
1639
|
Passing an explicit `--seed` therefore buys full determinism at a
|
|
1051
1640
|
throughput cost (typically 10–20%); omit it for maximum speed:
|
|
1052
1641
|
|
|
1642
|
+
<!-- genml-cli-test
|
|
1643
|
+
name: reproducibility-seed
|
|
1644
|
+
override:
|
|
1645
|
+
- --dataset {tmp}/img
|
|
1646
|
+
- --label_column label
|
|
1647
|
+
extra: --epochs 1 --batch_size 4 --max_items 16
|
|
1648
|
+
-->
|
|
1053
1649
|
```bash
|
|
1054
|
-
genml-kit-train --model convvit --seed 42
|
|
1650
|
+
genml-kit-train --model convvit --seed 42 \
|
|
1651
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1652
|
+
--epochs 5 --batch_size 32
|
|
1055
1653
|
```
|
|
1056
1654
|
|
|
1057
1655
|
The train/val split and the XGBoost stage follow the user's seed: with
|
|
@@ -1071,6 +1669,67 @@ Checkpointing is atomic: `_latest.pt` is written to a temporary file
|
|
|
1071
1669
|
and renamed into place, so an interrupted run never leaves a truncated
|
|
1072
1670
|
resume point.
|
|
1073
1671
|
|
|
1672
|
+
### Run Manifest
|
|
1673
|
+
|
|
1674
|
+
Every training run writes `<log_dir>/run.json` — a reproducibility
|
|
1675
|
+
snapshot of *how* the run was launched: the fully resolved argparse
|
|
1676
|
+
namespace (defaults filled in), the git SHA of the checkout, the
|
|
1677
|
+
`torch` / `transformers` versions, hostname, device, resolved seeds, and
|
|
1678
|
+
timestamp. Together with `metrics.jsonl` this makes any result traceable
|
|
1679
|
+
back to the exact code and invocation that produced it:
|
|
1680
|
+
|
|
1681
|
+
<!-- genml-cli-test-exclude: illustrative ellipsis, no dataset; not a runnable command -->
|
|
1682
|
+
```bash
|
|
1683
|
+
genml-kit-train --model convvit --method classification ...
|
|
1684
|
+
cat <checkpoint_dir>/logs/run.json
|
|
1685
|
+
```
|
|
1686
|
+
|
|
1687
|
+
The same facts are emitted to the metric sinks as `manifest/*` and
|
|
1688
|
+
`args/<name>` run-facts, so they land in TensorBoard and any configured
|
|
1689
|
+
sink alongside the metrics. `--dry_run` logs the manifest as JSON and
|
|
1690
|
+
exits before the dataset is touched — a fast way to inspect what an
|
|
1691
|
+
invocation would actually do:
|
|
1692
|
+
|
|
1693
|
+
<!-- genml-cli-test
|
|
1694
|
+
name: run-manifest-dry-run
|
|
1695
|
+
-->
|
|
1696
|
+
```bash
|
|
1697
|
+
genml-kit-train --model convvit --method classification --dry_run
|
|
1698
|
+
```
|
|
1699
|
+
|
|
1700
|
+
### Metric Sinks
|
|
1701
|
+
|
|
1702
|
+
`--metrics_sinks` selects where training metrics go, as a comma-separated
|
|
1703
|
+
fan-out: `tb` (TensorBoard event files), `jsonl` (one JSON record per line in
|
|
1704
|
+
`<log_dir>/metrics.jsonl` — the file the search subsystem reads), and `wandb`
|
|
1705
|
+
(a live Weights & Biases run). The default `tb,jsonl` is unchanged; sinks
|
|
1706
|
+
are independent, so any combination works:
|
|
1707
|
+
|
|
1708
|
+
<!-- genml-cli-test
|
|
1709
|
+
name: metric-sinks
|
|
1710
|
+
extras: wandb
|
|
1711
|
+
override:
|
|
1712
|
+
- --dataset {tmp}/img
|
|
1713
|
+
- --label_column label
|
|
1714
|
+
extra: --epochs 1 --max_items 16 --wandb_mode disabled
|
|
1715
|
+
-->
|
|
1716
|
+
```bash
|
|
1717
|
+
genml-kit-train --model convvit --method classification \
|
|
1718
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
1719
|
+
--metrics_sinks tb,jsonl,wandb --wandb_project my-team \
|
|
1720
|
+
--epochs 5 --batch_size 32
|
|
1721
|
+
```
|
|
1722
|
+
|
|
1723
|
+
The `wandb` sink requires the optional extra (`pip install
|
|
1724
|
+
"genml_kit[wandb]"`); the run name defaults to `--run_name` when given, the
|
|
1725
|
+
run finishes on exit, and the fully resolved args are uploaded as the run
|
|
1726
|
+
config. `--wandb_mode` selects how the run is recorded: `online` (default,
|
|
1727
|
+
syncs to the W&B service), `offline` (a local `.wandb` transaction log,
|
|
1728
|
+
synced later with `wandb sync` — no network or credentials needed), or
|
|
1729
|
+
`disabled` (wandb's no-op stub). New sinks implement the same tiny
|
|
1730
|
+
duck-typed protocol (`scalar` / `image` / `video` / `run_fact` / `close`) in
|
|
1731
|
+
`genml_kit/training/metrics_sinks.py`.
|
|
1732
|
+
|
|
1074
1733
|
### Fine-Tuning CLI Reference
|
|
1075
1734
|
|
|
1076
1735
|
| Argument | Default | Description |
|
|
@@ -1106,7 +1765,12 @@ resume point.
|
|
|
1106
1765
|
| `--num_workers` | `2` | DataLoader worker processes. |
|
|
1107
1766
|
| `--log_level` | `INFO` | Logging level. |
|
|
1108
1767
|
| `--log_targets` | `STDERR` | Comma-separated log destinations. `STDERR` logs to standard error; any other entry is a log file path (appended). Example: `STDERR,/tmp/train.log` logs to both. |
|
|
1109
|
-
| `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package
|
|
1768
|
+
| `--log_dir` | `None` | TensorBoard log directory (default: `<checkpoint_dir>/logs`). Requires the `tensorboard` package. |
|
|
1769
|
+
| `--metrics_sinks` | `tb,jsonl` | Comma-separated metric sinks: `tb` (TensorBoard), `jsonl` (`<log_dir>/metrics.jsonl`), `wandb` (Weights & Biases, requires `pip install "genml_kit[wandb]"`), `none`. |
|
|
1770
|
+
| `--wandb_project` | `None` | W&B project name when `wandb` is in `--metrics_sinks` (default: `genml-kit`). |
|
|
1771
|
+
| `--wandb_mode` | `online` | W&B run mode: `online` (sync to the service), `offline` (local `.wandb` log, sync later with `wandb sync`), `disabled` (stub, no network/files). |
|
|
1772
|
+
| `--run_name` | `None` | W&B run name when `wandb` is in `--metrics_sinks`. |
|
|
1773
|
+
| `--fit_temperature` | `False` | After training, fit a temperature-scaling T on the validation set of the best checkpoint and write `temperature.json` next to it. Classification methods only. |
|
|
1110
1774
|
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
1111
1775
|
| `--remote_checkpoint` | `None` | Remote URI for checkpoint sync (`gs://BUCKET/PREFIX`, `r2://BUCKET/PREFIX`, or `s3://BUCKET/PREFIX`). |
|
|
1112
1776
|
| `--source_checkpoint` | `None` | Path to source checkpoint to absorb parameters from. |
|
|
@@ -1116,11 +1780,12 @@ resume point.
|
|
|
1116
1780
|
| `--lora_r` | `8` | LoRA rank. |
|
|
1117
1781
|
| `--lora_alpha` | `16` | LoRA alpha (scaling = `alpha / r`). |
|
|
1118
1782
|
| `--lora_dropout` | `0.0` | Dropout on LoRA layers. |
|
|
1119
|
-
| `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"
|
|
1783
|
+
| `--lora_target_modules` | `None` | Comma-separated module names for LoRA (e.g. `"q_proj,k_proj,v_proj"`). When omitted, targets are resolved from the model (see LoRA section). |
|
|
1120
1784
|
| `--optimizer` | `AdamW` | `torch.optim` optimizer class name or `.py` script path. |
|
|
1121
1785
|
| `--opt_arg` | `{}` | Extra optimizer kwargs (repeatable). |
|
|
1122
|
-
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path
|
|
1786
|
+
| `--scheduler` | `None` | `torch.optim.lr_scheduler` class name or `.py` script path, or a built-in warmup shape: `warmup_cosine`, `warmup_linear`, `warmup_rsqrt`. |
|
|
1123
1787
|
| `--sched_arg` | `{}` | Extra scheduler kwargs (repeatable). |
|
|
1788
|
+
| `--warmup_epochs` | `0` | Linear LR ramp length in epochs (integer or fraction) for the `warmup_*` schedulers; 0 disables the ramp. See [Warmup](#warmup). |
|
|
1124
1789
|
| `--state_save` | `opt,sched,amp` | States to save: `opt`, `sched`, `amp`, `none`. |
|
|
1125
1790
|
| `--state_load` | `opt,sched,amp` | States to restore on resume. |
|
|
1126
1791
|
| `--xgboost_model` | `None` | Output path for XGBoost model (trains after PyTorch). |
|
|
@@ -1134,6 +1799,15 @@ resume point.
|
|
|
1134
1799
|
Training automatically resumes from an existing `_latest.pt` or `_best.pt`
|
|
1135
1800
|
checkpoint if one exists at the `--checkpoint` path.
|
|
1136
1801
|
|
|
1802
|
+
Alongside the epoch-end and exit saves, `--save_every` writes `_latest.pt`
|
|
1803
|
+
every N optimizer steps (grad-accum flush boundary) so a hard kill between
|
|
1804
|
+
epochs does not lose the whole epoch. When such a checkpoint carries the epoch
|
|
1805
|
+
in progress, resume **continues** it rather than replaying it: supervised runs
|
|
1806
|
+
fast-forward to the recorded step, and PPO skips re-collecting the rollout and
|
|
1807
|
+
re-enters the optimize sweep (a resume taken mid-collection re-runs the epoch,
|
|
1808
|
+
since the environment is not persisted). Off-policy RL (DQN/SAC) restarts the
|
|
1809
|
+
epoch: the replay buffer is not checkpointed.
|
|
1810
|
+
|
|
1137
1811
|
### Remote Checkpoint Sync (GCS / R2 / S3)
|
|
1138
1812
|
|
|
1139
1813
|
`--remote_checkpoint` uploads each saved checkpoint to cloud storage.
|
|
@@ -1209,6 +1883,15 @@ Only $`A`$, $`B`$ (and the classifier head) receive gradients.
|
|
|
1209
1883
|
|
|
1210
1884
|
LoRA can be combined with a custom classifier:
|
|
1211
1885
|
|
|
1886
|
+
<!-- genml-cli-test
|
|
1887
|
+
name: lora-custom-classifier
|
|
1888
|
+
network: true
|
|
1889
|
+
extras: lora
|
|
1890
|
+
override:
|
|
1891
|
+
- --dataset {tmp}/img
|
|
1892
|
+
- --label_column label
|
|
1893
|
+
extra: --epochs 1
|
|
1894
|
+
-->
|
|
1212
1895
|
```bash
|
|
1213
1896
|
genml-kit-train \
|
|
1214
1897
|
--model cls_model_wrapper:facebook/dinov2-with-registers-large \
|
|
@@ -1216,13 +1899,23 @@ genml-kit-train \
|
|
|
1216
1899
|
--lora --lora_r 16 --lora_alpha 32 \
|
|
1217
1900
|
--freeze 'classifier\.(head|pool|encoder)' \
|
|
1218
1901
|
--lr_group 'backbone.*=1e-5' 'classifier.*=3e-4' \
|
|
1219
|
-
|
|
1902
|
+
--dataset my-org/my-labeled-images \
|
|
1903
|
+
--label_column category \
|
|
1904
|
+
--epochs 20
|
|
1220
1905
|
```
|
|
1221
1906
|
|
|
1222
1907
|
### Cross-Dataset Resume
|
|
1223
1908
|
|
|
1224
1909
|
Switch from one dataset to another while keeping backbone weights:
|
|
1225
1910
|
|
|
1911
|
+
<!-- genml-cli-test
|
|
1912
|
+
name: cross-dataset-resume
|
|
1913
|
+
network: true
|
|
1914
|
+
override:
|
|
1915
|
+
- --dataset {tmp}/img
|
|
1916
|
+
- --label_column label
|
|
1917
|
+
extra: --epochs 1
|
|
1918
|
+
-->
|
|
1226
1919
|
```bash
|
|
1227
1920
|
genml-kit-train --model facebook/convnextv2-base-22k-224 \
|
|
1228
1921
|
--dataset my-org/other-labeled-images \
|
|
@@ -1248,14 +1941,24 @@ limits the maximum batch size even on a 24 GB GPU. Gradient checkpointing
|
|
|
1248
1941
|
trades compute for memory by discarding intermediate activations during the
|
|
1249
1942
|
forward pass and recomputing them during the backward pass.
|
|
1250
1943
|
|
|
1944
|
+
<!-- genml-cli-test
|
|
1945
|
+
name: gradient-checkpointing
|
|
1946
|
+
network: true
|
|
1947
|
+
override:
|
|
1948
|
+
- --dataset {tmp}/img
|
|
1949
|
+
- --label_column label
|
|
1950
|
+
extra: --epochs 1
|
|
1951
|
+
-->
|
|
1251
1952
|
```bash
|
|
1252
1953
|
genml-kit-train \
|
|
1253
1954
|
--model timm:eva02_base_patch14_448.mim_in22k_ft_in22k_in1k \
|
|
1955
|
+
--dataset my-org/my-labeled-images \
|
|
1956
|
+
--label_column category \
|
|
1254
1957
|
--batch_size 32 \
|
|
1255
1958
|
--grad_accum_steps 2 \
|
|
1256
1959
|
--grad_checkpoint \
|
|
1257
1960
|
--amp_dtype bfloat16 \
|
|
1258
|
-
|
|
1961
|
+
--epochs 20
|
|
1259
1962
|
```
|
|
1260
1963
|
|
|
1261
1964
|
**What it does:** Each transformer block stores only its input and output.
|
|
@@ -1282,10 +1985,27 @@ HuggingFace, ConvViT, and UVito — via native APIs or per-block
|
|
|
1282
1985
|
|
|
1283
1986
|
Run inference on individual images:
|
|
1284
1987
|
|
|
1988
|
+
<!-- genml-cli-test
|
|
1989
|
+
name: inference-guide
|
|
1990
|
+
network: true
|
|
1991
|
+
override:
|
|
1992
|
+
- --checkpoint {tmp}/chk_infer/model_best.pt
|
|
1993
|
+
- --model convvit --image_size 32
|
|
1994
|
+
extra: --top_k 3 --image_size 32
|
|
1995
|
+
-->
|
|
1285
1996
|
```bash
|
|
1286
1997
|
genml-kit-infer --model facebook/convnextv2-base-22k-224 \
|
|
1287
1998
|
--checkpoint genml_kit_best.pt \
|
|
1288
1999
|
path/to/image.jpg path/to/other_image.png
|
|
2000
|
+
|
|
2001
|
+
# Structured output and an XGBoost head on the backbone features (the .json
|
|
2002
|
+
# model is the one trained by `train_xgboost_on_backbone`, see
|
|
2003
|
+
# tests/test_xgb_pipeline.py).
|
|
2004
|
+
genml-kit-infer --model facebook/convnextv2-base-22k-224 \
|
|
2005
|
+
--checkpoint genml_kit_best.pt \
|
|
2006
|
+
--xgboost_model path/to/xgboost_model.json \
|
|
2007
|
+
--output results.json \
|
|
2008
|
+
path/to/image.jpg
|
|
1289
2009
|
```
|
|
1290
2010
|
|
|
1291
2011
|
Output is JSON with per-class probabilities:
|
|
@@ -1306,7 +2026,7 @@ The neural classifier makes decisions through its head. The XGBoost option
|
|
|
1306
2026
|
takes the encoder representation $`h = f_\theta(x)`$ instead and fits an
|
|
1307
2027
|
ensemble of decision trees to those vectors.
|
|
1308
2028
|
Reference: Chen & Guestrin, [XGBoost: A Scalable Tree Boosting System](https://arxiv.org/abs/1603.02754), KDD 2016.
|
|
1309
|
-
A tree partitions feature space with rules such as $`h_{37}
|
|
2029
|
+
A tree partitions feature space with rules such as $`h_{37} \lt t`$ (comparing
|
|
1310
2030
|
the 37th feature of $`h`$ against a learned threshold $`t`$); boosting adds
|
|
1311
2031
|
trees sequentially so each new tree focuses on errors left by previous trees.
|
|
1312
2032
|
This can work well when the dataset is small
|
|
@@ -1337,6 +2057,7 @@ does not erase the cue that distinguishes the classes.
|
|
|
1337
2057
|
| `--model` | (required) | HuggingFace model name or custom model. |
|
|
1338
2058
|
| `--checkpoint` | (required) | Path to state dict or wrapped checkpoint. |
|
|
1339
2059
|
| `--top_k` | `None` | Show top-K predictions; omit for all classes. |
|
|
2060
|
+
| `--temperature` | `1.0` | Divide logits by this before the softmax (1.0 = unchanged). Use the value written to `temperature.json` by `--fit_temperature`. |
|
|
1340
2061
|
| `--output` | `None` | Write JSON results to file. |
|
|
1341
2062
|
| `--device` | `None` | Force PyTorch device (`cuda`, `cpu`). Auto-detected if omitted. |
|
|
1342
2063
|
| `--cache_dir` | `None` | HuggingFace cache directory. |
|
|
@@ -1372,12 +2093,26 @@ training run. `--source_checkpoint` imports weights from another run before
|
|
|
1372
2093
|
the new optimizer is created. When moving a SupCon encoder into classification,
|
|
1373
2094
|
use:
|
|
1374
2095
|
|
|
2096
|
+
<!-- genml-cli-test
|
|
2097
|
+
name: param-rename
|
|
2098
|
+
network: true
|
|
2099
|
+
override:
|
|
2100
|
+
- --dataset {tmp}/img
|
|
2101
|
+
- --label_column label
|
|
2102
|
+
- --checkpoint {tmp}/ckpt_param_rename
|
|
2103
|
+
- -.--source_checkpoint
|
|
2104
|
+
extra: --epochs 1
|
|
2105
|
+
-->
|
|
1375
2106
|
```bash
|
|
1376
2107
|
genml-kit-train \
|
|
2108
|
+
--model google/vit-base-patch16-224 \
|
|
2109
|
+
--image_size 224 \
|
|
1377
2110
|
--checkpoint /content/eva02_finetune \
|
|
1378
2111
|
--source_checkpoint /content/eva02_supcon_latest.pt \
|
|
1379
2112
|
--param_rename 'encoder\\.model\\.(.*);model.$1' \
|
|
1380
|
-
|
|
2113
|
+
--dataset my-org/my-labeled-images \
|
|
2114
|
+
--label_column category \
|
|
2115
|
+
--epochs 20
|
|
1381
2116
|
```
|
|
1382
2117
|
|
|
1383
2118
|
The rename is needed because the pre-training wrapper stores backbone keys
|
|
@@ -1417,7 +2152,10 @@ The periodic `GPU:` line reports memory in MiB and SM utilization as a
|
|
|
1417
2152
|
percentage sampled at log time (`mem=allocated/totalMiB (pct) res=…MiB
|
|
1418
2153
|
util=…%`). `mem` counts live tensors, `res` is the caching allocator's
|
|
1419
2154
|
pool, and `util` is one instant — not an average — so an isolated 0 is
|
|
1420
|
-
normal.
|
|
2155
|
+
normal. The `util=` field needs the optional NVML bindings
|
|
2156
|
+
(`pip install "genml_kit[gpu]"`); when they are absent, `util=` is simply
|
|
2157
|
+
omitted from the `GPU:` line — the run is never affected.
|
|
2158
|
+
A *persistent* pattern of `util=0` at every log line, with
|
|
1421
2159
|
`mem`/`res` byte-identical between samples, means the GPU finished the
|
|
1422
2160
|
last batch and is idle waiting for the next one: the run is **input
|
|
1423
2161
|
bound**, not compute bound. The throughput line (`img/s=`) is the ground
|
|
@@ -1536,9 +2274,20 @@ def build_loss(alpha=1.0, **kwargs):
|
|
|
1536
2274
|
return MyLoss(alpha=alpha)
|
|
1537
2275
|
```
|
|
1538
2276
|
|
|
2277
|
+
<!-- genml-cli-test
|
|
2278
|
+
name: custom-loss
|
|
2279
|
+
network: true
|
|
2280
|
+
override:
|
|
2281
|
+
- --dataset {tmp}/img
|
|
2282
|
+
- --label_column label
|
|
2283
|
+
extra: --epochs 1
|
|
2284
|
+
-->
|
|
1539
2285
|
```bash
|
|
1540
2286
|
genml-kit-train --model google/vit-base-patch16-224 \
|
|
1541
|
-
--
|
|
2287
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
2288
|
+
--image_size 224 \
|
|
2289
|
+
--loss ./my_loss.py --loss_args '{"alpha": 2.0}' \
|
|
2290
|
+
--epochs 20
|
|
1542
2291
|
```
|
|
1543
2292
|
|
|
1544
2293
|
**RL losses** work the same way but return a *bundle* (dict of
|
|
@@ -1552,8 +2301,14 @@ def build_loss(**kwargs):
|
|
|
1552
2301
|
return {"entropy": lambda ent: ent.mean() * 2.0} # override only entropy
|
|
1553
2302
|
```
|
|
1554
2303
|
|
|
2304
|
+
<!-- genml-cli-test
|
|
2305
|
+
name: rl-custom-loss
|
|
2306
|
+
extras: rl
|
|
2307
|
+
extra: --epochs 1
|
|
2308
|
+
-->
|
|
1555
2309
|
```bash
|
|
1556
|
-
genml-kit-train --method ppo --
|
|
2310
|
+
genml-kit-train --pipeline rl --method ppo --env_id CartPole-v1 \
|
|
2311
|
+
--loss ./my_ppo_entropy.py
|
|
1557
2312
|
```
|
|
1558
2313
|
|
|
1559
2314
|
The built-in bundles are merged over (`ppo`: `policy`/`value`/`entropy`;
|
|
@@ -1610,8 +2365,18 @@ def create_model(num_labels, image_size=224, id2label=None, label2id=None,
|
|
|
1610
2365
|
return Tiny(num_labels=num_labels, hidden=kwargs.get("hidden", 128))
|
|
1611
2366
|
```
|
|
1612
2367
|
|
|
2368
|
+
<!-- genml-cli-test
|
|
2369
|
+
name: custom-model
|
|
2370
|
+
network: true
|
|
2371
|
+
override:
|
|
2372
|
+
- --dataset {tmp}/img
|
|
2373
|
+
- --label_column label
|
|
2374
|
+
extra: --epochs 1
|
|
2375
|
+
-->
|
|
1613
2376
|
```bash
|
|
1614
|
-
genml-kit-train --model ./my_model.py --loss focal
|
|
2377
|
+
genml-kit-train --model ./my_model.py --loss focal \
|
|
2378
|
+
--dataset my-org/my-labeled-images --label_column category \
|
|
2379
|
+
--epochs 20
|
|
1615
2380
|
```
|
|
1616
2381
|
|
|
1617
2382
|
The script is **fully responsible for weight loading**: when
|
|
@@ -1635,11 +2400,13 @@ class Classifier(nn.Module):
|
|
|
1635
2400
|
super().__init__()
|
|
1636
2401
|
self.head = nn.Linear(hidden_size, num_labels)
|
|
1637
2402
|
|
|
1638
|
-
def forward(self, hidden_states):
|
|
2403
|
+
def forward(self, hidden_states):
|
|
2404
|
+
# (B, N, D) tensor.
|
|
1639
2405
|
features = hidden_states[:, 0] # CLS token
|
|
1640
2406
|
return self.head(features)
|
|
1641
2407
|
|
|
1642
|
-
def extract_features(self, hidden_states):
|
|
2408
|
+
def extract_features(self, hidden_states):
|
|
2409
|
+
# (B, N, D) → (B, D).
|
|
1643
2410
|
return hidden_states[:, 0]
|
|
1644
2411
|
```
|
|
1645
2412
|
|
|
@@ -1665,6 +2432,855 @@ head before loading the checkpoint weights.
|
|
|
1665
2432
|
|
|
1666
2433
|
---
|
|
1667
2434
|
|
|
2435
|
+
## Semantic Segmentation
|
|
2436
|
+
|
|
2437
|
+
Segmentation predicts a class for **every pixel**: an image `(B, 3, H, W)`
|
|
2438
|
+
goes in and per-pixel logits `(B, num_classes, H, W)` come out. `genml_kit`
|
|
2439
|
+
models this as `encoder + decoder`: the encoder is any hierarchical timm
|
|
2440
|
+
backbone (ResNet / ConvNeXt / Swin / MiT) emitting a feature pyramid, and
|
|
2441
|
+
the decoder fuses that pyramid back up to full resolution (with skip
|
|
2442
|
+
connections for `unet`/`fpn`).
|
|
2443
|
+
|
|
2444
|
+
### Quick start
|
|
2445
|
+
|
|
2446
|
+
<!-- genml-cli-test
|
|
2447
|
+
name: segmentation-quick-start
|
|
2448
|
+
network: true
|
|
2449
|
+
override:
|
|
2450
|
+
- --dataset segfolder/{tmp}/seg
|
|
2451
|
+
extra: --epochs 1
|
|
2452
|
+
-->
|
|
2453
|
+
```bash
|
|
2454
|
+
genml-kit-train \
|
|
2455
|
+
--pipeline segmentation --method segmentation \
|
|
2456
|
+
--dataset segfolder//data/voc2012 \
|
|
2457
|
+
--model seg_wrapper:timm/resnet34 \
|
|
2458
|
+
--model_arg decoder=unet:depth=4 \
|
|
2459
|
+
--loss dice_ce \
|
|
2460
|
+
--image_size 512 --batch_size 8 --epochs 40
|
|
2461
|
+
```
|
|
2462
|
+
|
|
2463
|
+
The layout expected by `segfolder/` is a paired tree:
|
|
2464
|
+
|
|
2465
|
+
```
|
|
2466
|
+
/data/voc2012/images/2007_000032.jpg
|
|
2467
|
+
/data/voc2012/masks/2007_000032.png
|
|
2468
|
+
```
|
|
2469
|
+
|
|
2470
|
+
or, for a corpus with named splits, `train/`, `val/` and `test/`
|
|
2471
|
+
sub-directories each containing `images/` + `masks/`. Image and mask are
|
|
2472
|
+
matched on the file stem (case-insensitive) and a mismatch fails fast.
|
|
2473
|
+
|
|
2474
|
+
HuggingFace datasets that carry a mask feature are used with `--mask_column`:
|
|
2475
|
+
|
|
2476
|
+
<!-- genml-cli-test
|
|
2477
|
+
name: segmentation-mask-column
|
|
2478
|
+
network: true
|
|
2479
|
+
override:
|
|
2480
|
+
- --dataset {tmp}/hf_seg
|
|
2481
|
+
extra: --epochs 1
|
|
2482
|
+
-->
|
|
2483
|
+
```bash
|
|
2484
|
+
genml-kit-train --pipeline segmentation --method segmentation \
|
|
2485
|
+
--dataset scene_parse_150 --mask_column annotation \
|
|
2486
|
+
--model seg_wrapper:timm/convnext_tiny --model_arg decoder=fpn
|
|
2487
|
+
```
|
|
2488
|
+
|
|
2489
|
+
### Class-id compaction
|
|
2490
|
+
|
|
2491
|
+
Masks are expected to carry integer class ids, but the ids do **not** have
|
|
2492
|
+
to be dense. The pipeline scans every mask (train **and** val, so a class
|
|
2493
|
+
that only appears in validation is still mapped) and compacts whatever ids
|
|
2494
|
+
it finds into a dense `[0, C)` range:
|
|
2495
|
+
|
|
2496
|
+
| Source id | Compact id |
|
|
2497
|
+
|---|---|
|
|
2498
|
+
| 12 | 0 |
|
|
2499
|
+
| 17 | 1 |
|
|
2500
|
+
| 255 (= ignore) | dropped |
|
|
2501
|
+
|
|
2502
|
+
- **segfolder path:** the mapping is rebuilt on every run (a full scan)
|
|
2503
|
+
and remapped through `apply_mask_map` on every access.
|
|
2504
|
+
- **HuggingFace path:** the `{raw → compact}` map is cached to disk under
|
|
2505
|
+
`<cache_dir>/seg_class_maps/segclassmap__<hash>.json`, or
|
|
2506
|
+
`~/.cache/genml_kit/seg_class_maps/` when no `--cache_dir` is given
|
|
2507
|
+
(falling back to a temp dir if that is unwritable). The cache is keyed
|
|
2508
|
+
on the dataset spec **and** a content hash of the masks themselves, so
|
|
2509
|
+
editing a mask invalidates the entry even if the spec string is
|
|
2510
|
+
unchanged.
|
|
2511
|
+
|
|
2512
|
+
The class count reported by training (and the default for
|
|
2513
|
+
`--seg_num_classes`) is the **compact** count, i.e. the number of
|
|
2514
|
+
distinct ids found after dropping `--seg_ignore_index` (`len(mask_map)`).
|
|
2515
|
+
(The count falls back to `max_label + 1` over a sample of loader masks
|
|
2516
|
+
only when the pipeline did not build a mask map, e.g. a method wired
|
|
2517
|
+
without the segmentation pipeline.)
|
|
2518
|
+
|
|
2519
|
+
### Data augmentation
|
|
2520
|
+
|
|
2521
|
+
The default transforms for the `segmentation` method (in
|
|
2522
|
+
`genml_kit/datasets/transforms.py`) are split into a geometric stage and
|
|
2523
|
+
a photometric stage:
|
|
2524
|
+
|
|
2525
|
+
- **Geometric stage** — applied to the image **and** the mask together,
|
|
2526
|
+
so they always stay pixel-aligned:
|
|
2527
|
+
- `v2.RandomResizedCrop(image_size, scale=(0.5, 2.0),
|
|
2528
|
+
ratio=(0.75, 4/3))`
|
|
2529
|
+
- `v2.RandomHorizontalFlip(p=0.5)`
|
|
2530
|
+
- **Photometric stage** — applied to the **image only** (masks contain
|
|
2531
|
+
class ids, never color):
|
|
2532
|
+
- `v2.RandomApply([v2.ColorJitter(brightness=0.4, contrast=0.6,
|
|
2533
|
+
saturation=0.8, hue=0.1)], p=0.8)`
|
|
2534
|
+
|
|
2535
|
+
Validation is deterministic: `v2.Resize(image_size, BICUBIC)` then
|
|
2536
|
+
`CenterCrop(image_size)`.
|
|
2537
|
+
|
|
2538
|
+
The geometric/photometric split is a correctness contract: geometric ops
|
|
2539
|
+
go through `tv_tensors.Mask`, so torchvision issues one random draw per
|
|
2540
|
+
sample and the mask gets the same crop/translation as the image; color
|
|
2541
|
+
ops never see the mask.
|
|
2542
|
+
|
|
2543
|
+
To take full control, pass `--seg_augmentation_script <path>.py` pointing
|
|
2544
|
+
at a module that defines
|
|
2545
|
+
|
|
2546
|
+
```python
|
|
2547
|
+
def create_seg_transform(image_size: int, *, is_train: bool):
|
|
2548
|
+
...
|
|
2549
|
+
```
|
|
2550
|
+
|
|
2551
|
+
When set, the script **owns the whole paired transform** (image + mask),
|
|
2552
|
+
including normalization (the model processor's `image_mean`/`image_std`
|
|
2553
|
+
are *not* applied automatically in this mode), and is loaded lazily so a
|
|
2554
|
+
plain run never imports it.
|
|
2555
|
+
|
|
2556
|
+
### Encoder + decoder
|
|
2557
|
+
|
|
2558
|
+
`--model seg_wrapper:<backbone>` selects the encoder; the decoder is chosen
|
|
2559
|
+
with `--model_arg decoder=<spec>`. Built-in decoders:
|
|
2560
|
+
|
|
2561
|
+
| Decoder | Skip connections | Notes |
|
|
2562
|
+
|---|---|---|
|
|
2563
|
+
| `unet` | yes (all levels) | Classic U-Net top-down fusion; `depth=N` selects the deepest N encoder levels |
|
|
2564
|
+
| `fpn` | yes (additive) | Feature-pyramid fusion into one high-resolution map |
|
|
2565
|
+
| `linear` | no | 1×1 conv + bilinear upsample; good baseline, ViT-friendly |
|
|
2566
|
+
|
|
2567
|
+
The decoder spec accepts `:key=value` kwargs (`decoder=unet:depth=4,decoder_channels=256`)
|
|
2568
|
+
and, like classifier heads, a path to a `.py` file defining
|
|
2569
|
+
`create_decoder(in_channels, num_classes, **kwargs)` where `in_channels` is
|
|
2570
|
+
the probed per-level encoder channel list.
|
|
2571
|
+
|
|
2572
|
+
### Training from scratch vs. fine-tuning
|
|
2573
|
+
|
|
2574
|
+
The decoder is always randomly initialized and trained from scratch; the
|
|
2575
|
+
encoder is what you initialise. Options, in increasing order of GPU cost:
|
|
2576
|
+
|
|
2577
|
+
- **Pretrained encoder (recommended):** `seg_wrapper` loads timm ImageNet
|
|
2578
|
+
weights by default (`pretrained=False` disables this).
|
|
2579
|
+
- **Reuse a self-supervised checkpoint:** pre-train with SimMIM / I-JEPA /
|
|
2580
|
+
DINO, then load the encoder with `--source_checkpoint <ckpt>` (see
|
|
2581
|
+
[From A Pre-Trained Checkpoint](#from-a-pre-trained-checkpoint)).
|
|
2582
|
+
- **CNN encoder with a frozen backbone:** `--freeze 'decoder.*'` keeps the
|
|
2583
|
+
decoder trainable and freezes everything else.
|
|
2584
|
+
- **Transformer encoder with LoRA:** `--lora` (PEFT discovers the
|
|
2585
|
+
`q_proj`/`k_proj`/`v_proj` projections automatically). CNN encoders are
|
|
2586
|
+
almost all `Conv2d`, so LoRA does not apply to them — use `--freeze`.
|
|
2587
|
+
|
|
2588
|
+
### Loss and metric
|
|
2589
|
+
|
|
2590
|
+
The default criterion is `dice_ce` (Dice + cross-entropy), selectable via
|
|
2591
|
+
`--loss dice_ce`; the validation metric is **mean IoU** (`miou`, maximize,
|
|
2592
|
+
also reporting `mdice` and `pixel_acc`). The class count is inferred from
|
|
2593
|
+
the training masks (ignoring `--seg_ignore_index`, default 255) unless
|
|
2594
|
+
`--seg_num_classes` overrides it. Pixels equal to `--seg_ignore_index` are
|
|
2595
|
+
excluded from both the loss and the metric.
|
|
2596
|
+
|
|
2597
|
+
### Validation overlays
|
|
2598
|
+
|
|
2599
|
+
Set `--vis_every N` to have the trainer call the method's
|
|
2600
|
+
`log_validation` every N epochs, which logs a `seg/overlay` TensorBoard
|
|
2601
|
+
grid of **image / ground-truth mask / predicted mask** rows (8 samples by
|
|
2602
|
+
default) into the run's log dir (default `<checkpoint_dir>/logs`, see
|
|
2603
|
+
`--log_dir`). The masks are rendered with a fixed 20-hue palette; the
|
|
2604
|
+
ignore class (`--seg_ignore_index`, or 255) renders black. Overlays use
|
|
2605
|
+
the *current* model state, so they double as a quick training-progress
|
|
2606
|
+
check (e.g. whether the decoder is learning the class layout before the
|
|
2607
|
+
mIoU catches up).
|
|
2608
|
+
|
|
2609
|
+
Note that some backbones (e.g. plain ViT, Swin) require their native input
|
|
2610
|
+
resolution (224) and reject other sizes.
|
|
2611
|
+
|
|
2612
|
+
### Inference
|
|
2613
|
+
|
|
2614
|
+
`genml-kit-infer` is classification-only today: it predicts a top-K label
|
|
2615
|
+
for each image and does **not** emit per-pixel masks. For segmentation,
|
|
2616
|
+
use the validation loop (mIoU + overlays above) or the checkpoint's
|
|
2617
|
+
weights directly.
|
|
2618
|
+
|
|
2619
|
+
---
|
|
2620
|
+
|
|
2621
|
+
## Object Detection
|
|
2622
|
+
|
|
2623
|
+
Detection predicts a set of **bounding boxes**, each with a class label, for
|
|
2624
|
+
an image. `genml_kit` ships an anchor-free FCOS head over the same encoder
|
|
2625
|
+
feature pyramid that segmentation uses, wired as the `detection` pipeline and
|
|
2626
|
+
the `detection` method.
|
|
2627
|
+
|
|
2628
|
+
### How It Works
|
|
2629
|
+
|
|
2630
|
+
The model is `det_wrapper:<backbone>` = **encoder + FCOS head**:
|
|
2631
|
+
|
|
2632
|
+
* The encoder is any timm backbone loaded with `features_only=True`, exactly
|
|
2633
|
+
as for segmentation, so it yields a feature pyramid (finest level first).
|
|
2634
|
+
* The head runs a small convolutional tower on **every** pyramid level and
|
|
2635
|
+
predicts, per spatial location, a `num_classes`-way class score, four
|
|
2636
|
+
positive box distances `(l, t, r, b)` in input pixels, and a centerness
|
|
2637
|
+
score. The distances are `exp()`-activated and scaled by the level's
|
|
2638
|
+
stride, so a single location grid spans all scales.
|
|
2639
|
+
|
|
2640
|
+
Training uses the FCOS objective (`--loss fcos` by default), a sum of three
|
|
2641
|
+
terms:
|
|
2642
|
+
|
|
2643
|
+
* **Focal loss** over all locations (the class imbalance between the few
|
|
2644
|
+
positives and the many background locations is large).
|
|
2645
|
+
* **GIoU loss** over the positive locations only.
|
|
2646
|
+
* **Centerness BCE**, which teaches each positive location how central it is
|
|
2647
|
+
inside its target box, so NMS can rank central boxes higher.
|
|
2648
|
+
|
|
2649
|
+
The **assignment** is purely geometric and needs no Hungarian matcher: a
|
|
2650
|
+
location is a *candidate* for a box when it lies inside the box, and a
|
|
2651
|
+
*positive* when it additionally lies inside the box's centre region
|
|
2652
|
+
(`--loss_args center_sampling_radius=1.5`). When several boxes cover a
|
|
2653
|
+
location, the **smallest-area** box wins, so small objects are not starved of
|
|
2654
|
+
positives by large ones.
|
|
2655
|
+
|
|
2656
|
+
At validation time the dense predictions are decoded to `xyxy` boxes,
|
|
2657
|
+
per-class NMS is applied (`--det_nms_thresh`), and the result feeds a
|
|
2658
|
+
COCO-style **mAP** meter (`--det_max_detections`, default 100). The reported
|
|
2659
|
+
metric is `map` (COCO `mAP@[.5:.95]`, maximize), so the best checkpoint is
|
|
2660
|
+
stored under `best_map`; `map_50` and `map_75` are logged alongside it.
|
|
2661
|
+
|
|
2662
|
+
### Dataset Format
|
|
2663
|
+
|
|
2664
|
+
The `detection` pipeline reads **COCO-JSON** corpora via the `detfolder/`
|
|
2665
|
+
spec:
|
|
2666
|
+
|
|
2667
|
+
```
|
|
2668
|
+
<root>/images/<file_name>
|
|
2669
|
+
<root>/annotations.json
|
|
2670
|
+
```
|
|
2671
|
+
|
|
2672
|
+
or, with named splits:
|
|
2673
|
+
|
|
2674
|
+
```
|
|
2675
|
+
<root>/train/images/<file_name>
|
|
2676
|
+
<root>/train/annotations.json
|
|
2677
|
+
<root>/val/images/<file_name>
|
|
2678
|
+
<root>/val/annotations.json
|
|
2679
|
+
```
|
|
2680
|
+
|
|
2681
|
+
`annotations.json` is the standard COCO object-detection schema (`images`,
|
|
2682
|
+
`annotations` with `[x, y, width, height]` `bbox`, and `categories`). The
|
|
2683
|
+
loader converts `bbox` to `xyxy` and **compacts** the (1-based, sparse) COCO
|
|
2684
|
+
category ids into a dense `[0, C)` label space, so the head width equals the
|
|
2685
|
+
number of categories actually present. Images with no annotations are legal
|
|
2686
|
+
(pure-background samples) and still contribute negatives to the loss. The
|
|
2687
|
+
annotation file name and image sub-directory are configurable via
|
|
2688
|
+
`--det_annotation_file` and `--det_image_subdir`.
|
|
2689
|
+
|
|
2690
|
+
To generate a throwaway corpus for a CPU smoke run without downloading COCO,
|
|
2691
|
+
use the bundled generator (`scripts/make_tiny_coco.py`), then point the
|
|
2692
|
+
pipeline at it:
|
|
2693
|
+
|
|
2694
|
+
<!-- genml-cli-test
|
|
2695
|
+
name: detection-dataset-format
|
|
2696
|
+
network: true
|
|
2697
|
+
override:
|
|
2698
|
+
- --dataset detfolder/{tmp}/det
|
|
2699
|
+
extra: --epochs 1
|
|
2700
|
+
-->
|
|
2701
|
+
```bash
|
|
2702
|
+
genml-kit-train --pipeline detection --method detection \
|
|
2703
|
+
--dataset detfolder/tiny_coco --model det_wrapper:timm/resnet18 \
|
|
2704
|
+
--image_size 640 --batch_size 4 --epochs 20
|
|
2705
|
+
```
|
|
2706
|
+
|
|
2707
|
+
### Example
|
|
2708
|
+
|
|
2709
|
+
<!-- genml-cli-test
|
|
2710
|
+
name: detection-example
|
|
2711
|
+
network: true
|
|
2712
|
+
override:
|
|
2713
|
+
- --dataset detfolder/{tmp}/det
|
|
2714
|
+
extra: --epochs 1
|
|
2715
|
+
-->
|
|
2716
|
+
```bash
|
|
2717
|
+
genml-kit-train \
|
|
2718
|
+
--pipeline detection --method detection \
|
|
2719
|
+
--dataset detfolder//data/coco_tiny \
|
|
2720
|
+
--model det_wrapper:timm/resnet50 \
|
|
2721
|
+
--model_arg head=fcos_head:num_convs=2 \
|
|
2722
|
+
--image_size 640 --batch_size 4 \
|
|
2723
|
+
--epochs 20 --lr 1e-4
|
|
2724
|
+
```
|
|
2725
|
+
|
|
2726
|
+
Override the inferred class count with `--det_num_classes`; tune the decode
|
|
2727
|
+
with `--det_score_thresh` and `--det_nms_thresh`; override the criterion with
|
|
2728
|
+
`--loss` / `--loss_args` (e.g. `--loss_args center_sampling_radius=1.0`).
|
|
2729
|
+
|
|
2730
|
+
### Transforms
|
|
2731
|
+
|
|
2732
|
+
Detection does **not** reuse the segmentation transform stack: `torchvision`
|
|
2733
|
+
remaps masks automatically through geometric ops but leaves boxes untouched,
|
|
2734
|
+
so every resize / flip must move the coordinates itself. The default
|
|
2735
|
+
pipeline is therefore a hand-written **letterbox** (aspect-preserving resize
|
|
2736
|
+
then pad, which keeps boxes rectangular) followed by a box-aware horizontal
|
|
2737
|
+
flip (train only) and a `(C, H, W)` float conversion. Boxes stay in absolute
|
|
2738
|
+
`xyxy` pixels throughout.
|
|
2739
|
+
|
|
2740
|
+
---
|
|
2741
|
+
|
|
2742
|
+
## Visual Odometry
|
|
2743
|
+
|
|
2744
|
+
Visual odometry (VO) estimates the 2-D **similarity transform** — rotation,
|
|
2745
|
+
uniform scale, and translation — that maps one image onto another taken a
|
|
2746
|
+
fraction of a second later. `genml_kit` ships a learned VO front-end plus a
|
|
2747
|
+
synthetic pair generator, wired as the `vo_pair` pipeline and the `vo_pair`
|
|
2748
|
+
method.
|
|
2749
|
+
|
|
2750
|
+
The full mathematics (coordinate frames, the rotation matrix, Umeyama's
|
|
2751
|
+
closed-form solution, the photometric objective, the `mce` metric, and a
|
|
2752
|
+
failure-mode appendix) lives in [`vo/README.md`](vo/README.md) — a
|
|
2753
|
+
from-the-ground-up tutorial. This section is the practical *usage* guide.
|
|
2754
|
+
|
|
2755
|
+
### Quick start
|
|
2756
|
+
|
|
2757
|
+
<!-- genml-cli-test
|
|
2758
|
+
name: vo-quick-start
|
|
2759
|
+
extra: --epochs 1 --vo_length 16 --vo_val_length 8
|
|
2760
|
+
-->
|
|
2761
|
+
```bash
|
|
2762
|
+
# Supervised VO on synthetic pairs (default npu-small profile)
|
|
2763
|
+
genml-kit-train \
|
|
2764
|
+
--pipeline vo_pair --method vo_pair \
|
|
2765
|
+
--model vo/npu-small \
|
|
2766
|
+
--image_size 64 --batch_size 32 --epochs 10
|
|
2767
|
+
|
|
2768
|
+
# Photometric self-supervised stage (adds the photometric residual term)
|
|
2769
|
+
genml-kit-train \
|
|
2770
|
+
--pipeline vo_pair --method vo_pair \
|
|
2771
|
+
--vo_stage photometric --epochs 10
|
|
2772
|
+
```
|
|
2773
|
+
|
|
2774
|
+
### Model profiles
|
|
2775
|
+
|
|
2776
|
+
The VO net is selected through the model registry as `vo/<profile>`:
|
|
2777
|
+
|
|
2778
|
+
| `--model` | Profile | Encoder widths |
|
|
2779
|
+
|---|---|---|
|
|
2780
|
+
| `vo/npu-small` | default | `(16, 32, 64, 128)` |
|
|
2781
|
+
| `vo/edge-mid` | mid-size | see `genml_kit/models/vo/vo_similar.py` |
|
|
2782
|
+
| `vo/station` | large | see `genml_kit/models/vo/vo_similar.py` |
|
|
2783
|
+
|
|
2784
|
+
### Pipeline flags (`--pipeline vo_pair`)
|
|
2785
|
+
|
|
2786
|
+
| Flag | Default | Meaning |
|
|
2787
|
+
|---|---|---|
|
|
2788
|
+
| `--vo_length` | `20000` | number of synthetic training pairs |
|
|
2789
|
+
| `--vo_val_length` | `2000` | number of synthetic validation pairs |
|
|
2790
|
+
| `--pitch_deg` | `45.0` | camera pitch for the ground-plane render |
|
|
2791
|
+
| `--terrain` | `field` | terrain profile name |
|
|
2792
|
+
| `--seeded_pairs` | `0` | seed the pair generator for reproducible pairs |
|
|
2793
|
+
|
|
2794
|
+
### Method flags (`--method vo_pair`)
|
|
2795
|
+
|
|
2796
|
+
| Flag | Default | Meaning |
|
|
2797
|
+
|---|---|---|
|
|
2798
|
+
| `--vo_stage` | `supervised` | `supervised` (mce + similarity deltas) or `photometric` (adds the photometric residual) |
|
|
2799
|
+
| `--vo_profile` | `npu-small` | VO model profile (also selectable via `--model vo/<profile>`) |
|
|
2800
|
+
| `--vo_loss_cfg` | — | comma-separated loss weights `w_log_s,w_theta,w_conf,w_photo` |
|
|
2801
|
+
|
|
2802
|
+
The reported metric is `mce` (mean corner error, in pixels), minimized — the
|
|
2803
|
+
best checkpoint is stored under `best_mce`. Set `--image_size 64` for the
|
|
2804
|
+
synthetic pairs, whose default resolution is 64×64.
|
|
2805
|
+
|
|
2806
|
+
---
|
|
2807
|
+
|
|
2808
|
+
## Knowledge Distillation
|
|
2809
|
+
|
|
2810
|
+
Knowledge distillation trains a small **student** to mimic a large **teacher**.
|
|
2811
|
+
`genml_kit` implements the three families of distillation (response / logit
|
|
2812
|
+
matching, feature / FitNets, attention, and relation / RKD) and both teacher
|
|
2813
|
+
sources: a separately-loaded frozen model (`--teacher_kind external`) and an
|
|
2814
|
+
exponential-moving-average copy of the student itself
|
|
2815
|
+
(`--teacher_kind ema`, i.e. self-distillation). It is wired as the
|
|
2816
|
+
`distillation` method over the standard `images` pipeline.
|
|
2817
|
+
|
|
2818
|
+
The full mathematics (temperature and the soft-target insight, the KL↔CE
|
|
2819
|
+
collapse and the `T²` scaling, attention-transfer normalisation, RKD, and the
|
|
2820
|
+
EMA-teacher variance argument) lives in
|
|
2821
|
+
[`knowledge_distillation/README.md`](knowledge_distillation/README.md) — a
|
|
2822
|
+
from-the-ground-up tutorial. This section is the practical *usage* guide.
|
|
2823
|
+
|
|
2824
|
+
### Quick start
|
|
2825
|
+
|
|
2826
|
+
<!-- genml-cli-test
|
|
2827
|
+
name: distillation-quick-start
|
|
2828
|
+
network: true
|
|
2829
|
+
override:
|
|
2830
|
+
- --dataset {tmp}/img
|
|
2831
|
+
- -.--image_column
|
|
2832
|
+
- --teacher_model google/vit-base-patch16-224
|
|
2833
|
+
- --teacher_checkpoint {tmp}/teacher/model_best.pt
|
|
2834
|
+
extra: --epochs 1
|
|
2835
|
+
-->
|
|
2836
|
+
```bash
|
|
2837
|
+
# Self-distillation: teacher is an EMA of the student (no external model)
|
|
2838
|
+
genml-kit-train \
|
|
2839
|
+
--pipeline images --method distillation \
|
|
2840
|
+
--model google/vit-base-patch16-224 --image_size 224 \
|
|
2841
|
+
--dataset uoft-cs/cifar10 --image_column img --label_column label \
|
|
2842
|
+
--teacher_kind ema --kd_temperature 4.0 --kd_alpha 1.0 \
|
|
2843
|
+
--epochs 10
|
|
2844
|
+
|
|
2845
|
+
# External teacher: frozen checkpoint hands over its soft targets
|
|
2846
|
+
genml-kit-train \
|
|
2847
|
+
--pipeline images --method distillation \
|
|
2848
|
+
--model google/vit-base-patch16-224 --image_size 224 \
|
|
2849
|
+
--dataset uoft-cs/cifar10 --image_column img --label_column label \
|
|
2850
|
+
--teacher_kind external \
|
|
2851
|
+
--teacher_model google/vit-large-patch16-224 \
|
|
2852
|
+
--teacher_checkpoint /path/to/teacher.pt \
|
|
2853
|
+
--kd_temperature 4.0 --kd_alpha 1.0 --epochs 10
|
|
2854
|
+
```
|
|
2855
|
+
|
|
2856
|
+
### Method flags (`--method distillation`)
|
|
2857
|
+
|
|
2858
|
+
| Flag | Default | Meaning |
|
|
2859
|
+
|---|---|---|
|
|
2860
|
+
| `--teacher_kind` | `external` | `external` loads a frozen teacher; `ema` self-distils from an EMA of the student |
|
|
2861
|
+
| `--teacher_model` | — | teacher model spec (required for `--teacher_kind external`) |
|
|
2862
|
+
| `--teacher_checkpoint` | — | checkpoint holding the external teacher's weights |
|
|
2863
|
+
| `--kd_temperature` | `4.0` | distillation temperature $`T`$ |
|
|
2864
|
+
| `--kd_alpha` | `1.0` | weight of the response (logit) KD term (`0` disables it) |
|
|
2865
|
+
| `--kd_feature_alpha` | `0.0` | weight of the FitNets feature KD term (`0` disables it) |
|
|
2866
|
+
| `--kd_attention` / `--no-kd_attention` | off | use attention transfer on the matched layers instead of raw feature MSE |
|
|
2867
|
+
| `--kd_relation_alpha` | `0.0` | weight of the relational (RKD) KD term (`0` disables it) |
|
|
2868
|
+
| `--kd_layer_strategy` | `uniform` | how to pair teacher/student layers: `by_index`, `uniform`, `by_name` |
|
|
2869
|
+
| `--kd_layer_pattern` | — | regex selecting the feature layers to pair (feature KD) |
|
|
2870
|
+
| `--kd_proj_hidden` | `512` | hidden width of the FitNets projection |
|
|
2871
|
+
| `--kd_ema_momentum` | `0.996` | initial EMA momentum for `--teacher_kind ema` |
|
|
2872
|
+
| `--kd_ema_final_momentum` | `1.0` | final EMA momentum for `--teacher_kind ema` |
|
|
2873
|
+
|
|
2874
|
+
Enable one family at a time and watch the validation metric after each addition
|
|
2875
|
+
(see Appendix B of the distillation tutorial for the pitfalls). A labeled
|
|
2876
|
+
`--dataset` is required: the ground-truth cross-entropy branch anchors the
|
|
2877
|
+
student on examples where the teacher is confidently wrong.
|
|
2878
|
+
|
|
2879
|
+
---
|
|
2880
|
+
|
|
2881
|
+
## Large Language Models (causal LM)
|
|
2882
|
+
|
|
2883
|
+
genml_kit can fine-tune decoder-only language models through the same
|
|
2884
|
+
branchless driver: a new `text` pipeline (token collation) pairs with a new
|
|
2885
|
+
`causal_lm` method (shifted next-token cross-entropy, perplexity metric).
|
|
2886
|
+
|
|
2887
|
+
### Quick start
|
|
2888
|
+
|
|
2889
|
+
<!-- genml-cli-test
|
|
2890
|
+
name: causal-lm-quick-start
|
|
2891
|
+
network: true
|
|
2892
|
+
extras: lora
|
|
2893
|
+
override:
|
|
2894
|
+
- --dataset {tmp}/text
|
|
2895
|
+
extra: --epochs 1
|
|
2896
|
+
-->
|
|
2897
|
+
```bash
|
|
2898
|
+
# Fine-tune a tiny GPT-2 on a local text corpus
|
|
2899
|
+
genml-kit-train --pipeline text \
|
|
2900
|
+
--method causal_lm \
|
|
2901
|
+
--model sshleifer/tiny-gpt2 \
|
|
2902
|
+
--dataset ./my_text_corpus \
|
|
2903
|
+
--text_column text \
|
|
2904
|
+
--max_seq_len 256 \
|
|
2905
|
+
--lora
|
|
2906
|
+
```
|
|
2907
|
+
|
|
2908
|
+
The `--dataset` value may be a HuggingFace dataset name, or a directory saved
|
|
2909
|
+
with `datasets.Dataset.save_to_disk` (containing `train` / `validation`
|
|
2910
|
+
splits). With `--val_split` a held-out fraction of the training split is used
|
|
2911
|
+
for validation instead.
|
|
2912
|
+
|
|
2913
|
+
### Pipeline flags (`--pipeline text`)
|
|
2914
|
+
|
|
2915
|
+
| Flag | Default | Meaning |
|
|
2916
|
+
|---|---|---|
|
|
2917
|
+
| `--text_column` | — | column holding a raw LM sequence (plain-LM mode) |
|
|
2918
|
+
| `--instruction_column` | — | prompt column (instruction mode, with `--response_column`) |
|
|
2919
|
+
| `--response_column` | — | response column (instruction mode) |
|
|
2920
|
+
| `--max_seq_len` | `512` | token sequence length (truncation / packing window) |
|
|
2921
|
+
| `--sequence_mode` | `document` | sequence recipe: `document` (no packing, document-exact boundary loss, prompt framed as a document prefix) or `open` (packed concatenation, boundary loss kept, prompt framed as a whole document) |
|
|
2922
|
+
| `--mask_prompt` | off | mask prompt tokens out of the loss (instruction tuning) |
|
|
2923
|
+
| `--packing [wrapped\|bfd]` | off | pack sequences into fixed `max_seq_len` windows (`wrapped` = concatenate-and-slice, `bfd` = best-fit-decreasing) |
|
|
2924
|
+
| `--length_bucket` | off | batch items of similar length so padded batches waste little compute (mutually exclusive with `--packing`) |
|
|
2925
|
+
| `--pad_to_multiple_of` | — | round padded length up to a multiple (e.g. `8`) |
|
|
2926
|
+
| `--max_items` | — | cap the number of loaded items (smoke tests) |
|
|
2927
|
+
|
|
2928
|
+
#### Packing and document boundaries
|
|
2929
|
+
|
|
2930
|
+
Padding every item up to the batch's longest wastes compute on short documents;
|
|
2931
|
+
packing removes that waste but, naively, lets tokens of one document attend to a
|
|
2932
|
+
*different* document sharing the same window. Three modes trade padding for
|
|
2933
|
+
compute, and packing stays document-aware without binding the data side to any
|
|
2934
|
+
model:
|
|
2935
|
+
|
|
2936
|
+
| Mode | How it batches | Document isolation |
|
|
2937
|
+
|---|---|---|
|
|
2938
|
+
| *(default)* | pad every row to the batch's longest item | plain, document-exact 2-D mask |
|
|
2939
|
+
| `--packing wrapped` | concatenate the tokenized stream (EOS-separated) and slice into `max_seq_len` windows | via `segment_ids` (model-dependent, below) |
|
|
2940
|
+
| `--packing bfd` | best-fit-decreasing: longest document first into the open bin with the least remaining space | via `segment_ids` (model-dependent, below) |
|
|
2941
|
+
| `--length_bucket` | no packing: sort by a cheap character-length proxy, group into buckets, shuffle within and across them, emit similar-length batches | plain, document-exact 2-D mask |
|
|
2942
|
+
|
|
2943
|
+
`wrapped` is the historical behaviour and may split a document across two
|
|
2944
|
+
windows; `bfd` yields fewer, fuller windows but reorders documents, so batch
|
|
2945
|
+
order is **not** reproducible with respect to `--seed` (it warns once).
|
|
2946
|
+
`--length_bucket` is mutually exclusive with `--packing`.
|
|
2947
|
+
|
|
2948
|
+
**Batching and memory.** On every path except `wrapped`, `--batch_size` counts
|
|
2949
|
+
*documents* (each becomes one model row, or one packed row for `bfd`). Under
|
|
2950
|
+
`wrapped` the collator concatenates the batch and slices it into `max_seq_len`
|
|
2951
|
+
windows, so the row count is `ceil(total_tokens / max_seq_len)` -- which the
|
|
2952
|
+
document count does **not** bound. A batch of long documents can therefore fan
|
|
2953
|
+
out into far more rows than `--batch_size` and exhaust VRAM (one long document
|
|
2954
|
+
alone becomes `len(doc) / max_seq_len` rows), and lowering `--batch_size` or
|
|
2955
|
+
`--grad_accum_steps` barely helps because the cost tracks *tokens*, not
|
|
2956
|
+
documents.
|
|
2957
|
+
|
|
2958
|
+
To make `wrapped` safe, `--batch_size` is honored as a **token budget**:
|
|
2959
|
+
`PackingBatchSampler` groups documents until a batch fills `--batch_size *
|
|
2960
|
+
max_seq_len` tokens (the constant-length packing recipe Megatron-LM and
|
|
2961
|
+
HuggingFace packed datasets use), and the collator hard-caps every emitted batch
|
|
2962
|
+
at `--batch_size` windows. A document longer than one window is still streamed
|
|
2963
|
+
in -- its tail lands in a later window of the same batch rather than being
|
|
2964
|
+
truncated away -- and only the overflow past the whole batch budget (a single
|
|
2965
|
+
document longer than `--batch_size` windows) is dropped, with a one-time
|
|
2966
|
+
warning. If you see that warning, raise `--batch_size` or lower `--max_items`.
|
|
2967
|
+
|
|
2968
|
+
`--sequence_mode` compiles the windowing flags and the two coupled policies
|
|
2969
|
+
below into one validated object (`genml_kit/training/sequence_mode.py`), so an
|
|
2970
|
+
incoherent pairing cannot be expressed. The two recipes are:
|
|
2971
|
+
|
|
2972
|
+
- `document` (default) -- unpacked, **boundary loss masked**, prompts framed as
|
|
2973
|
+
a document *prefix* (`[BOS] ...`, no trailing `EOS`). The validation
|
|
2974
|
+
perplexity is document-exact and the first sampled token is a trained
|
|
2975
|
+
"continue this document" state.
|
|
2976
|
+
- `open` -- packed (`wrapped`), **boundary loss kept**, prompts framed as a whole
|
|
2977
|
+
*document* (`[BOS] ... [EOS]`). This is the GPT-2/GPT-3 pre-training recipe:
|
|
2978
|
+
the token following a document's `EOS` keeps a trained next-token
|
|
2979
|
+
distribution, so "what follows a document" is a learned state rather than an
|
|
2980
|
+
untrained one. `open` requires packing, and `--length_bucket` is rejected
|
|
2981
|
+
with it.
|
|
2982
|
+
|
|
2983
|
+
Explicit `--packing` / `--length_bucket` override the recipe's default
|
|
2984
|
+
windowing; the boundary-loss and framing policies still come from the recipe.
|
|
2985
|
+
A kept boundary loss is meaningful only under packing: unpacked, the tokens
|
|
2986
|
+
after a document's `EOS` are padding, so keeping the boundary label would train
|
|
2987
|
+
`EOS -> PAD` -- which is why `open` (and the collator) reject it.
|
|
2988
|
+
|
|
2989
|
+
Document isolation (independent of the boundary loss) works as follows:
|
|
2990
|
+
|
|
2991
|
+
- Under `--packing`, the collator emits two extra tensors per batch, requiring
|
|
2992
|
+
no model knowledge: `segment_ids` (the contiguous run -- i.e. source document
|
|
2993
|
+
-- of each token, `-1` for padding) and `position_ids` (restarts at `0` at
|
|
2994
|
+
every run start, so a window is self-describing).
|
|
2995
|
+
- The `causal_lm` method, the only place that holds both the batch and the
|
|
2996
|
+
built model, translates those tensors to whatever the model declares via its
|
|
2997
|
+
`PACKED_ATTENTION` class attribute:
|
|
2998
|
+
|
|
2999
|
+
| `PACKED_ATTENTION` | What the model receives | Document isolation |
|
|
3000
|
+
|---|---|---|
|
|
3001
|
+
| `"none"` (default) | nothing extra | `--packing` is refused with a clear error |
|
|
3002
|
+
| `"block_diagonal"` | a 4-D `[B, 1, L, L]` additive mask | exact (no flash kernel) |
|
|
3003
|
+
| `"varlen"` | `FlashAttentionKwargs` (`cu_seq_lens_q/k`, `max_length_q/k`) on a flattened `[1, total]` batch | exact (flash when `flash-attn` is installed) |
|
|
3004
|
+
| `"segment_ids"` | the raw `[B, L]` `segment_ids` runs plus the 2-D padding mask and `position_ids` | model-defined (reset state per run) |
|
|
3005
|
+
| `"causal"` | only the 2-D padding mask and `position_ids` (no boundary tensors) | none -- documents are concatenated and may attend across boundaries (the standard LLM pre-training recipe) |
|
|
3006
|
+
|
|
3007
|
+
HuggingFace decoders are inferred automatically: `--attn_implementation
|
|
3008
|
+
flash_attention_2` implies `varlen`, otherwise `block_diagonal`. Custom
|
|
3009
|
+
models (subclasses of `GenerativeLM`) opt in explicitly. The `segment_ids`
|
|
3010
|
+
mode is for models that compute attention *without* materializing the `L x L`
|
|
3011
|
+
score matrix (linear / kernelized, streaming or recurrent attention): a dense
|
|
3012
|
+
block-diagonal mask would defeat the cost such a mechanism is built to avoid,
|
|
3013
|
+
so the model instead receives the raw per-token document runs and resets its
|
|
3014
|
+
state at each boundary. The `causal` mode opts *out* of document isolation
|
|
3015
|
+
altogether: the model receives only the ordinary 2-D key-padding mask and
|
|
3016
|
+
`position_ids`, and a plain causal mask attends across document boundaries.
|
|
3017
|
+
This is the recipe used by GPT-3, PaLM, LLaMA, Falcon and Mistral
|
|
3018
|
+
(concatenate with an EOD marker, no cross-document blocking), so it is the
|
|
3019
|
+
right choice when reproducing that setup. The collator still emits
|
|
3020
|
+
`segment_ids`; a `causal` model simply ignores them. Use `causal` for models
|
|
3021
|
+
that compute attention without an `L x L` score matrix *and* want to match
|
|
3022
|
+
mainstream pre-training; keep `segment_ids` when document isolation is itself
|
|
3023
|
+
the property under test.
|
|
3024
|
+
|
|
3025
|
+
Because document boundaries are derived from `segment_ids` (not the raw
|
|
3026
|
+
`attention_mask`), unless the model declares `causal`, packing a model that only
|
|
3027
|
+
understands a 2-D mask is a hard error rather than a silent cross-document leak.
|
|
3028
|
+
The collator also masks the label opening each document, so the causal shift
|
|
3029
|
+
never predicts across a boundary -- packed training stays equivalent to the
|
|
3030
|
+
per-document padded case and `--mask_prompt` is honoured. `flash-attn` is an
|
|
3031
|
+
optional extra (`pip install genml_kit[flash]`); without it everything runs on
|
|
3032
|
+
the block-diagonal (pure-torch) path, so CPU and CI never require a GPU.
|
|
3033
|
+
|
|
3034
|
+
`--dataset`, `--split`, `--val_split`, `--cache_dir`, `--image_column`,
|
|
3035
|
+
`--max_seq_len`, `--sequence_mode`, `--mask_prompt`, `--pad_to_multiple_of` and
|
|
3036
|
+
`--max_items` are
|
|
3037
|
+
**globally shared** CLI arguments: they are declared once by the harness
|
|
3038
|
+
(`genml_kit/utils/shared_args.py`), so every pipeline and method can read
|
|
3039
|
+
`args.<flag>` directly instead of guarding with `getattr(...)`.
|
|
3040
|
+
|
|
3041
|
+
### Method flags (`--method causal_lm`)
|
|
3042
|
+
|
|
3043
|
+
| Flag | Default | Meaning |
|
|
3044
|
+
|---|---|---|
|
|
3045
|
+
| `--tokenizer` | `--model` | tokenizer name/path (defaults to the model) |
|
|
3046
|
+
| `--trust_remote_code` | off | allow custom code from the model hub |
|
|
3047
|
+
| `--attn_implementation` | — | attention backend (`sdpa`, `eager`, `flash_attention_2`) |
|
|
3048
|
+
|
|
3049
|
+
The metric is `ppl` (perplexity, lower is better), so the best checkpoint is
|
|
3050
|
+
stored under `best_ppl`. `--lora`, `--grad_checkpoint`, `--freeze`,
|
|
3051
|
+
`--source_checkpoint` and `--resume` all work for causal LMs through the same
|
|
3052
|
+
shared model-extras path every other objective uses.
|
|
3053
|
+
|
|
3054
|
+
### Efficient fine-tuning (LoRA, gradient checkpointing, bf16)
|
|
3055
|
+
|
|
3056
|
+
LoRA is the recommended way to fine-tune a causal LM. When
|
|
3057
|
+
`--lora_target_modules` is omitted, genml_kit **resolves the target modules
|
|
3058
|
+
from the model's own structure** rather than from a hardcoded name list, so it
|
|
3059
|
+
works across architectures without configuration:
|
|
3060
|
+
|
|
3061
|
+
| Model family | Auto-resolved targets |
|
|
3062
|
+
|---|---|
|
|
3063
|
+
| GPT-2 (HF `Conv1D`) | `c_attn`, `attn.c_proj` |
|
|
3064
|
+
| Llama / Mistral / Qwen2 / HF ViT | `q_proj`, `k_proj`, `v_proj`, `o_proj` |
|
|
3065
|
+
| BERT | `query`, `key`, `value`, `attention.output.dense` |
|
|
3066
|
+
| timm ViT | `qkv`, `attn.proj` |
|
|
3067
|
+
| No attention (plain MLP backbone) | every `nn.Linear` (with a warning) |
|
|
3068
|
+
|
|
3069
|
+
The resolver detects attention blocks by class/structure, keeps only their
|
|
3070
|
+
projection layers, and emits the shortest dotted suffix that does **not** also
|
|
3071
|
+
match a non-attention layer (so GPT-2's `attn.c_proj` is not confused with
|
|
3072
|
+
`mlp.c_proj`). Pass `--lora_target_modules` explicitly to override.
|
|
3073
|
+
|
|
3074
|
+
<!-- genml-cli-test
|
|
3075
|
+
name: causal-lm-lora
|
|
3076
|
+
network: true
|
|
3077
|
+
extras: lora
|
|
3078
|
+
override:
|
|
3079
|
+
- --dataset {tmp}/text
|
|
3080
|
+
- --model sshleifer/tiny-gpt2
|
|
3081
|
+
-->
|
|
3082
|
+
```bash
|
|
3083
|
+
# LoRA + gradient checkpointing + bf16 on a small LM
|
|
3084
|
+
genml-kit-train --pipeline text \
|
|
3085
|
+
--method causal_lm \
|
|
3086
|
+
--model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
|
|
3087
|
+
--dataset ./my_text_corpus \
|
|
3088
|
+
--text_column text \
|
|
3089
|
+
--max_seq_len 1024 \
|
|
3090
|
+
--lora --lora_r 16 --lora_alpha 32 \
|
|
3091
|
+
--grad_checkpoint \
|
|
3092
|
+
--amp_dtype bfloat16
|
|
3093
|
+
```
|
|
3094
|
+
|
|
3095
|
+
`--lora`, `--lora_r`, `--lora_alpha`, `--lora_dropout` and
|
|
3096
|
+
`--lora_target_modules` are the shared LoRA flags; `--grad_checkpoint` enables
|
|
3097
|
+
per-block activation checkpointing (HF `gradient_checkpointing_enable` for
|
|
3098
|
+
transformers models, native APIs for timm/custom backbones); `--amp_dtype
|
|
3099
|
+
bfloat16` selects the autocast dtype.
|
|
3100
|
+
|
|
3101
|
+
When the model is a PEFT `PeftModel`, checkpoints also embed a **portable
|
|
3102
|
+
adapter blob** (`lora_state_blob`: a tar of `adapter_model.safetensors` +
|
|
3103
|
+
`adapter_config.json`), so the adapters can be loaded with stock PEFT or
|
|
3104
|
+
published to the Hub, while `--resume` continues to restore the raw tensor
|
|
3105
|
+
state exactly.
|
|
3106
|
+
|
|
3107
|
+
### Tokenizers
|
|
3108
|
+
|
|
3109
|
+
`load_tokenizer(name)` mirrors the model/processor split. Normal model IDs
|
|
3110
|
+
resolve through `transformers.AutoTokenizer`, which already handles both
|
|
3111
|
+
SentencePiece and tiktoken Hub vocabularies. Two registry entries cover the
|
|
3112
|
+
gaps: `byte` (a dependency-free tokenizer for offline tests) and
|
|
3113
|
+
`spm` (a bare SentencePiece `tokenizer.model`, behind the optional
|
|
3114
|
+
`sentencepiece` dependency).
|
|
3115
|
+
|
|
3116
|
+
A byte-level BPE is built separately, from a corpus subset, by
|
|
3117
|
+
`genml-kit-tokenizer`. Building is a one-shot offline job rather than a side
|
|
3118
|
+
effect of a training run, so the text behind the vocabulary is a reproducible
|
|
3119
|
+
artifact: pass the resulting directory straight to `--tokenizer`. The
|
|
3120
|
+
`tokenizer_meta.json` sidecar records the corpus, split, fraction, seed and
|
|
3121
|
+
row counts it was trained on.
|
|
3122
|
+
|
|
3123
|
+
<!-- genml-cli-test
|
|
3124
|
+
name: tokenizer-build
|
|
3125
|
+
override:
|
|
3126
|
+
- --dataset {tmp}/text
|
|
3127
|
+
- --out {tmp}/tok
|
|
3128
|
+
extra: --vocab_size 128 --fraction 0.5 --seed 0
|
|
3129
|
+
-->
|
|
3130
|
+
```bash
|
|
3131
|
+
genml-kit-tokenizer --dataset roneneldan/TinyStories --text_column text \
|
|
3132
|
+
--fraction 0.1 --seed 42 --vocab_size 4096 \
|
|
3133
|
+
--cache_dir hf --out hf/tokenizers/ts_bpe_4096_s0.1
|
|
3134
|
+
```
|
|
3135
|
+
|
|
3136
|
+
`--fraction (0.0, 1.0]` trains on that share of the rows: an indexable dataset
|
|
3137
|
+
yields exactly `round(n*fraction)` rows, drawn without replacement and
|
|
3138
|
+
seed-stable, while `--streaming` corpora get a seeded per-row sample (use
|
|
3139
|
+
`--max_rows` to bound how much of a stream is read).
|
|
3140
|
+
|
|
3141
|
+
### Generation
|
|
3142
|
+
|
|
3143
|
+
`genml-kit-llm generate` turns prompts into completions. Generation is a
|
|
3144
|
+
**model contract**, not a HuggingFace-only path: a model opts in by
|
|
3145
|
+
subclassing `GenerativeLM` (`genml_kit/models/generative.py`), whose default
|
|
3146
|
+
`generate` runs a portable token loop over the architecture-agnostic
|
|
3147
|
+
`forward(...) -> .logits` contract that `causal_lm` already trains against.
|
|
3148
|
+
HuggingFace models are *not* wrapped (their `state_dict` keys must stay intact
|
|
3149
|
+
for `--resume`); the dispatcher calls their own `generate` directly. So custom
|
|
3150
|
+
models registered in `MODELS` generate exactly like HF ones.
|
|
3151
|
+
|
|
3152
|
+
<!-- genml-cli-test
|
|
3153
|
+
name: llm-generate
|
|
3154
|
+
network: true
|
|
3155
|
+
override:
|
|
3156
|
+
- --prompts {tmp}/prompts/prompts.txt
|
|
3157
|
+
- --dataset {tmp}/prompt_ds --split test
|
|
3158
|
+
- --model sshleifer/tiny-gpt2
|
|
3159
|
+
- --checkpoint {tmp}/chk_infer/model_best.pt
|
|
3160
|
+
- -.--lora_adapter
|
|
3161
|
+
-->
|
|
3162
|
+
```bash
|
|
3163
|
+
# Prompts from a file (one per line, or JSONL with --prompt_column)
|
|
3164
|
+
genml-kit-llm generate --model sshleifer/tiny-gpt2 \
|
|
3165
|
+
--prompts prompts.txt \
|
|
3166
|
+
--max_new_tokens 64 \
|
|
3167
|
+
--do_sample --temperature 0.8 --top_p 0.95 \
|
|
3168
|
+
--output completions.jsonl
|
|
3169
|
+
|
|
3170
|
+
# Prompts from a HuggingFace dataset column, with a fine-tuned checkpoint
|
|
3171
|
+
genml-kit-llm generate --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 \
|
|
3172
|
+
--dataset my-org/prompts --split test \
|
|
3173
|
+
--prompt_column question \
|
|
3174
|
+
--checkpoint ./checkpoints/llm_best.pt \
|
|
3175
|
+
--lora_adapter ./adapters/run1
|
|
3176
|
+
```
|
|
3177
|
+
|
|
3178
|
+
| Argument | Default | Description |
|
|
3179
|
+
|---|---|---|
|
|
3180
|
+
| `--model` | (required) | HuggingFace id or registered custom model. |
|
|
3181
|
+
| `--tokenizer` | `--model` | Tokenizer name/path override. |
|
|
3182
|
+
| `--checkpoint` | `None` | State-dict `.pt`, or a `<root>` resolving to `<root>_best.pt` / `<root>_latest.pt`. |
|
|
3183
|
+
| `--lora_adapter` | `None` | PEFT adapter directory (`save_pretrained` layout). |
|
|
3184
|
+
| `--prompts` / `--dataset` | — | Prompt source (file/JSONL, or HF dataset + `--prompt_column`). |
|
|
3185
|
+
| `--max_new_tokens` | `64` | Tokens to generate. |
|
|
3186
|
+
| `--do_sample` | off | Sample instead of greedy decoding. |
|
|
3187
|
+
| `--temperature` / `--top_k` / `--top_p` | `1.0` / `0` / `1.0` | Sampling knobs. |
|
|
3188
|
+
| `--repetition_penalty` | `1.0` | Down-weight already-seen tokens (`1.0` disables). |
|
|
3189
|
+
| `--num_beams` | `1` | Beam count (portable backend; `1` = greedy). |
|
|
3190
|
+
| `--num_return_sequences` | `1` | Completions per prompt. |
|
|
3191
|
+
| `--no_chat_template` | off | Skip the tokenizer's chat template. |
|
|
3192
|
+
|
|
3193
|
+
When the tokenizer exposes a chat template (most instruct models do),
|
|
3194
|
+
prompts are automatically wrapped as a single user turn; `--no_chat_template`
|
|
3195
|
+
keeps the raw prompt for base-model completion.
|
|
3196
|
+
|
|
3197
|
+
### Evaluation
|
|
3198
|
+
|
|
3199
|
+
`genml-kit-llm eval` reports quality metrics for a labeled dataset:
|
|
3200
|
+
|
|
3201
|
+
- **perplexity** — teacher-forced (`exp` of the mean shifted cross-entropy),
|
|
3202
|
+
over the `--text_column`; no generation involved.
|
|
3203
|
+
- **exact-match** — fraction of generations matching the reference exactly.
|
|
3204
|
+
- **token-F1** — SQuAD-style token-overlap F1 against the reference.
|
|
3205
|
+
- **pass@k** — optional and pluggable: `--pass_at_k K --scorer score.py`
|
|
3206
|
+
invokes `score(prompt, completion, reference) -> float` `K` times per item
|
|
3207
|
+
and reports the pass rate. No code execution is built in.
|
|
3208
|
+
|
|
3209
|
+
<!-- genml-cli-test
|
|
3210
|
+
name: llm-eval
|
|
3211
|
+
network: true
|
|
3212
|
+
override:
|
|
3213
|
+
- --dataset {tmp}/eval
|
|
3214
|
+
- --model sshleifer/tiny-gpt2
|
|
3215
|
+
-->
|
|
3216
|
+
```bash
|
|
3217
|
+
genml-kit-llm eval --model sshleifer/tiny-gpt2 \
|
|
3218
|
+
--dataset ./my_eval_corpus --split test \
|
|
3219
|
+
--text_column text --prompt_column prompt \
|
|
3220
|
+
--reference_column reference \
|
|
3221
|
+
--metrics ppl exact_match token_f1 \
|
|
3222
|
+
--output report.json
|
|
3223
|
+
```
|
|
3224
|
+
|
|
3225
|
+
Both subcommands load their model through the same registry the trainer uses,
|
|
3226
|
+
so a registered custom model generates and evaluates with no code changes.
|
|
3227
|
+
|
|
3228
|
+
### Alignment (DPO and reward models)
|
|
3229
|
+
|
|
3230
|
+
Preference-based post-training runs through the same branchless driver: a new
|
|
3231
|
+
`preference` pipeline (chosen/rejected pairs) pairs with the `dpo` and
|
|
3232
|
+
`reward_model` methods. The typical recipe is SFT → DPO → (optional) reward
|
|
3233
|
+
model:
|
|
3234
|
+
|
|
3235
|
+
<!-- genml-cli-test
|
|
3236
|
+
name: preference-chain
|
|
3237
|
+
network: true
|
|
3238
|
+
extras: lora
|
|
3239
|
+
extra: --epochs 1
|
|
3240
|
+
-->
|
|
3241
|
+
```bash
|
|
3242
|
+
# 1. Supervised fine-tune (masking the prompt out of the loss)
|
|
3243
|
+
genml-kit-train --pipeline text --method causal_lm \
|
|
3244
|
+
--model sshleifer/tiny-gpt2 --dataset ./sft_corpus \
|
|
3245
|
+
--instruction_column prompt --response_column response \
|
|
3246
|
+
--mask_prompt --checkpoint ./sft/model
|
|
3247
|
+
|
|
3248
|
+
# 2. DPO against a frozen reference (defaults to --model)
|
|
3249
|
+
genml-kit-train --pipeline preference --method dpo \
|
|
3250
|
+
--model sshleifer/tiny-gpt2 \
|
|
3251
|
+
--reference_checkpoint ./sft/model_best.pt \
|
|
3252
|
+
--dataset ./preferences --chosen_column chosen \
|
|
3253
|
+
--rejected_column rejected --prompt_column prompt \
|
|
3254
|
+
--mask_prompt --beta 0.1 --lora
|
|
3255
|
+
|
|
3256
|
+
# 3. Train a reward model on the same pairs
|
|
3257
|
+
genml-kit-train --pipeline preference --method reward_model \
|
|
3258
|
+
--model sshleifer/tiny-gpt2 --dataset ./preferences \
|
|
3259
|
+
--chosen_column chosen --rejected_column rejected
|
|
3260
|
+
```
|
|
3261
|
+
|
|
3262
|
+
**DPO** compares the policy against a **frozen reference** and pushes the
|
|
3263
|
+
chosen response's implicit reward above the rejected one. The reference is
|
|
3264
|
+
loaded explicitly (`--reference_model`, defaulting to `--model`, plus
|
|
3265
|
+
`--reference_checkpoint`) and held as a plain attribute — it is never written
|
|
3266
|
+
to `state_dict` and is rebuilt from args on resume. The sequence score is the
|
|
3267
|
+
mean per-token log-probability of the response tokens, and `--beta` (default
|
|
3268
|
+
`0.1`) sets the KL strength. Validation reports pairwise preference accuracy.
|
|
3269
|
+
|
|
3270
|
+
**Reward model** attaches a scalar `nn.Linear` head directly to the backbone
|
|
3271
|
+
as `model.score` — no wrapper class, so the backbone's `state_dict` keys stay
|
|
3272
|
+
unprefixed and `--source_checkpoint` matches them exactly (only `score.*` is
|
|
3273
|
+
new). It scores the last non-pad hidden state and trains with the
|
|
3274
|
+
Bradley–Terry ranking loss; validation reports pairwise ranking accuracy. The
|
|
3275
|
+
hidden size is read from the config (`hidden_size` / `n_embd` / `d_model`),
|
|
3276
|
+
falling back to a one-token probe.
|
|
3277
|
+
|
|
3278
|
+
The backbone must expose last-layer hidden states (accept
|
|
3279
|
+
`output_hidden_states=True`); HuggingFace models do, custom models opt in —
|
|
3280
|
+
the same contract used by `GenerativeLM`.
|
|
3281
|
+
|
|
3282
|
+
---
|
|
3283
|
+
|
|
1668
3284
|
## Reinforcement Learning
|
|
1669
3285
|
|
|
1670
3286
|
genml_kit ships a self-contained reinforcement-learning (RL) toolkit built on
|
|
@@ -1691,7 +3307,12 @@ pip install 'genml_kit[rl]'
|
|
|
1691
3307
|
|
|
1692
3308
|
### Quick start
|
|
1693
3309
|
|
|
1694
|
-
|
|
3310
|
+
<!-- genml-cli-test
|
|
3311
|
+
name: rl-quick-start
|
|
3312
|
+
extras: rl
|
|
3313
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3314
|
+
-->
|
|
3315
|
+
```bash
|
|
1695
3316
|
# DQN on CartPole (discrete)
|
|
1696
3317
|
genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
|
|
1697
3318
|
--epochs 20 --batch_size 64
|
|
@@ -1716,7 +3337,12 @@ Every RL method resolves its network through the same model registry as the
|
|
|
1716
3337
|
rest of the framework: `--model` selects a registered factory and
|
|
1717
3338
|
`--model_arg key=value` forwards constructor kwargs.
|
|
1718
3339
|
|
|
1719
|
-
|
|
3340
|
+
<!-- genml-cli-test
|
|
3341
|
+
name: rl-custom-models
|
|
3342
|
+
extras: rl
|
|
3343
|
+
extra: --epochs 1
|
|
3344
|
+
-->
|
|
3345
|
+
```bash
|
|
1720
3346
|
# DQN: dueling Q-head (also the default for --dqn_dueling)
|
|
1721
3347
|
genml-kit-train --pipeline rl --method dqn --model rl/qnet_dueling
|
|
1722
3348
|
|
|
@@ -1910,12 +3536,34 @@ a small positive default as light regularization.
|
|
|
1910
3536
|
|
|
1911
3537
|
A typical "serious" continuous-PPO setup (Pendulum-style) looks like:
|
|
1912
3538
|
|
|
1913
|
-
|
|
3539
|
+
<!-- genml-cli-test
|
|
3540
|
+
name: rl-ppo-serious
|
|
3541
|
+
extras: rl
|
|
3542
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3543
|
+
-->
|
|
3544
|
+
```bash
|
|
1914
3545
|
genml-kit-train --pipeline rl --method ppo --env_id Pendulum-v1 \
|
|
1915
3546
|
--ppo_continuous --obs_normalize --reward_normalize --param_init ortho \
|
|
1916
3547
|
--ppo_target_kl 0.03 --epochs 500
|
|
1917
3548
|
```
|
|
1918
3549
|
|
|
3550
|
+
SAC's temperature auto-tuning and critic/policy learning rates follow the
|
|
3551
|
+
same pattern (the critic head is `rl/sac_critic`, the policy is
|
|
3552
|
+
`rl/actor_critic`):
|
|
3553
|
+
|
|
3554
|
+
<!-- genml-cli-test
|
|
3555
|
+
name: rl-sac-tuning
|
|
3556
|
+
extras: rl
|
|
3557
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3558
|
+
-->
|
|
3559
|
+
```bash
|
|
3560
|
+
genml-kit-train --pipeline rl --method sac --env_id Pendulum-v1 \
|
|
3561
|
+
--sac_auto_alpha --sac_alpha_lr 3e-4 --sac_target_entropy -1.0 \
|
|
3562
|
+
--sac_critic_lr 3e-4 --sac_actor_lr 3e-4 --sac_tau 0.005 \
|
|
3563
|
+
--replay_capacity 5000 --batch_size 64 \
|
|
3564
|
+
--epochs 200
|
|
3565
|
+
```
|
|
3566
|
+
|
|
1919
3567
|
### Custom environments
|
|
1920
3568
|
|
|
1921
3569
|
Pass `--env_script /path/to/env.py` to load a custom environment. The
|
|
@@ -1931,6 +3579,31 @@ Relevant flags:
|
|
|
1931
3579
|
- `--env_id`, `--obs_dim`, `--env_script` -- environment selection.
|
|
1932
3580
|
- `--replay_capacity`, `--batch_size`, `--n_step`, `--prioritized`,
|
|
1933
3581
|
`--per_alpha`, `--per_beta_start`, `--per_beta_frames` -- replay tuning.
|
|
3582
|
+
|
|
3583
|
+
A DQN run with the full replay/epsilon machinery (dueling head, double-Q,
|
|
3584
|
+
n-step returns, prioritized replay):
|
|
3585
|
+
|
|
3586
|
+
<!-- genml-cli-test
|
|
3587
|
+
name: rl-dqn-replay
|
|
3588
|
+
extras: rl
|
|
3589
|
+
extra: --epochs 1 --env_steps_per_epoch 64 --eval_episodes 2
|
|
3590
|
+
-->
|
|
3591
|
+
```bash
|
|
3592
|
+
genml-kit-train --pipeline rl --method dqn --env_id CartPole-v1 \
|
|
3593
|
+
--dqn_dueling --dqn_ddqn --n_step 3 --prioritized \
|
|
3594
|
+
--replay_capacity 5000 --batch_size 32 \
|
|
3595
|
+
--dqn_gamma 0.99 --dqn_tau 0.005 --dqn_target_update_freq 100 \
|
|
3596
|
+
--dqn_epsilon_start 1.0 --dqn_epsilon_end 0.05 \
|
|
3597
|
+
--dqn_epsilon_decay_steps 500 \
|
|
3598
|
+
--epochs 50
|
|
3599
|
+
```
|
|
3600
|
+
- **Discount factor** -- there is intentionally **no** global `--gamma`: each
|
|
3601
|
+
method owns its own because RL objectives are not active simultaneously and
|
|
3602
|
+
tune it differently. DQN uses `--dqn_gamma`, PPO `--ppo_gamma`, SAC
|
|
3603
|
+
`--sac_gamma` (all default `0.99`). The n-step returns computed by the
|
|
3604
|
+
shared replay buffer use the discount of the **selected method**; the
|
|
3605
|
+
method installs it into the buffer during `wire_data`, so a single run never
|
|
3606
|
+
mixes discounts.
|
|
1934
3607
|
- `--obs_normalize`, `--obs_norm_clip` -- running observation
|
|
1935
3608
|
normalization (Welford). Both train and eval observations are
|
|
1936
3609
|
normalized with the same RMS statistics.
|
|
@@ -1963,6 +3636,209 @@ bootstraps `γ V(s′)`, while a terminated one does not. Environments using
|
|
|
1963
3636
|
the legacy 4-tuple gym API (no `terminated`/`truncated` split) fall back to
|
|
1964
3637
|
treating `done` as `terminated`.
|
|
1965
3638
|
|
|
3639
|
+
## Multi-GPU Training
|
|
3640
|
+
|
|
3641
|
+
The harness runs on a single device by default and needs no launcher for that.
|
|
3642
|
+
To use more than one GPU, launch the same `genml-kit-train` command under
|
|
3643
|
+
`torchrun` (which ships with PyTorch -- it is the `torch.distributed.run`
|
|
3644
|
+
module, so there is nothing extra to install):
|
|
3645
|
+
|
|
3646
|
+
<!-- genml-cli-test-exclude: illustrative GPU/nccl launch; the runnable distributed example follows -->
|
|
3647
|
+
```bash
|
|
3648
|
+
# 4 GPUs, one process per GPU
|
|
3649
|
+
torchrun --standalone --nproc_per_node=4 -m genml_kit.cli.train \
|
|
3650
|
+
--model convvit \
|
|
3651
|
+
--dataset /data/imagenet \
|
|
3652
|
+
--epochs 90 \
|
|
3653
|
+
--batch_size 64 \
|
|
3654
|
+
--dist_arg backend=nccl
|
|
3655
|
+
```
|
|
3656
|
+
|
|
3657
|
+
The same command runs on CPU with the `gloo` backend, which is what the
|
|
3658
|
+
runnable example below does (two ranks, a tiny synthetic dataset):
|
|
3659
|
+
|
|
3660
|
+
<!-- genml-cli-test
|
|
3661
|
+
name: multi-gpu-torchrun
|
|
3662
|
+
launcher: torchrun
|
|
3663
|
+
nproc: 2
|
|
3664
|
+
extra: --device cpu --batch_size 4 --image_size 32 --num_workers 0
|
|
3665
|
+
override:
|
|
3666
|
+
- --dataset {tmp}/img_ddp
|
|
3667
|
+
-->
|
|
3668
|
+
```bash
|
|
3669
|
+
# 2 processes on CPU (gloo); swap in 4 GPUs + `backend=nccl` for a real run
|
|
3670
|
+
torchrun --standalone --nproc_per_node=2 -m genml_kit.cli.train \
|
|
3671
|
+
--model convvit \
|
|
3672
|
+
--dataset /data/imagenet \
|
|
3673
|
+
--epochs 1 \
|
|
3674
|
+
--dist_arg backend=gloo
|
|
3675
|
+
```
|
|
3676
|
+
|
|
3677
|
+
Every rank reads `RANK` / `WORLD_SIZE` / `LOCAL_RANK` / `MASTER_ADDR` /
|
|
3678
|
+
`MASTER_PORT` from the environment; when they are absent (a plain
|
|
3679
|
+
`genml-kit-train ...` invocation) the distributed layer is completely inert --
|
|
3680
|
+
no process group is created, the model is not wrapped, and the data loader is
|
|
3681
|
+
untouched. Single-GPU behaviour is therefore unchanged, and `torchrun` is
|
|
3682
|
+
never required.
|
|
3683
|
+
|
|
3684
|
+
What the launcher turns on automatically:
|
|
3685
|
+
|
|
3686
|
+
| Behaviour | Effect |
|
|
3687
|
+
|---|---|
|
|
3688
|
+
| Process group | `nccl` on CUDA, `gloo` otherwise; override with `--dist_arg backend=` |
|
|
3689
|
+
| Device | each rank pins to `cuda:$LOCAL_RANK` |
|
|
3690
|
+
| Seeding | `base_seed + rank`, so ranks shuffle differently but reproducibly |
|
|
3691
|
+
| Data sharding | each rank gets a disjoint slice of the training set (see below) |
|
|
3692
|
+
| Checkpoints | written by rank 0 only; keys carry no `module.` prefix, so a multi-GPU checkpoint loads back into a single-GPU run unchanged |
|
|
3693
|
+
| Metrics / log files | TensorBoard, JSONL and log files are written by rank 0 only |
|
|
3694
|
+
| Validation | sharded across ranks like training; each rank evaluates its disjoint slice, the per-rank results are merged (`Method.merge_validation`), and rank 0 reports the whole-set metric |
|
|
3695
|
+
|
|
3696
|
+
Distributed knobs travel through a single repeatable `--dist_arg KEY=VALUE`
|
|
3697
|
+
flag, matching `--model_arg` / `--opt_arg`. A bare key means `true`:
|
|
3698
|
+
|
|
3699
|
+
| Key | Default | Description |
|
|
3700
|
+
|---|---|---|
|
|
3701
|
+
| `backend` | auto (`nccl` on CUDA, else `gloo`) | Process-group backend. |
|
|
3702
|
+
| `shard` | `true` when more than one rank | Shard the training set across ranks. Set `shard=false` to run every rank over the full dataset (rarely what you want: it multiplies the compute with no error). |
|
|
3703
|
+
| `find_unused_parameters` | `false` (declared `true` by BYOL / DINO / SupCon) | Pass through to `DistributedDataParallel`. Lets a forward that skips some trainable parameters survive DDP's unused-parameter check. The SSL methods declare it defensively; a stock run leaves nothing unused and would pass without it. |
|
|
3704
|
+
| `static_graph` | `false` | Pass through to `DistributedDataParallel`. Only valid when the autograd graph is identical every step. |
|
|
3705
|
+
| `rollout_shard` | `true` when more than one rank | RL pipelines only. Shard rollout collection (and the minibatch it yields) across ranks. Set `rollout_shard=false` to collect one full rollout per rank (rarely what you want: it multiplies the environment compute with no error). |
|
|
3706
|
+
|
|
3707
|
+
Defaults are chosen so that a multi-GPU run only has to be correct, not
|
|
3708
|
+
tuned: sharding is on whenever it can matter. The validation set is sharded the
|
|
3709
|
+
same way as training, each rank evaluates its disjoint slice, and the per-rank
|
|
3710
|
+
results are merged (`Method.merge_validation`) so the reported metric always
|
|
3711
|
+
describes the whole set; only rank 0 logs it.
|
|
3712
|
+
|
|
3713
|
+
These defaults are *method-aware*: the selected pipeline or method may declare
|
|
3714
|
+
its own default for any flag via `arg_defaults()`, and `--dist_arg` merges on
|
|
3715
|
+
top of it key by key. An explicit command-line flag always wins, so
|
|
3716
|
+
`--dist_arg static_graph` still runs a method's declared
|
|
3717
|
+
`find_unused_parameters` alongside it.
|
|
3718
|
+
|
|
3719
|
+
Three caveats worth knowing before reaching for more GPUs:
|
|
3720
|
+
|
|
3721
|
+
- **Effective batch size scales with `--nproc_per_node`.** Four ranks at
|
|
3722
|
+
`--batch_size 64` see 256 samples per optimizer step; scale `--lr`
|
|
3723
|
+
accordingly.
|
|
3724
|
+
- **`shard` assumes a trainable `DataLoader`.** Pipelines that bypass it are
|
|
3725
|
+
not covered by *that* path: the RL pipelines drive environments and replay
|
|
3726
|
+
buffers directly, so they use their own `rollout_shard` knob (see the table
|
|
3727
|
+
above) instead of `shard`.
|
|
3728
|
+
- **A checkpoint from a multi-GPU run is portable**, but resuming with a
|
|
3729
|
+
*different* `--nproc_per_node` changes the shard boundaries, so the epoch
|
|
3730
|
+
position is approximate if the dataset size is not divisible by the rank
|
|
3731
|
+
count.
|
|
3732
|
+
|
|
3733
|
+
---
|
|
3734
|
+
|
|
3735
|
+
## Hyperparameter Search
|
|
3736
|
+
|
|
3737
|
+
`genml-kit-tune` runs a hyperparameter *study* on top of the unified trainer.
|
|
3738
|
+
It is deliberately decoupled from the training code: **every trial is a fresh
|
|
3739
|
+
`genml-kit-train` subprocess**, and the only contracts between the tuner and a
|
|
3740
|
+
trial are the command line it is handed and the `<log_dir>/metrics.jsonl` file
|
|
3741
|
+
the JSONL sink already writes. No training internals are imported, so the
|
|
3742
|
+
isolation boundary *is* the process boundary.
|
|
3743
|
+
|
|
3744
|
+
A study is data, not code — describe it in YAML (or all on the command line):
|
|
3745
|
+
|
|
3746
|
+
```yaml
|
|
3747
|
+
# study.yaml
|
|
3748
|
+
objective: val/macro_f1 # metric tag to optimise
|
|
3749
|
+
direction: max # max | min
|
|
3750
|
+
sampler: tpe # grid | random | tpe | gpei
|
|
3751
|
+
pruner: asha # none | median | asha
|
|
3752
|
+
max_trials: 50
|
|
3753
|
+
max_epochs_per_trial: 20 # budget caps each trial
|
|
3754
|
+
space:
|
|
3755
|
+
opt.lr: log(1e-4:1e-1) # log-uniform float range
|
|
3756
|
+
batch_size: [32, 64, 128] # categorical choice
|
|
3757
|
+
dropout: 0.0:0.5 # uniform float range
|
|
3758
|
+
model.embed_dim: 64:1024 # integer range
|
|
3759
|
+
base_args: ["--pipeline", "images", "--method", "classification"]
|
|
3760
|
+
```
|
|
3761
|
+
|
|
3762
|
+
### Running a study
|
|
3763
|
+
|
|
3764
|
+
Points after a literal `--` are forwarded verbatim to every trial:
|
|
3765
|
+
|
|
3766
|
+
<!-- genml-cli-test
|
|
3767
|
+
name: tune-run-study
|
|
3768
|
+
override:
|
|
3769
|
+
- --max_trials 2
|
|
3770
|
+
- --workers 1
|
|
3771
|
+
- -.--model
|
|
3772
|
+
- -.--dataset
|
|
3773
|
+
- -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
|
|
3774
|
+
-->
|
|
3775
|
+
```bash
|
|
3776
|
+
genml-kit-tune run \
|
|
3777
|
+
--space opt.lr=log(1e-4:1e-1) --space batch_size=32,64 \
|
|
3778
|
+
--objective val/macro_f1 --direction max --sampler tpe --pruner asha \
|
|
3779
|
+
--max_trials 8 --workers 2 \
|
|
3780
|
+
--epochs 1 \
|
|
3781
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3782
|
+
```
|
|
3783
|
+
|
|
3784
|
+
The equivalent all-YAML form is `genml-kit-tune run --study study.yaml`; command
|
|
3785
|
+
line flags override the file. Trials land under
|
|
3786
|
+
`runs/study-<timestamp>/trials/<id>/` (each with its own `metrics.jsonl` and
|
|
3787
|
+
checkpoints) and the study ledger is `runs/study-<timestamp>/study.jsonl`.
|
|
3788
|
+
|
|
3789
|
+
### Samplers and pruners
|
|
3790
|
+
|
|
3791
|
+
| Name | Kind | Notes |
|
|
3792
|
+
|---|---|---|
|
|
3793
|
+
| `grid` | sampler | Deterministic Cartesian product; resume-safe. |
|
|
3794
|
+
| `random` | sampler | Uniform draws over the space. |
|
|
3795
|
+
| `tpe` | sampler | Tree-structured Parzen Estimator — sample-efficient Bayesian optimisation (numpy only). The default. |
|
|
3796
|
+
| `gpei` | sampler | Gaussian-Process Expected Improvement (registered only when `scipy` is installed; `pip install "genml_kit[tune]"`). |
|
|
3797
|
+
| `none` | pruner | Run every trial to its full budget. |
|
|
3798
|
+
| `median` | pruner | Stop trials below the running median at a rung. |
|
|
3799
|
+
| `asha` | pruner | Asynchronous Successive Halving; keeps the top `1/reduction_factor` at each rung. |
|
|
3800
|
+
| `hyperband` | pruner | Multiple ASHA brackets over `min_resource * rf^s` budgets; trials are assigned round-robin so every resource level gets explored. |
|
|
3801
|
+
|
|
3802
|
+
Pruning is delivered as a clean `SIGTERM` to the trial subprocess, which the
|
|
3803
|
+
trainer's existing signal guard turns into a consistent checkpoint before exit.
|
|
3804
|
+
|
|
3805
|
+
### Concurrency, GPUs and resume
|
|
3806
|
+
|
|
3807
|
+
By default studies run **one trial at a time**. `--workers N` runs `N` trials
|
|
3808
|
+
concurrently, and `--gpus 0,1,2,3` pins each trial to a slot from that pool via
|
|
3809
|
+
Interrupt a study at any time and continue it later (the second command
|
|
3810
|
+
prints the leaderboard of the same study):
|
|
3811
|
+
|
|
3812
|
+
<!-- genml-cli-test
|
|
3813
|
+
name: tune-resume-report
|
|
3814
|
+
override:
|
|
3815
|
+
- --max_trials 2
|
|
3816
|
+
- -.--model
|
|
3817
|
+
- -.--dataset
|
|
3818
|
+
- -- --model convvit --dataset {tmp}/img --image_size 32 --max_items 16
|
|
3819
|
+
-->
|
|
3820
|
+
```bash
|
|
3821
|
+
# First invocation creates the study; a later one resumes it (the budget
|
|
3822
|
+
# already spent is subtracted, so only the remaining trials run).
|
|
3823
|
+
genml-kit-tune run --study_dir runs/study-20250101-120000 \
|
|
3824
|
+
--space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
|
|
3825
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3826
|
+
genml-kit-tune run --study_dir runs/study-20250101-120000 --resume \
|
|
3827
|
+
--space opt.lr=log(1e-4:1e-1) --objective val/macro_f1 --max_trials 50 \
|
|
3828
|
+
-- --model google/vit-base-patch16-224 --dataset uoft-cs/cifar10
|
|
3829
|
+
genml-kit-tune report runs/study-20250101-120000 --top 10
|
|
3830
|
+
```
|
|
3831
|
+
|
|
3832
|
+
### Extending
|
|
3833
|
+
|
|
3834
|
+
Both algorithm families are registries, exactly like `METHODS`/`PIPELINES`.
|
|
3835
|
+
A new sampler subclasses `Sampler` (implementing `suggest(observed, n)`) and
|
|
3836
|
+
decorates itself with `@SEARCHERS.register`; a new pruner subclasses `Pruner`
|
|
3837
|
+
(implementing `_decide(rung, value)`) and uses `@PRUNERS.register`. The
|
|
3838
|
+
subsystem stays zero-dependency: `tpe`, `grid` and `random` need only numpy,
|
|
3839
|
+
and the scipy-backed `gpei` registers itself only when scipy is importable
|
|
3840
|
+
(`pip install "genml_kit[tune]"`).
|
|
3841
|
+
|
|
1966
3842
|
## References And Further Reading
|
|
1967
3843
|
|
|
1968
3844
|
- Xie et al., [SimMIM: A Simple Framework for Masked Image Modeling](https://arxiv.org/abs/2111.09886), CVPR 2022.
|
|
@@ -2003,6 +3879,42 @@ instead of scrolling past. Code is formatted with
|
|
|
2003
3879
|
(Google style, 2-space indent) and linted with
|
|
2004
3880
|
[Ruff](https://docs.astral.sh/ruff/) (`ruff check .`).
|
|
2005
3881
|
|
|
3882
|
+
### Verifying README commands
|
|
3883
|
+
|
|
3884
|
+
Command blocks preceded by a `<!-- genml-cli-test ... -->` HTML comment are
|
|
3885
|
+
executed by `tests/test_readme_cli.py`. The comment is invisible on
|
|
3886
|
+
GitHub; its YAML body controls limiting args, network/token markers, and
|
|
3887
|
+
substitutions. Run them with `pytest -m network` (needs internet, no
|
|
3888
|
+
credentials) — the offline suite excludes them by default.
|
|
3889
|
+
|
|
3890
|
+
A fence marked with `launcher: torchrun` is rewritten to a multi-process
|
|
3891
|
+
`torch.distributed.run` invocation and executed that way (run it with
|
|
3892
|
+
`pytest -m distributed`); this is opt-in per fence, so every other example
|
|
3893
|
+
keeps running single-process.
|
|
3894
|
+
|
|
3895
|
+
A fence that talks to an external service names it with `service: <name>`
|
|
3896
|
+
(`hf`, `s3`, `r2`, `gcs`, `github`, `wandb`). Such a fence is run when the
|
|
3897
|
+
service is reachable either through its credentials **or** through an
|
|
3898
|
+
endpoint-override proxy/emulator, because the underlying client libraries
|
|
3899
|
+
honour their own override variables natively:
|
|
3900
|
+
|
|
3901
|
+
| `service:` | credentials | endpoint override |
|
|
3902
|
+
|---|---|---|
|
|
3903
|
+
| `hf` | `HF_TOKEN`, `HUGGING_FACE_HUB_TOKEN` | `HF_ENDPOINT` |
|
|
3904
|
+
| `s3` | `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` | `AWS_ENDPOINT_URL_S3`, `AWS_ENDPOINT_URL` |
|
|
3905
|
+
| `r2` | `R2_ACCESS_KEY_ID` + `R2_SECRET_ACCESS_KEY` | `R2_ENDPOINT_URL`, `AWS_ENDPOINT_URL_S3` |
|
|
3906
|
+
| `gcs` | `GOOGLE_APPLICATION_CREDENTIALS` | `STORAGE_EMULATOR_HOST` |
|
|
3907
|
+
| `github` | `GITHUB_TOKEN`, `GH_TOKEN` | `GITHUB_API_URL` |
|
|
3908
|
+
| `wandb` | `WANDB_API_KEY` | `WANDB_BASE_URL` |
|
|
3909
|
+
|
|
3910
|
+
Select them with `pytest -m service` (all services) or a per-service marker
|
|
3911
|
+
such as `pytest -m hf`. `service:` implies `network: true`.
|
|
3912
|
+
|
|
3913
|
+
Every `bash`/`sh` block that contains a `genml-kit-*` command must either be
|
|
3914
|
+
marked for execution as above or explicitly opted out with a
|
|
3915
|
+
`<!-- genml-cli-test-exclude: <reason> -->` comment; a test enforces this so a
|
|
3916
|
+
new example cannot be added untested by accident.
|
|
3917
|
+
|
|
2006
3918
|
### Naming Conventions For Class Members
|
|
2007
3919
|
|
|
2008
3920
|
- Anything that PyTorch registers — child `nn.Module`, `nn.Parameter`,
|
|
@@ -2014,7 +3926,7 @@ instead of scrolling past. Code is formatted with
|
|
|
2014
3926
|
leading underscore. A member called from other classes, overridden as
|
|
2015
3927
|
an extension point, or documented as API stays public.
|
|
2016
3928
|
- Extension-point methods that *callers* invoke (e.g.
|
|
2017
|
-
`
|
|
3929
|
+
`Method.build_transform`) are public; same-named hooks the
|
|
2018
3930
|
base class *itself* dispatches (e.g. `BaseImageProcessor._build_transform`)
|
|
2019
3931
|
are private.
|
|
2020
3932
|
|