diffsynth 2.1.0__tar.gz → 2.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {diffsynth-2.1.0 → diffsynth-2.1.2}/PKG-INFO +1 -1
- {diffsynth-2.1.0 → diffsynth-2.1.2}/README.md +48 -35
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/configs/model_configs.py +30 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/configs/vram_management_module_maps.py +13 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/attention/attention.py +41 -3
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/quant/__init__.py +1 -1
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/quant/backends/torchao.py +25 -8
- diffsynth-2.1.2/diffsynth/core/quant/base.py +241 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/quant/config.py +126 -34
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/vram/layers.py +19 -9
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/flow_match.py +25 -2
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/loss.py +12 -11
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/template.py +5 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/training_module.py +6 -3
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ideogram4_dit.py +18 -14
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/minimax_h3_dit.py +4 -3
- diffsynth-2.1.2/diffsynth/models/minimax_h3_dit_comfy_pruned.py +72 -0
- diffsynth-2.1.2/diffsynth/models/wan_animate_2_dit.py +913 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/lingbot_video.py +13 -6
- diffsynth-2.1.2/diffsynth/pipelines/minimax_h3_audio_video.py +911 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/wan_video.py +263 -8
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/data/audio_video.py +83 -7
- diffsynth-2.1.2/diffsynth/utils/lora/minimax_h3.py +97 -0
- diffsynth-2.1.2/diffsynth/utils/quant/serialization.py +57 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/xfuser/__init__.py +1 -1
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/xfuser/xdit_context_parallel.py +13 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth.egg-info/PKG-INFO +1 -1
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth.egg-info/SOURCES.txt +4 -1
- {diffsynth-2.1.0 → diffsynth-2.1.2}/pyproject.toml +1 -1
- diffsynth-2.1.0/diffsynth/core/quant/base.py +0 -127
- diffsynth-2.1.0/diffsynth/models/minimax_constant.py +0 -23
- diffsynth-2.1.0/diffsynth/pipelines/minimax_h3_audio_video.py +0 -1062
- {diffsynth-2.1.0 → diffsynth-2.1.2}/LICENSE +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/configs/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/attention/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/data/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/data/operators.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/data/unified_dataset.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/device/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/device/npu_compatible_device.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/gradient/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/gradient/gradient_checkpoint.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/loader/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/loader/config.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/loader/file.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/loader/model.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/npu_patch/npu_fused_operator.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/offload_training/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/offload_training/manager.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/offload_training/memory_buffer.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/offload_training/offloader.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/quant/backends/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/quant/backends/bitsandbytes.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/vram/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/vram/disk_map.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/core/vram/initialization.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/base_pipeline.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/ddim_scheduler.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/dmd2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/logger.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/parsers.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/diffusion/runner.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/aesthetic.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/base.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/bioclip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/clip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/fid.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/hpsv2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/hpsv3.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/image_reward.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/lpips.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/pickscore.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/qwen_image_bench.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/unified_reward_2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/metrics/unified_reward_edit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_conditioner.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_residual_fsq.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_tokenizer.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ace_step_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/aesthetic.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/anima_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/bioclip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/boogu_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/clip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/demucs.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/dinov3_image_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ernie_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ernie_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/fid.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux2_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux2_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_controlnet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_infiniteyou.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_ipadapter.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_lora_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_lora_patcher.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_text_encoder_clip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_text_encoder_t5.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/flux_value_control.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/general_modules.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/hidream_common.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/hidream_o1_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/hpsv2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/hpsv3.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ideogram4_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ideogram4_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/image_reward.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/joyai_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/joyai_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/krea2_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/krea2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/lingbot_video_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/longcat_video_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/lpips.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_audio_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_common.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_upsampler.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/ltx2_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/minimax_h3_audio_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/minimax_h3_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/minimax_h3_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/model_loader.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/mova_audio_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/mova_audio_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/mova_dual_tower_bridge.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/nexus_gen.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/nexus_gen_ar_model.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/nexus_gen_projector.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/pickscore.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_bench.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_controlnet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_image2lora.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/qwen_image_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/sd_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/siglip2_image_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/stable_diffusion_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/stable_diffusion_unet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/stable_diffusion_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/stable_diffusion_xl_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/stable_diffusion_xl_unet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/step1x_connector.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/step1x_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/unified_reward_2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/unified_reward_edit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_animate_adapter.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_camera_controller.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_dit_s2v.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_image_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_mot.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_motion_controller.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_vace.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wan_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wantodance.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/wav2vec.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/z_image_controlnet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/z_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/z_image_image2lora.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/models/z_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/ace_step.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/anima_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/boogu_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/ernie_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/flux2_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/flux_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/hidream_o1_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/ideogram4.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/joyai_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/krea2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/ltx2_audio_video.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/mova_audio_video.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/qwen_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/stable_diffusion.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/stable_diffusion_xl.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/pipelines/z_image.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/controlnet/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/controlnet/annotator.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/controlnet/controlnet_input.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/data/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/data/audio.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/data/media_io_ltx2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/data/minimax_h3.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/demucs/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/dequantizer/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/flux.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/flux_timestep.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/general.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/krea2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/merge.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/reset_rank.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/lora/sdxl.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/ses/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/ses/ses.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ace_step_conditioner.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ace_step_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ace_step_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ace_step_tokenizer.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/anima_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/dino_v3.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ernie_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_controlnet.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_infiniteyou.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_ipadapter.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_text_encoder_clip.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_text_encoder_t5.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/flux_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ideogram4_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/image_metrics.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/joyai_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/krea2_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/krea2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/lingbot_video_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ltx2_audio_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ltx2_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ltx2_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/ltx2_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/minimax_h3_audio_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/minimax_h3_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/minimax_h3_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/nexus_gen.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/nexus_gen_projector.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/qwen_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/sdxl.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/sdxl_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/sdxl_text_encoder_2.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/sdxl_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/stable_diffusion_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/stable_diffusion_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/stable_diffusion_xl_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/step1x_connector.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_animate_adapter.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_image_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_mot.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_vace.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wan_video_vae.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/wans2v_audio_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/z_image_dit.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/state_dict_converters/z_image_text_encoder.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/tile/__init__.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/utils/tile/tile_worker.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth/version.py +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth.egg-info/dependency_links.txt +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth.egg-info/requires.txt +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/diffsynth.egg-info/top_level.txt +0 -0
- {diffsynth-2.1.0 → diffsynth-2.1.2}/setup.cfg +0 -0
|
@@ -36,6 +36,8 @@ We believe that a well-developed open-source code framework can lower the thresh
|
|
|
36
36
|
|
|
37
37
|
> Currently, the development personnel of this project are limited, with most of the work handled by [Artiprocher](https://github.com/Artiprocher) and [mi804](https://github.com/mi804). Therefore, the progress of new feature development will be relatively slow, and the speed of responding to and resolving issues is limited. We apologize for this and ask developers to understand.
|
|
38
38
|
|
|
39
|
+
- **August 7, 2026** We add support for Wan-Animate-2 in the Wan series. Given a reference image and a driving video, it makes the reference character perform the motions in the driving video, generating high-quality character animation, with both standard and distilled variants. For details, please refer to the [documentation](/docs/en/Model_Details/Wan.md) and [example code](/examples/wanvideo/).
|
|
40
|
+
|
|
39
41
|
- **August 3, 2026** MiniMax-H3 open-sourced, welcome a new member to the video model family! Support includes text-to-video-audio generation, keyframe-guided generation, reference-driven generation, low VRAM inference, and NF4-quantized inference. For details, please refer to the [documentation](/docs/en/Model_Details/MiniMax-H3.md) and [example code](/examples/minimax_h3/).
|
|
40
42
|
|
|
41
43
|
- **July 28, 2026** LingBot-Video open-sourced, welcome a new member to the video model family! This release includes two variants, Dense-1.3B and MoE-30B-A3B (30B total parameters, ~3B active per token), both supporting text-to-video, image-to-video and text-to-image generation, low VRAM inference, and LoRA / full training capabilities. For details, please refer to the [documentation](/docs/en/Model_Details/LingBot-Video.md) and [example code](/examples/lingbot_video/). Huge thanks to [NancyFyong](https://github.com/NancyFyong) for contributing the integration of this model!
|
|
@@ -1413,41 +1415,43 @@ Example code for Wan is available at: [/examples/wanvideo/](/examples/wanvideo/)
|
|
|
1413
1415
|
|
|
1414
1416
|
| Model ID | Extra Inputs | Inference | Low VRAM Inference | Full Training | Validation After Full Training | LoRA Training | Validation After LoRA Training |
|
|
1415
1417
|
|-|-|-|-|-|-|-|-|
|
|
1416
|
-
|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](
|
|
1417
|
-
|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](
|
|
1418
|
-
|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](
|
|
1419
|
-
|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](
|
|
1420
|
-
|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](
|
|
1421
|
-
|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](
|
|
1422
|
-
|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](
|
|
1423
|
-
|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](
|
|
1424
|
-
|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](
|
|
1425
|
-
|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](
|
|
1426
|
-
|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](
|
|
1427
|
-
|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](
|
|
1428
|
-
|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](
|
|
1429
|
-
|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](
|
|
1430
|
-
|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](
|
|
1431
|
-
|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](
|
|
1432
|
-
|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](
|
|
1433
|
-
|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](
|
|
1434
|
-
|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](
|
|
1435
|
-
|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](
|
|
1436
|
-
|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](
|
|
1437
|
-
|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](
|
|
1438
|
-
|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](
|
|
1439
|
-
|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](
|
|
1440
|
-
|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](
|
|
1441
|
-
|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](
|
|
1442
|
-
|[Wan-AI/Wan2.2-
|
|
1443
|
-
|[
|
|
1444
|
-
|[
|
|
1445
|
-
|[PAI/Wan2.2-Fun-A14B
|
|
1446
|
-
|[PAI/Wan2.2-Fun-A14B-
|
|
1447
|
-
|[
|
|
1448
|
-
|[
|
|
1449
|
-
|[
|
|
1450
|
-
|[
|
|
1418
|
+
|[Wan-AI/Wan2.1-T2V-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-1.3B.py)|
|
|
1419
|
+
|[Wan-AI/Wan2.1-T2V-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B)||[code](/examples/wanvideo/model_inference/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-T2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-T2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-T2V-14B.py)|
|
|
1420
|
+
|[Wan-AI/Wan2.1-I2V-14B-480P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-480P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-480P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-480P.py)|
|
|
1421
|
+
|[Wan-AI/Wan2.1-I2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-I2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-I2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-I2V-14B-720P.py)|
|
|
1422
|
+
|[Wan-AI/Wan2.1-FLF2V-14B-720P](https://modelscope.cn/models/Wan-AI/Wan2.1-FLF2V-14B-720P)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-FLF2V-14B-720P.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-FLF2V-14B-720P.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-FLF2V-14B-720P.py)|
|
|
1423
|
+
|[iic/VACE-Wan2.1-1.3B-Preview](https://modelscope.cn/models/iic/VACE-Wan2.1-1.3B-Preview)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B-Preview.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B-Preview.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B-Preview.py)|
|
|
1424
|
+
|[Wan-AI/Wan2.1-VACE-1.3B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-1.3B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-1.3B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-1.3B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-1.3B.py)|
|
|
1425
|
+
|[Wan-AI/Wan2.1-VACE-14B](https://modelscope.cn/models/Wan-AI/Wan2.1-VACE-14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-VACE-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-VACE-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-VACE-14B.py)|
|
|
1426
|
+
|[PAI/Wan2.1-Fun-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-InP.py)|
|
|
1427
|
+
|[PAI/Wan2.1-Fun-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-1.3B-Control.py)|
|
|
1428
|
+
|[PAI/Wan2.1-Fun-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-InP.py)|
|
|
1429
|
+
|[PAI/Wan2.1-Fun-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)|`control_video`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-14B-Control.py)|
|
|
1430
|
+
|[PAI/Wan2.1-Fun-V1.1-1.3B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control.py)|
|
|
1431
|
+
|[PAI/Wan2.1-Fun-V1.1-14B-Control](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control.py)|
|
|
1432
|
+
|[PAI/Wan2.1-Fun-V1.1-1.3B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-InP.py)|
|
|
1433
|
+
|[PAI/Wan2.1-Fun-V1.1-14B-InP](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-InP.py)|
|
|
1434
|
+
|[PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-1.3B-Control-Camera.py)|
|
|
1435
|
+
|[PAI/Wan2.1-Fun-V1.1-14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-Fun-V1.1-14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-Fun-V1.1-14B-Control-Camera.py)|
|
|
1436
|
+
|[DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1](https://modelscope.cn/models/DiffSynth-Studio/Wan2.1-1.3b-speedcontrol-v1)|`motion_bucket_id`|[code](/examples/wanvideo/model_inference/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/full/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.1-1.3b-speedcontrol-v1.py)|[code](/examples/wanvideo/model_training/lora/Wan2.1-1.3b-speedcontrol-v1.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.1-1.3b-speedcontrol-v1.py)|
|
|
1437
|
+
|[krea/krea-realtime-video](https://www.modelscope.cn/models/krea/krea-realtime-video)||[code](/examples/wanvideo/model_inference/krea-realtime-video.py)|[code](/examples/wanvideo/model_inference_low_vram/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/full/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_full/krea-realtime-video.py)|[code](/examples/wanvideo/model_training/lora/krea-realtime-video.sh)|[code](/examples/wanvideo/model_training/validate_lora/krea-realtime-video.py)|
|
|
1438
|
+
|[meituan-longcat/LongCat-Video](https://www.modelscope.cn/models/meituan-longcat/LongCat-Video)|`longcat_video`|[code](/examples/wanvideo/model_inference/LongCat-Video.py)|[code](/examples/wanvideo/model_inference_low_vram/LongCat-Video.py)|[code](/examples/wanvideo/model_training/full/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_full/LongCat-Video.py)|[code](/examples/wanvideo/model_training/lora/LongCat-Video.sh)|[code](/examples/wanvideo/model_training/validate_lora/LongCat-Video.py)|
|
|
1439
|
+
|[ByteDance/Video-As-Prompt-Wan2.1-14B](https://modelscope.cn/models/ByteDance/Video-As-Prompt-Wan2.1-14B)|`vap_video`, `vap_prompt`|[code](/examples/wanvideo/model_inference/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/full/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Video-As-Prompt-Wan2.1-14B.py)|[code](/examples/wanvideo/model_training/lora/Video-As-Prompt-Wan2.1-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Video-As-Prompt-Wan2.1-14B.py)|
|
|
1440
|
+
|[Wan-AI/Wan2.2-T2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B)||[code](/examples/wanvideo/model_inference/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-T2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-T2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-T2V-A14B.py)|
|
|
1441
|
+
|[Wan-AI/Wan2.2-I2V-A14B](https://modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-I2V-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-I2V-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-I2V-A14B.py)|
|
|
1442
|
+
|[Wan-AI/Wan2.2-TI2V-5B](https://modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B)|`input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-TI2V-5B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-TI2V-5B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-TI2V-5B.py)|
|
|
1443
|
+
|[Wan-AI/Wan2.2-Animate-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-14B)|`input_image`, `animate_pose_video`, `animate_face_video`, `animate_inpaint_video`, `animate_mask_video`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-14B.py)|
|
|
1444
|
+
|[Wan-AI/Wan2.2-Animate-2-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B.py)|
|
|
1445
|
+
|[Wan-AI/Wan2.2-Animate-2-14B: Distilled](https://www.modelscope.cn/models/Wan-AI/Wan2.2-Animate-2-14B)|`animate2_reference_image`, `animate2_reference_video`, `animate2_prompt_ref`|[code](/examples/wanvideo/model_inference/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Animate-2-14B-Distilled.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Animate-2-14B-Distilled.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Animate-2-14B-Distilled.py)|
|
|
1446
|
+
|[Wan-AI/Wan2.2-S2V-14B](https://www.modelscope.cn/models/Wan-AI/Wan2.2-S2V-14B)|`input_image`, `input_audio`, `audio_sample_rate`, `s2v_pose_video`|[code](/examples/wanvideo/model_inference/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-S2V-14B_multi_clips.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-S2V-14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-S2V-14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-S2V-14B.py)|
|
|
1447
|
+
|[PAI/Wan2.2-VACE-Fun-A14B](https://www.modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)|`vace_control_video`, `vace_reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-VACE-Fun-A14B.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-VACE-Fun-A14B.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-VACE-Fun-A14B.py)|
|
|
1448
|
+
|[PAI/Wan2.2-Fun-A14B-InP](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP)|`input_image`, `end_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-InP.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-InP.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-InP.py)|
|
|
1449
|
+
|[PAI/Wan2.2-Fun-A14B-Control](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)|`control_video`, `reference_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control.py)|
|
|
1450
|
+
|[PAI/Wan2.2-Fun-A14B-Control-Camera](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)|`control_camera_video`, `input_image`|[code](/examples/wanvideo/model_inference/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/full/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan2.2-Fun-A14B-Control-Camera.py)|[code](/examples/wanvideo/model_training/lora/Wan2.2-Fun-A14B-Control-Camera.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan2.2-Fun-A14B-Control-Camera.py)|
|
|
1451
|
+
|[openmoss/MOVA-360p](https://modelscope.cn/models/openmoss/MOVA-360p)|`input_image`|[code](/examples/mova/model_inference/MOVA-360p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-360p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-360P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-360p-I2AV.py)|
|
|
1452
|
+
|[openmoss/MOVA-720p](https://modelscope.cn/models/openmoss/MOVA-720p)|`input_image`|[code](/examples/mova/model_inference/MOVA-720p-I2AV.py)|[code](/examples/mova/model_inference_low_vram/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/full/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_full/MOVA-720p-I2AV.py)|[code](/examples/mova/model_training/lora/MOVA-720P-I2AV.sh)|[code](/examples/mova/model_training/validate_lora/MOVA-720p-I2AV.py)|
|
|
1453
|
+
|[Wan-AI/Wan-Dancer-14B (global model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-global.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-global.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-global.py)|
|
|
1454
|
+
|[Wan-AI/Wan-Dancer-14B (local model)](https://modelscope.cn/models/Wan-AI/Wan-Dancer-14B)|`wantodance_music_path`, `wantodance_reference_image`, `wantodance_fps`, `wantodance_keyframes`, `wantodance_keyframes_mask`|[code](/examples/wanvideo/model_inference/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_inference_low_vram/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/full/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_full/Wan-Dancer-14B-local.py)|[code](/examples/wanvideo/model_training/lora/Wan-Dancer-14B-local.sh)|[code](/examples/wanvideo/model_training/validate_lora/Wan-Dancer-14B-local.py)|
|
|
1451
1455
|
|
|
1452
1456
|
</details>
|
|
1453
1457
|
|
|
@@ -1523,6 +1527,8 @@ Example code for LingBot-Video is available at: [/examples/lingbot_video/](/exam
|
|
|
1523
1527
|
|[Robbyant/lingbot-video-moe-30b-a3b: T2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_t2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_t2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_t2v.py)|
|
|
1524
1528
|
|[Robbyant/lingbot-video-moe-30b-a3b: TI2V](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/full/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_full/lingbot-video-moe-30b-a3b_ti2v.py)|[code](/examples/lingbot_video/model_training/lora/lingbot-video-moe-30b-a3b_ti2v.sh)|[code](/examples/lingbot_video/model_training/validate_lora/lingbot-video-moe-30b-a3b_ti2v.py)|
|
|
1525
1529
|
|[Robbyant/lingbot-video-moe-30b-a3b: T2I](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2i.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2i.py)|-|-|-|-|
|
|
1530
|
+
|[Robbyant/lingbot-video-moe-30b-a3b: T2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_t2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_t2v_refiner.py)|-|-|-|-|
|
|
1531
|
+
|[Robbyant/lingbot-video-moe-30b-a3b: TI2V + Refinement](https://modelscope.cn/models/Robbyant/lingbot-video-moe-30b-a3b)|[code](/examples/lingbot_video/model_inference/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|[code](/examples/lingbot_video/model_inference_low_vram/lingbot-video-moe-30b-a3b_ti2v_refiner.py)|-|-|-|-|
|
|
1526
1532
|
|
|
1527
1533
|
</details>
|
|
1528
1534
|
|
|
@@ -1586,8 +1592,15 @@ Example code for MiniMax-H3 is available at: [/examples/minimax_h3/](/examples/m
|
|
|
1586
1592
|
|-|-|-|-|-|-|-|
|
|
1587
1593
|
|[MiniMax/MiniMax-H3: FL2VA](https://www.modelscope.cn/models/MiniMax/MiniMax-H3)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-FL2VA.py)|[code](/examples/minimax_h3/model_training/full/MiniMax-H3-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_full/MiniMax-H3-FL2VA.py)|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-FL2VA.py)|
|
|
1588
1594
|
|[MiniMax/MiniMax-H3: Ref2VA](https://www.modelscope.cn/models/MiniMax/MiniMax-H3)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Ref2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Ref2VA.py)|[code](/examples/minimax_h3/model_training/full/MiniMax-H3-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_full/MiniMax-H3-Ref2VA.py)|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-Ref2VA.py)|
|
|
1595
|
+
|[MiniMax/MiniMax-H3: Retake](https://www.modelscope.cn/models/MiniMax/MiniMax-H3)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Retake.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Retake.py)|-|-|-|-|
|
|
1589
1596
|
|[DiffSynth-Studio/MiniMax-H3-NF4: FL2VA](https://www.modelscope.cn/models/DiffSynth-Studio/MiniMax-H3-NF4)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-NF4-FL2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-NF4-FL2VA.py)|-|-|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-NF4-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-NF4-FL2VA.py)|
|
|
1590
1597
|
|[DiffSynth-Studio/MiniMax-H3-NF4: Ref2VA](https://www.modelscope.cn/models/DiffSynth-Studio/MiniMax-H3-NF4)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-NF4-Ref2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-NF4-Ref2VA.py)|-|-|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-NF4-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-NF4-Ref2VA.py)|
|
|
1598
|
+
|[Comfy-Org/MiniMax-H3: FL2VA pruned](https://www.modelscope.cn/models/Comfy-Org/MiniMax-H3)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Pruned-FL2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Pruned-FL2VA.py)|[code](/examples/minimax_h3/model_training/full/MiniMax-H3-Pruned-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_full/MiniMax-H3-Pruned-FL2VA.py)|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-Pruned-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-Pruned-FL2VA.py)|
|
|
1599
|
+
|[Comfy-Org/MiniMax-H3: Ref2VA pruned](https://www.modelscope.cn/models/Comfy-Org/MiniMax-H3)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Pruned-Ref2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Pruned-Ref2VA.py)|[code](/examples/minimax_h3/model_training/full/MiniMax-H3-Pruned-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_full/MiniMax-H3-Pruned-Ref2VA.py)|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-Pruned-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-Pruned-Ref2VA.py)|
|
|
1600
|
+
|[DiffSynth-Studio/MiniMax-H3-NF4: FL2VA pruned](https://www.modelscope.cn/models/DiffSynth-Studio/MiniMax-H3-NF4)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Pruned-NF4-FL2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Pruned-NF4-FL2VA.py)|-|-|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-Pruned-NF4-FL2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-Pruned-NF4-FL2VA.py)|
|
|
1601
|
+
|[DiffSynth-Studio/MiniMax-H3-NF4: Ref2VA pruned](https://www.modelscope.cn/models/DiffSynth-Studio/MiniMax-H3-NF4)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Pruned-NF4-Ref2VA.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Pruned-NF4-Ref2VA.py)|-|-|[code](/examples/minimax_h3/model_training/lora/MiniMax-H3-Pruned-NF4-Ref2VA.sh)|[code](/examples/minimax_h3/model_training/validate_lora/MiniMax-H3-Pruned-NF4-Ref2VA.py)|
|
|
1602
|
+
|[lightx2v/Minimax-h3-Turbo: FL2VA 4steps](https://www.modelscope.cn/models/lightx2v/Minimax-h3-Turbo)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/minimax_h3/model_inference/MiniMax-H3-FL2VA-Turbo.py)|[code](https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-FL2VA-Turbo.py)|-|-|-|-|
|
|
1603
|
+
|[DiffSynth-Studio/MiniMax-H3-Text-Embeddings](https://www.modelscope.cn/models/DiffSynth-Studio/MiniMax-H3-Text-Embeddings)|[code](/examples/minimax_h3/model_inference/MiniMax-H3-Text-Embeddings.py)|[code](/examples/minimax_h3/model_inference_low_vram/MiniMax-H3-Text-Embeddings.py)|[code](/examples/minimax_h3/model_training/full/MiniMax-H3-Text-Embeddings.sh)|[code](/examples/minimax_h3/model_training/validate_full/MiniMax-H3-Text-Embeddings.py)|-|-|
|
|
1591
1604
|
|
|
1592
1605
|
</details>
|
|
1593
1606
|
|
|
@@ -81,6 +81,13 @@ qwen_image_series = [
|
|
|
81
81
|
]
|
|
82
82
|
|
|
83
83
|
wan_series = [
|
|
84
|
+
{
|
|
85
|
+
# Example: ModelConfig(model_id="Wan-AI/Wan2.2-Animate-2-14B", origin_file_pattern="wan_animate_2/wan_animate_2_bf16.safetensors")
|
|
86
|
+
"model_hash": "4536c21ad8740ba78367af4216ae85bf",
|
|
87
|
+
"model_name": "wan_video_dit",
|
|
88
|
+
"model_class": "diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer",
|
|
89
|
+
"extra_kwargs": {},
|
|
90
|
+
},
|
|
84
91
|
{
|
|
85
92
|
# Example: ModelConfig(model_id="krea/krea-realtime-video", origin_file_pattern="krea-realtime-video-14b.safetensors")
|
|
86
93
|
"model_hash": "5ec04e02b42d2580483ad69f4e76346a",
|
|
@@ -1393,6 +1400,13 @@ minimax_h3_series = [
|
|
|
1393
1400
|
"model_name": "minimax_h3_dit",
|
|
1394
1401
|
"model_class": "diffsynth.models.minimax_h3_dit.MiniMaxH3DiT",
|
|
1395
1402
|
},
|
|
1403
|
+
{
|
|
1404
|
+
# Example: ModelConfig(model_id="Comfy-Org/MiniMax-H3", origin_file_pattern="diffusion_models/minimax_h3_fl2va_pruned_bf16.safetensors")
|
|
1405
|
+
"model_hash": "b4e0df87da0c079979ae857edbcef49f",
|
|
1406
|
+
"model_name": "minimax_h3_dit",
|
|
1407
|
+
"model_class": "diffsynth.models.minimax_h3_dit_comfy_pruned.MiniMaxH3DiTComfyPruned",
|
|
1408
|
+
"extra_kwargs": {"adaln_curve_grid": 1025, "time_embed_dim": 8},
|
|
1409
|
+
},
|
|
1396
1410
|
{
|
|
1397
1411
|
# Example: ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="minimax-h3-fl2va-nf4.safetensors")
|
|
1398
1412
|
"model_hash": "4b27efacefbc4d8670e0d7b876699648",
|
|
@@ -1407,6 +1421,22 @@ minimax_h3_series = [
|
|
|
1407
1421
|
"model_class": "diffsynth.models.minimax_h3_dit.MiniMaxH3DiT",
|
|
1408
1422
|
"quant_config": {"method": "bitsandbytes_nf4", "load_prequantized": True, "exclude_modules": ["time_embedder.proj_in", "time_embedder.proj_out", "video_patch_proj", "audio_patch_proj", "condition_proj", "final_layer.video_out", "final_layer.audio_out"]},
|
|
1409
1423
|
},
|
|
1424
|
+
{
|
|
1425
|
+
# Example: ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="minimax-h3-fl2va-pruned-nf4.safetensors")
|
|
1426
|
+
"model_hash": "9b8f1cbc6fb5fcd2e2c02b6691480427",
|
|
1427
|
+
"model_name": "minimax_h3_dit",
|
|
1428
|
+
"model_class": "diffsynth.models.minimax_h3_dit_comfy_pruned.MiniMaxH3DiTComfyPruned",
|
|
1429
|
+
"extra_kwargs": {"adaln_curve_grid": 1025, "time_embed_dim": 8},
|
|
1430
|
+
"quant_config": {"method": "bitsandbytes_nf4", "load_prequantized": True, "exclude_modules": ["time_embedder.proj_in", "time_embedder.proj_out", "video_patch_proj", "audio_patch_proj", "condition_proj", "final_layer.video_out", "final_layer.audio_out", "adaln_proj.linear"]},
|
|
1431
|
+
},
|
|
1432
|
+
{
|
|
1433
|
+
# Example: ModelConfig(model_id="DiffSynth-Studio/MiniMax-H3-NF4", origin_file_pattern="minimax-h3-ref2va-pruned-nf4.safetensors")
|
|
1434
|
+
"model_hash": "072bd13242d6d67df79a154e7c4274b3",
|
|
1435
|
+
"model_name": "minimax_h3_dit",
|
|
1436
|
+
"model_class": "diffsynth.models.minimax_h3_dit_comfy_pruned.MiniMaxH3DiTComfyPruned",
|
|
1437
|
+
"extra_kwargs": {"adaln_curve_grid": 1025, "time_embed_dim": 8},
|
|
1438
|
+
"quant_config": {"method": "bitsandbytes_nf4", "load_prequantized": True, "exclude_modules": ["time_embedder.proj_in", "time_embedder.proj_out", "video_patch_proj", "audio_patch_proj", "condition_proj", "final_layer.video_out", "final_layer.audio_out", "adaln_proj.linear"]},
|
|
1439
|
+
},
|
|
1410
1440
|
{
|
|
1411
1441
|
# Example: ModelConfig(model_id="MiniMax/MiniMax-H3-FL2VA", origin_file_pattern="video_vae/source/model.safetensors")
|
|
1412
1442
|
"model_hash": "24b80900992e2024fab17c991c57da23",
|
|
@@ -14,6 +14,10 @@ VRAM_MANAGEMENT_MODULE_MAPS = {
|
|
|
14
14
|
"torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear",
|
|
15
15
|
"torch.nn.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
16
16
|
},
|
|
17
|
+
"diffsynth.models.minimax_h3_dit_comfy_pruned.MiniMaxH3DiTComfyPruned": {
|
|
18
|
+
"torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear",
|
|
19
|
+
"torch.nn.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
20
|
+
},
|
|
17
21
|
"diffsynth.models.minimax_h3_text_encoder.MiniMaxH3TextEncoder": {
|
|
18
22
|
"torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear",
|
|
19
23
|
"torch.nn.Embedding": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
@@ -114,6 +118,15 @@ VRAM_MANAGEMENT_MODULE_MAPS = {
|
|
|
114
118
|
"diffsynth.models.wan_video_dit.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
115
119
|
"torch.nn.Conv2d": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
116
120
|
},
|
|
121
|
+
"diffsynth.models.wan_animate_2_dit.WanAnimate2Transformer": {
|
|
122
|
+
"diffsynth.models.wan_video_dit.MLP": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
123
|
+
"diffsynth.models.wan_animate_2_dit.AttentionBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule",
|
|
124
|
+
"diffsynth.models.wan_animate_2_dit.Head": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
125
|
+
"torch.nn.Linear": "diffsynth.core.vram.layers.AutoWrappedLinear",
|
|
126
|
+
"torch.nn.Conv3d": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
127
|
+
"torch.nn.LayerNorm": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
128
|
+
"diffsynth.models.wan_video_dit.RMSNorm": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
129
|
+
},
|
|
117
130
|
"diffsynth.models.wan_video_dit.WanModel": {
|
|
118
131
|
"diffsynth.models.wan_video_dit.MLP": "diffsynth.core.vram.layers.AutoWrappedModule",
|
|
119
132
|
"diffsynth.models.wan_video_dit.DiTBlock": "diffsynth.core.vram.layers.AutoWrappedNonRecurseModule",
|
|
@@ -1,6 +1,11 @@
|
|
|
1
1
|
import torch, os, inspect
|
|
2
2
|
from einops import rearrange, repeat
|
|
3
3
|
|
|
4
|
+
try:
|
|
5
|
+
from flash_attn.cute import flash_attn_func as flash_attn_func_cute
|
|
6
|
+
FLASH_ATTN_4_AVAILABLE = True
|
|
7
|
+
except ModuleNotFoundError:
|
|
8
|
+
FLASH_ATTN_4_AVAILABLE = False
|
|
4
9
|
|
|
5
10
|
try:
|
|
6
11
|
import flash_attn_interface
|
|
@@ -26,6 +31,13 @@ try:
|
|
|
26
31
|
except ModuleNotFoundError:
|
|
27
32
|
XFORMERS_AVAILABLE = False
|
|
28
33
|
|
|
34
|
+
try:
|
|
35
|
+
from torch.nn.attention.flex_attention import flex_attention as flex_attention_func
|
|
36
|
+
flex_attention_func = torch.compile(flex_attention_func, dynamic=False, mode="max-autotune-no-cudagraphs", fullgraph=True, backend="inductor")
|
|
37
|
+
FLEX_ATTN_AVAILABLE = True
|
|
38
|
+
except (ModuleNotFoundError, ImportError):
|
|
39
|
+
FLEX_ATTN_AVAILABLE = False
|
|
40
|
+
|
|
29
41
|
try:
|
|
30
42
|
if "enable_gqa" in inspect.signature(torch.nn.functional.scaled_dot_product_attention).parameters:
|
|
31
43
|
TORCH_SUPPORT_GQA = True
|
|
@@ -38,6 +50,8 @@ except:
|
|
|
38
50
|
def initialize_attention_priority():
|
|
39
51
|
if os.environ.get('DIFFSYNTH_ATTENTION_IMPLEMENTATION') is not None:
|
|
40
52
|
return os.environ.get('DIFFSYNTH_ATTENTION_IMPLEMENTATION').lower()
|
|
53
|
+
elif FLASH_ATTN_4_AVAILABLE:
|
|
54
|
+
return "flash_attention_4"
|
|
41
55
|
elif FLASH_ATTN_3_AVAILABLE:
|
|
42
56
|
return "flash_attention_3"
|
|
43
57
|
elif FLASH_ATTN_2_AVAILABLE:
|
|
@@ -133,11 +147,22 @@ def torch_sdpa_sliding_window(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor,
|
|
|
133
147
|
return output
|
|
134
148
|
|
|
135
149
|
|
|
150
|
+
def flash_attention_4(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, scale=None, is_causal=False, window_size=None):
|
|
151
|
+
required_in_pattern, required_out_pattern= "b s n d", "b s n d"
|
|
152
|
+
q, k, v = rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, required_in_pattern, dims)
|
|
153
|
+
window_size = (window_size, window_size) if window_size is not None else (-1, -1)
|
|
154
|
+
out = flash_attn_func_cute(q, k, v, softmax_scale=scale, causal=is_causal, window_size=window_size)
|
|
155
|
+
if isinstance(out, tuple):
|
|
156
|
+
out = out[0]
|
|
157
|
+
out = rearrange_out(out, out_pattern, required_out_pattern, dims)
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
136
161
|
def flash_attention_3(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, scale=None, is_causal=False, window_size=None):
|
|
137
162
|
required_in_pattern, required_out_pattern= "b s n d", "b s n d"
|
|
138
163
|
q, k, v = rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, required_in_pattern, dims)
|
|
139
164
|
window_size = (window_size, window_size) if window_size is not None else (-1, -1)
|
|
140
|
-
out = flash_attn_interface.flash_attn_func(q, k, v, softmax_scale=scale, window_size=window_size)
|
|
165
|
+
out = flash_attn_interface.flash_attn_func(q, k, v, softmax_scale=scale, causal=is_causal, window_size=window_size)
|
|
141
166
|
if isinstance(out, tuple):
|
|
142
167
|
out = out[0]
|
|
143
168
|
out = rearrange_out(out, out_pattern, required_out_pattern, dims)
|
|
@@ -169,14 +194,27 @@ def xformers_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_patt
|
|
|
169
194
|
return out
|
|
170
195
|
|
|
171
196
|
|
|
172
|
-
def
|
|
197
|
+
def flex_attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, score_mod=None):
|
|
198
|
+
assert FLEX_ATTN_AVAILABLE, "Flex Attention is not available. Please upgrade torch to 2.5.0 or later."
|
|
199
|
+
required_in_pattern, required_out_pattern = "b n s d", "b n s d"
|
|
200
|
+
q, k, v = rearrange_qkv(q, k, v, q_pattern, k_pattern, v_pattern, required_in_pattern, dims)
|
|
201
|
+
out = flex_attention_func(query=q, key=k, value=v, block_mask=attn_mask, scale=scale, score_mod=score_mod)
|
|
202
|
+
out = rearrange_out(out, out_pattern, required_out_pattern, dims)
|
|
203
|
+
return out
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def attention_forward(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, q_pattern="b n s d", k_pattern="b n s d", v_pattern="b n s d", out_pattern="b n s d", dims=None, attn_mask=None, scale=None, is_causal=False, compatibility_mode=False, window_size=None, use_flex=False, score_mod=None):
|
|
173
207
|
if compatibility_mode or (attn_mask is not None) or ATTENTION_IMPLEMENTATION == "torch":
|
|
174
|
-
if
|
|
208
|
+
if use_flex or score_mod is not None:
|
|
209
|
+
return flex_attention(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale, score_mod=score_mod)
|
|
210
|
+
elif window_size is None:
|
|
175
211
|
return torch_sdpa(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, attn_mask=attn_mask, scale=scale, is_causal=is_causal)
|
|
176
212
|
else:
|
|
177
213
|
# Sliding Window Attention is not compatible with `is_causal` and `attn_mask`.
|
|
178
214
|
assert is_causal == False and attn_mask is None
|
|
179
215
|
return torch_sdpa_sliding_window(q, k, v, window_size, q_pattern, k_pattern, v_pattern, out_pattern, dims, scale=scale)
|
|
216
|
+
elif ATTENTION_IMPLEMENTATION == "flash_attention_4":
|
|
217
|
+
return flash_attention_4(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, scale=scale, is_causal=is_causal, window_size=window_size)
|
|
180
218
|
elif ATTENTION_IMPLEMENTATION == "flash_attention_3":
|
|
181
219
|
return flash_attention_3(q, k, v, q_pattern, k_pattern, v_pattern, out_pattern, dims, scale=scale, is_causal=is_causal, window_size=window_size)
|
|
182
220
|
elif ATTENTION_IMPLEMENTATION == "flash_attention_2":
|
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
from .base import QuantBackend, QUANT_BACKENDS, register_quant_backend, check_differentiable
|
|
1
|
+
from .base import QuantBackend, QUANT_BACKENDS, register_quant_backend, check_differentiable, check_backend_contract, resolve_checkpoint_keys
|
|
2
2
|
from .config import QuantizeConfig, MixedQuantizeConfig, QuantMethodSpec, QUANT_METHODS, register_quant_method, describe_quant_method
|
|
3
3
|
from . import backends
|
|
@@ -1,10 +1,15 @@
|
|
|
1
1
|
import importlib.util
|
|
2
|
+
import json
|
|
2
3
|
|
|
3
4
|
import torch
|
|
4
5
|
from ..base import QuantBackend, register_quant_backend
|
|
5
6
|
from ..config import register_quant_method
|
|
6
7
|
|
|
7
8
|
|
|
9
|
+
class TorchaoLinear(torch.nn.Linear):
|
|
10
|
+
"""Marker class for torchao-quantized Linears."""
|
|
11
|
+
|
|
12
|
+
|
|
8
13
|
@register_quant_backend("torchao")
|
|
9
14
|
class TorchaoQuantBackend(QuantBackend):
|
|
10
15
|
"""Adapter over torchao `quantize_` (weight-only configs); the quantization lives in the weight tensor subclass, not the module class."""
|
|
@@ -21,27 +26,32 @@ class TorchaoQuantBackend(QuantBackend):
|
|
|
21
26
|
def capabilities(self):
|
|
22
27
|
return {
|
|
23
28
|
"is_serializable": True,
|
|
24
|
-
"is_differentiable":
|
|
29
|
+
"is_differentiable": True,
|
|
25
30
|
"is_compileable": True,
|
|
26
31
|
"requires_calibration": False,
|
|
27
32
|
}
|
|
28
33
|
|
|
29
|
-
def
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
def quantized_linear_classes(self):
|
|
35
|
+
return (TorchaoLinear,)
|
|
36
|
+
|
|
37
|
+
def checkpoint_key_patterns(self):
|
|
38
|
+
return ("weight", "_weight_qdata", "_weight_scale", "_weight_zero_point", "bias")
|
|
32
39
|
|
|
33
40
|
def create_quantized_linear(self, linear, compute_device=None, model_device=None):
|
|
34
41
|
from torchao.quantization import quantize_
|
|
35
42
|
linear.requires_grad_(False)
|
|
36
43
|
if compute_device is not None:
|
|
37
44
|
linear = linear.to(device=compute_device)
|
|
38
|
-
|
|
45
|
+
quant_linear = TorchaoLinear(linear.in_features, linear.out_features, bias=linear.bias is not None, device="meta")
|
|
46
|
+
quant_linear.weight = linear.weight
|
|
47
|
+
quant_linear.bias = linear.bias
|
|
48
|
+
quantize_(quant_linear, self.config)
|
|
39
49
|
if model_device is not None:
|
|
40
|
-
|
|
41
|
-
return
|
|
50
|
+
quant_linear = quant_linear.to(device=model_device)
|
|
51
|
+
return quant_linear
|
|
42
52
|
|
|
43
53
|
def create_quantized_linear_shell(self, linear, compute_dtype):
|
|
44
|
-
return
|
|
54
|
+
return TorchaoLinear(linear.in_features, linear.out_features, bias=linear.bias is not None, device="meta")
|
|
45
55
|
|
|
46
56
|
def flatten_state_dict(self, state_dict):
|
|
47
57
|
self._require_safetensors_support()
|
|
@@ -61,9 +71,16 @@ class TorchaoQuantBackend(QuantBackend):
|
|
|
61
71
|
"safetensors header), so its tensor subclasses cannot be rebuilt. It was most "
|
|
62
72
|
"likely not saved by torchao."
|
|
63
73
|
)
|
|
74
|
+
tensor_names = json.loads(metadata["tensor_names"])
|
|
75
|
+
root_names = [name for name in tensor_names if "." not in name]
|
|
76
|
+
if root_names:
|
|
77
|
+
metadata = {**metadata, "tensor_names": json.dumps([name for name in tensor_names if "." in name])}
|
|
64
78
|
rebuilt = unflatten_tensor_state_dict(state_dict, metadata)
|
|
65
79
|
if isinstance(rebuilt, tuple):
|
|
66
80
|
rebuilt = rebuilt[0]
|
|
81
|
+
for name in root_names:
|
|
82
|
+
if name in state_dict:
|
|
83
|
+
rebuilt[name] = state_dict[name]
|
|
67
84
|
return rebuilt
|
|
68
85
|
|
|
69
86
|
def _require_safetensors_support(self):
|