diffusers 0.27.0__py3-none-any.whl → 0.32.2__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- diffusers/__init__.py +233 -6
- diffusers/callbacks.py +209 -0
- diffusers/commands/env.py +102 -6
- diffusers/configuration_utils.py +45 -16
- diffusers/dependency_versions_table.py +4 -3
- diffusers/image_processor.py +434 -110
- diffusers/loaders/__init__.py +42 -9
- diffusers/loaders/ip_adapter.py +626 -36
- diffusers/loaders/lora_base.py +900 -0
- diffusers/loaders/lora_conversion_utils.py +991 -125
- diffusers/loaders/lora_pipeline.py +3812 -0
- diffusers/loaders/peft.py +571 -7
- diffusers/loaders/single_file.py +405 -173
- diffusers/loaders/single_file_model.py +385 -0
- diffusers/loaders/single_file_utils.py +1783 -713
- diffusers/loaders/textual_inversion.py +41 -23
- diffusers/loaders/transformer_flux.py +181 -0
- diffusers/loaders/transformer_sd3.py +89 -0
- diffusers/loaders/unet.py +464 -540
- diffusers/loaders/unet_loader_utils.py +163 -0
- diffusers/models/__init__.py +76 -7
- diffusers/models/activations.py +65 -10
- diffusers/models/adapter.py +53 -53
- diffusers/models/attention.py +605 -18
- diffusers/models/attention_flax.py +1 -1
- diffusers/models/attention_processor.py +4304 -687
- diffusers/models/autoencoders/__init__.py +8 -0
- diffusers/models/autoencoders/autoencoder_asym_kl.py +15 -17
- diffusers/models/autoencoders/autoencoder_dc.py +620 -0
- diffusers/models/autoencoders/autoencoder_kl.py +110 -28
- diffusers/models/autoencoders/autoencoder_kl_allegro.py +1149 -0
- diffusers/models/autoencoders/autoencoder_kl_cogvideox.py +1482 -0
- diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +1176 -0
- diffusers/models/autoencoders/autoencoder_kl_ltx.py +1338 -0
- diffusers/models/autoencoders/autoencoder_kl_mochi.py +1166 -0
- diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +19 -24
- diffusers/models/autoencoders/autoencoder_oobleck.py +464 -0
- diffusers/models/autoencoders/autoencoder_tiny.py +21 -18
- diffusers/models/autoencoders/consistency_decoder_vae.py +45 -20
- diffusers/models/autoencoders/vae.py +41 -29
- diffusers/models/autoencoders/vq_model.py +182 -0
- diffusers/models/controlnet.py +47 -800
- diffusers/models/controlnet_flux.py +70 -0
- diffusers/models/controlnet_sd3.py +68 -0
- diffusers/models/controlnet_sparsectrl.py +116 -0
- diffusers/models/controlnets/__init__.py +23 -0
- diffusers/models/controlnets/controlnet.py +872 -0
- diffusers/models/{controlnet_flax.py → controlnets/controlnet_flax.py} +9 -9
- diffusers/models/controlnets/controlnet_flux.py +536 -0
- diffusers/models/controlnets/controlnet_hunyuan.py +401 -0
- diffusers/models/controlnets/controlnet_sd3.py +489 -0
- diffusers/models/controlnets/controlnet_sparsectrl.py +788 -0
- diffusers/models/controlnets/controlnet_union.py +832 -0
- diffusers/models/controlnets/controlnet_xs.py +1946 -0
- diffusers/models/controlnets/multicontrolnet.py +183 -0
- diffusers/models/downsampling.py +85 -18
- diffusers/models/embeddings.py +1856 -158
- diffusers/models/embeddings_flax.py +23 -9
- diffusers/models/model_loading_utils.py +480 -0
- diffusers/models/modeling_flax_pytorch_utils.py +2 -1
- diffusers/models/modeling_flax_utils.py +2 -7
- diffusers/models/modeling_outputs.py +14 -0
- diffusers/models/modeling_pytorch_flax_utils.py +1 -1
- diffusers/models/modeling_utils.py +611 -146
- diffusers/models/normalization.py +361 -20
- diffusers/models/resnet.py +18 -23
- diffusers/models/transformers/__init__.py +16 -0
- diffusers/models/transformers/auraflow_transformer_2d.py +544 -0
- diffusers/models/transformers/cogvideox_transformer_3d.py +542 -0
- diffusers/models/transformers/dit_transformer_2d.py +240 -0
- diffusers/models/transformers/dual_transformer_2d.py +9 -8
- diffusers/models/transformers/hunyuan_transformer_2d.py +578 -0
- diffusers/models/transformers/latte_transformer_3d.py +327 -0
- diffusers/models/transformers/lumina_nextdit2d.py +340 -0
- diffusers/models/transformers/pixart_transformer_2d.py +445 -0
- diffusers/models/transformers/prior_transformer.py +13 -13
- diffusers/models/transformers/sana_transformer.py +488 -0
- diffusers/models/transformers/stable_audio_transformer.py +458 -0
- diffusers/models/transformers/t5_film_transformer.py +17 -19
- diffusers/models/transformers/transformer_2d.py +297 -187
- diffusers/models/transformers/transformer_allegro.py +422 -0
- diffusers/models/transformers/transformer_cogview3plus.py +386 -0
- diffusers/models/transformers/transformer_flux.py +593 -0
- diffusers/models/transformers/transformer_hunyuan_video.py +791 -0
- diffusers/models/transformers/transformer_ltx.py +469 -0
- diffusers/models/transformers/transformer_mochi.py +499 -0
- diffusers/models/transformers/transformer_sd3.py +461 -0
- diffusers/models/transformers/transformer_temporal.py +21 -19
- diffusers/models/unets/unet_1d.py +8 -8
- diffusers/models/unets/unet_1d_blocks.py +31 -31
- diffusers/models/unets/unet_2d.py +17 -10
- diffusers/models/unets/unet_2d_blocks.py +225 -149
- diffusers/models/unets/unet_2d_condition.py +50 -53
- diffusers/models/unets/unet_2d_condition_flax.py +6 -5
- diffusers/models/unets/unet_3d_blocks.py +192 -1057
- diffusers/models/unets/unet_3d_condition.py +22 -27
- diffusers/models/unets/unet_i2vgen_xl.py +22 -18
- diffusers/models/unets/unet_kandinsky3.py +2 -2
- diffusers/models/unets/unet_motion_model.py +1413 -89
- diffusers/models/unets/unet_spatio_temporal_condition.py +40 -16
- diffusers/models/unets/unet_stable_cascade.py +19 -18
- diffusers/models/unets/uvit_2d.py +2 -2
- diffusers/models/upsampling.py +95 -26
- diffusers/models/vq_model.py +12 -164
- diffusers/optimization.py +1 -1
- diffusers/pipelines/__init__.py +202 -3
- diffusers/pipelines/allegro/__init__.py +48 -0
- diffusers/pipelines/allegro/pipeline_allegro.py +938 -0
- diffusers/pipelines/allegro/pipeline_output.py +23 -0
- diffusers/pipelines/amused/pipeline_amused.py +12 -12
- diffusers/pipelines/amused/pipeline_amused_img2img.py +14 -12
- diffusers/pipelines/amused/pipeline_amused_inpaint.py +13 -11
- diffusers/pipelines/animatediff/__init__.py +8 -0
- diffusers/pipelines/animatediff/pipeline_animatediff.py +122 -109
- diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +1106 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +1288 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +1010 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +236 -180
- diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +1341 -0
- diffusers/pipelines/animatediff/pipeline_output.py +3 -2
- diffusers/pipelines/audioldm/pipeline_audioldm.py +14 -14
- diffusers/pipelines/audioldm2/modeling_audioldm2.py +58 -39
- diffusers/pipelines/audioldm2/pipeline_audioldm2.py +121 -36
- diffusers/pipelines/aura_flow/__init__.py +48 -0
- diffusers/pipelines/aura_flow/pipeline_aura_flow.py +584 -0
- diffusers/pipelines/auto_pipeline.py +196 -28
- diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
- diffusers/pipelines/blip_diffusion/modeling_blip2.py +6 -6
- diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +1 -1
- diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +2 -2
- diffusers/pipelines/cogvideo/__init__.py +54 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox.py +772 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py +825 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py +885 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py +851 -0
- diffusers/pipelines/cogvideo/pipeline_output.py +20 -0
- diffusers/pipelines/cogview3/__init__.py +47 -0
- diffusers/pipelines/cogview3/pipeline_cogview3plus.py +674 -0
- diffusers/pipelines/cogview3/pipeline_output.py +21 -0
- diffusers/pipelines/consistency_models/pipeline_consistency_models.py +6 -6
- diffusers/pipelines/controlnet/__init__.py +86 -80
- diffusers/pipelines/controlnet/multicontrolnet.py +7 -182
- diffusers/pipelines/controlnet/pipeline_controlnet.py +134 -87
- diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +2 -2
- diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +93 -77
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +88 -197
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +136 -90
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +176 -80
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +125 -89
- diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +1790 -0
- diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +1501 -0
- diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +1627 -0
- diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +2 -2
- diffusers/pipelines/controlnet_hunyuandit/__init__.py +48 -0
- diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +1060 -0
- diffusers/pipelines/controlnet_sd3/__init__.py +57 -0
- diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +1133 -0
- diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +1153 -0
- diffusers/pipelines/controlnet_xs/__init__.py +68 -0
- diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py +916 -0
- diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +1111 -0
- diffusers/pipelines/ddpm/pipeline_ddpm.py +2 -2
- diffusers/pipelines/deepfloyd_if/pipeline_if.py +16 -30
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +20 -35
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +23 -41
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +22 -38
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +25 -41
- diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +19 -34
- diffusers/pipelines/deepfloyd_if/pipeline_output.py +6 -5
- diffusers/pipelines/deepfloyd_if/watermark.py +1 -1
- diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py +11 -11
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +70 -30
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +48 -25
- diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +2 -2
- diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +7 -7
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +21 -20
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +27 -29
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +33 -27
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +33 -23
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +36 -30
- diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +102 -69
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py +13 -13
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +10 -5
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +11 -6
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +10 -5
- diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +5 -5
- diffusers/pipelines/dit/pipeline_dit.py +7 -4
- diffusers/pipelines/flux/__init__.py +69 -0
- diffusers/pipelines/flux/modeling_flux.py +47 -0
- diffusers/pipelines/flux/pipeline_flux.py +957 -0
- diffusers/pipelines/flux/pipeline_flux_control.py +889 -0
- diffusers/pipelines/flux/pipeline_flux_control_img2img.py +945 -0
- diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +1141 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet.py +1006 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +998 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +1204 -0
- diffusers/pipelines/flux/pipeline_flux_fill.py +969 -0
- diffusers/pipelines/flux/pipeline_flux_img2img.py +856 -0
- diffusers/pipelines/flux/pipeline_flux_inpaint.py +1022 -0
- diffusers/pipelines/flux/pipeline_flux_prior_redux.py +492 -0
- diffusers/pipelines/flux/pipeline_output.py +37 -0
- diffusers/pipelines/free_init_utils.py +41 -38
- diffusers/pipelines/free_noise_utils.py +596 -0
- diffusers/pipelines/hunyuan_video/__init__.py +48 -0
- diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +687 -0
- diffusers/pipelines/hunyuan_video/pipeline_output.py +20 -0
- diffusers/pipelines/hunyuandit/__init__.py +48 -0
- diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +916 -0
- diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +33 -48
- diffusers/pipelines/kandinsky/pipeline_kandinsky.py +8 -8
- diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +32 -29
- diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +11 -11
- diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +12 -12
- diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +6 -6
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +34 -31
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +6 -6
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +8 -8
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +7 -7
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +6 -6
- diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py +3 -3
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +22 -35
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +26 -37
- diffusers/pipelines/kolors/__init__.py +54 -0
- diffusers/pipelines/kolors/pipeline_kolors.py +1070 -0
- diffusers/pipelines/kolors/pipeline_kolors_img2img.py +1250 -0
- diffusers/pipelines/kolors/pipeline_output.py +21 -0
- diffusers/pipelines/kolors/text_encoder.py +889 -0
- diffusers/pipelines/kolors/tokenizer.py +338 -0
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +82 -62
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +77 -60
- diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +12 -12
- diffusers/pipelines/latte/__init__.py +48 -0
- diffusers/pipelines/latte/pipeline_latte.py +881 -0
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +80 -74
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +85 -76
- diffusers/pipelines/ledits_pp/pipeline_output.py +2 -2
- diffusers/pipelines/ltx/__init__.py +50 -0
- diffusers/pipelines/ltx/pipeline_ltx.py +789 -0
- diffusers/pipelines/ltx/pipeline_ltx_image2video.py +885 -0
- diffusers/pipelines/ltx/pipeline_output.py +20 -0
- diffusers/pipelines/lumina/__init__.py +48 -0
- diffusers/pipelines/lumina/pipeline_lumina.py +890 -0
- diffusers/pipelines/marigold/__init__.py +50 -0
- diffusers/pipelines/marigold/marigold_image_processing.py +576 -0
- diffusers/pipelines/marigold/pipeline_marigold_depth.py +813 -0
- diffusers/pipelines/marigold/pipeline_marigold_normals.py +690 -0
- diffusers/pipelines/mochi/__init__.py +48 -0
- diffusers/pipelines/mochi/pipeline_mochi.py +748 -0
- diffusers/pipelines/mochi/pipeline_output.py +20 -0
- diffusers/pipelines/musicldm/pipeline_musicldm.py +14 -14
- diffusers/pipelines/pag/__init__.py +80 -0
- diffusers/pipelines/pag/pag_utils.py +243 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +1328 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +1543 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +1610 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +1683 -0
- diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +969 -0
- diffusers/pipelines/pag/pipeline_pag_kolors.py +1136 -0
- diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py +865 -0
- diffusers/pipelines/pag/pipeline_pag_sana.py +886 -0
- diffusers/pipelines/pag/pipeline_pag_sd.py +1062 -0
- diffusers/pipelines/pag/pipeline_pag_sd_3.py +994 -0
- diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +1058 -0
- diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +866 -0
- diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +1094 -0
- diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +1356 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl.py +1345 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +1544 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +1776 -0
- diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +17 -12
- diffusers/pipelines/pia/pipeline_pia.py +74 -164
- diffusers/pipelines/pipeline_flax_utils.py +5 -10
- diffusers/pipelines/pipeline_loading_utils.py +515 -53
- diffusers/pipelines/pipeline_utils.py +411 -222
- diffusers/pipelines/pixart_alpha/__init__.py +8 -1
- diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +76 -93
- diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py +873 -0
- diffusers/pipelines/sana/__init__.py +47 -0
- diffusers/pipelines/sana/pipeline_output.py +21 -0
- diffusers/pipelines/sana/pipeline_sana.py +884 -0
- diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +27 -23
- diffusers/pipelines/shap_e/pipeline_shap_e.py +3 -3
- diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +14 -14
- diffusers/pipelines/shap_e/renderer.py +1 -1
- diffusers/pipelines/stable_audio/__init__.py +50 -0
- diffusers/pipelines/stable_audio/modeling_stable_audio.py +158 -0
- diffusers/pipelines/stable_audio/pipeline_stable_audio.py +756 -0
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +71 -25
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +23 -19
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +35 -34
- diffusers/pipelines/stable_diffusion/__init__.py +0 -1
- diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +20 -11
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +2 -2
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +6 -6
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +145 -79
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +43 -28
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +13 -8
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +100 -68
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +109 -201
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +131 -32
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +247 -87
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +30 -29
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +35 -27
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +49 -42
- diffusers/pipelines/stable_diffusion/safety_checker.py +2 -1
- diffusers/pipelines/stable_diffusion_3/__init__.py +54 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_output.py +21 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +1140 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +1036 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +1250 -0
- diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +29 -20
- diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +59 -58
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +31 -25
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +38 -22
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +30 -24
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +24 -23
- diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +107 -67
- diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +316 -69
- diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +10 -5
- diffusers/pipelines/stable_diffusion_safe/safety_checker.py +1 -1
- diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +98 -30
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +121 -83
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +161 -105
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +142 -218
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +45 -29
- diffusers/pipelines/stable_diffusion_xl/watermark.py +9 -3
- diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +110 -57
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +69 -39
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +105 -74
- diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +3 -2
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +29 -49
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +32 -93
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +37 -25
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +54 -40
- diffusers/pipelines/unclip/pipeline_unclip.py +6 -6
- diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +6 -6
- diffusers/pipelines/unidiffuser/modeling_text_decoder.py +1 -1
- diffusers/pipelines/unidiffuser/modeling_uvit.py +12 -12
- diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +29 -28
- diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +5 -5
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +5 -10
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +6 -8
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +4 -4
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +12 -12
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +15 -14
- diffusers/{models/dual_transformer_2d.py → quantizers/__init__.py} +2 -6
- diffusers/quantizers/auto.py +139 -0
- diffusers/quantizers/base.py +233 -0
- diffusers/quantizers/bitsandbytes/__init__.py +2 -0
- diffusers/quantizers/bitsandbytes/bnb_quantizer.py +561 -0
- diffusers/quantizers/bitsandbytes/utils.py +306 -0
- diffusers/quantizers/gguf/__init__.py +1 -0
- diffusers/quantizers/gguf/gguf_quantizer.py +159 -0
- diffusers/quantizers/gguf/utils.py +456 -0
- diffusers/quantizers/quantization_config.py +669 -0
- diffusers/quantizers/torchao/__init__.py +15 -0
- diffusers/quantizers/torchao/torchao_quantizer.py +292 -0
- diffusers/schedulers/__init__.py +12 -2
- diffusers/schedulers/deprecated/__init__.py +1 -1
- diffusers/schedulers/deprecated/scheduling_karras_ve.py +25 -25
- diffusers/schedulers/scheduling_amused.py +5 -5
- diffusers/schedulers/scheduling_consistency_decoder.py +11 -11
- diffusers/schedulers/scheduling_consistency_models.py +23 -25
- diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py +572 -0
- diffusers/schedulers/scheduling_ddim.py +27 -26
- diffusers/schedulers/scheduling_ddim_cogvideox.py +452 -0
- diffusers/schedulers/scheduling_ddim_flax.py +2 -1
- diffusers/schedulers/scheduling_ddim_inverse.py +16 -16
- diffusers/schedulers/scheduling_ddim_parallel.py +32 -31
- diffusers/schedulers/scheduling_ddpm.py +27 -30
- diffusers/schedulers/scheduling_ddpm_flax.py +7 -3
- diffusers/schedulers/scheduling_ddpm_parallel.py +33 -36
- diffusers/schedulers/scheduling_ddpm_wuerstchen.py +14 -14
- diffusers/schedulers/scheduling_deis_multistep.py +150 -50
- diffusers/schedulers/scheduling_dpm_cogvideox.py +489 -0
- diffusers/schedulers/scheduling_dpmsolver_multistep.py +221 -84
- diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +2 -2
- diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +158 -52
- diffusers/schedulers/scheduling_dpmsolver_sde.py +153 -34
- diffusers/schedulers/scheduling_dpmsolver_singlestep.py +275 -86
- diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +81 -57
- diffusers/schedulers/scheduling_edm_euler.py +62 -39
- diffusers/schedulers/scheduling_euler_ancestral_discrete.py +30 -29
- diffusers/schedulers/scheduling_euler_discrete.py +255 -74
- diffusers/schedulers/scheduling_flow_match_euler_discrete.py +458 -0
- diffusers/schedulers/scheduling_flow_match_heun_discrete.py +320 -0
- diffusers/schedulers/scheduling_heun_discrete.py +174 -46
- diffusers/schedulers/scheduling_ipndm.py +9 -9
- diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +138 -29
- diffusers/schedulers/scheduling_k_dpm_2_discrete.py +132 -26
- diffusers/schedulers/scheduling_karras_ve_flax.py +6 -6
- diffusers/schedulers/scheduling_lcm.py +23 -29
- diffusers/schedulers/scheduling_lms_discrete.py +105 -28
- diffusers/schedulers/scheduling_pndm.py +20 -20
- diffusers/schedulers/scheduling_repaint.py +21 -21
- diffusers/schedulers/scheduling_sasolver.py +157 -60
- diffusers/schedulers/scheduling_sde_ve.py +19 -19
- diffusers/schedulers/scheduling_tcd.py +41 -36
- diffusers/schedulers/scheduling_unclip.py +19 -16
- diffusers/schedulers/scheduling_unipc_multistep.py +243 -47
- diffusers/schedulers/scheduling_utils.py +12 -5
- diffusers/schedulers/scheduling_utils_flax.py +1 -3
- diffusers/schedulers/scheduling_vq_diffusion.py +10 -10
- diffusers/training_utils.py +214 -30
- diffusers/utils/__init__.py +17 -1
- diffusers/utils/constants.py +3 -0
- diffusers/utils/doc_utils.py +1 -0
- diffusers/utils/dummy_pt_objects.py +592 -7
- diffusers/utils/dummy_torch_and_torchsde_objects.py +15 -0
- diffusers/utils/dummy_torch_and_transformers_and_sentencepiece_objects.py +47 -0
- diffusers/utils/dummy_torch_and_transformers_objects.py +1001 -71
- diffusers/utils/dynamic_modules_utils.py +34 -29
- diffusers/utils/export_utils.py +50 -6
- diffusers/utils/hub_utils.py +131 -17
- diffusers/utils/import_utils.py +210 -8
- diffusers/utils/loading_utils.py +118 -5
- diffusers/utils/logging.py +4 -2
- diffusers/utils/peft_utils.py +37 -7
- diffusers/utils/state_dict_utils.py +13 -2
- diffusers/utils/testing_utils.py +193 -11
- diffusers/utils/torch_utils.py +4 -0
- diffusers/video_processor.py +113 -0
- {diffusers-0.27.0.dist-info → diffusers-0.32.2.dist-info}/METADATA +82 -91
- diffusers-0.32.2.dist-info/RECORD +550 -0
- {diffusers-0.27.0.dist-info → diffusers-0.32.2.dist-info}/WHEEL +1 -1
- diffusers/loaders/autoencoder.py +0 -146
- diffusers/loaders/controlnet.py +0 -136
- diffusers/loaders/lora.py +0 -1349
- diffusers/models/prior_transformer.py +0 -12
- diffusers/models/t5_film_transformer.py +0 -70
- diffusers/models/transformer_2d.py +0 -25
- diffusers/models/transformer_temporal.py +0 -34
- diffusers/models/unet_1d.py +0 -26
- diffusers/models/unet_1d_blocks.py +0 -203
- diffusers/models/unet_2d.py +0 -27
- diffusers/models/unet_2d_blocks.py +0 -375
- diffusers/models/unet_2d_condition.py +0 -25
- diffusers-0.27.0.dist-info/RECORD +0 -399
- {diffusers-0.27.0.dist-info → diffusers-0.32.2.dist-info}/LICENSE +0 -0
- {diffusers-0.27.0.dist-info → diffusers-0.32.2.dist-info}/entry_points.txt +0 -0
- {diffusers-0.27.0.dist-info → diffusers-0.32.2.dist-info}/top_level.txt +0 -0
@@ -21,11 +21,15 @@ import numpy as np
|
|
21
21
|
import torch
|
22
22
|
|
23
23
|
from ..configuration_utils import ConfigMixin, register_to_config
|
24
|
-
from ..utils import deprecate
|
24
|
+
from ..utils import deprecate, is_scipy_available
|
25
25
|
from ..utils.torch_utils import randn_tensor
|
26
26
|
from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput
|
27
27
|
|
28
28
|
|
29
|
+
if is_scipy_available():
|
30
|
+
import scipy.stats
|
31
|
+
|
32
|
+
|
29
33
|
# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar
|
30
34
|
def betas_for_alpha_bar(
|
31
35
|
num_diffusion_timesteps,
|
@@ -61,7 +65,7 @@ def betas_for_alpha_bar(
|
|
61
65
|
return math.exp(t * -12.0)
|
62
66
|
|
63
67
|
else:
|
64
|
-
raise ValueError(f"Unsupported
|
68
|
+
raise ValueError(f"Unsupported alpha_transform_type: {alpha_transform_type}")
|
65
69
|
|
66
70
|
betas = []
|
67
71
|
for i in range(num_diffusion_timesteps):
|
@@ -124,6 +128,11 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
124
128
|
use_karras_sigmas (`bool`, *optional*, defaults to `False`):
|
125
129
|
Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`,
|
126
130
|
the sigmas are determined according to a sequence of noise levels {σi}.
|
131
|
+
use_exponential_sigmas (`bool`, *optional*, defaults to `False`):
|
132
|
+
Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process.
|
133
|
+
use_beta_sigmas (`bool`, *optional*, defaults to `False`):
|
134
|
+
Whether to use beta sigmas for step sizes in the noise schedule during the sampling process. Refer to [Beta
|
135
|
+
Sampling is All You Need](https://huggingface.co/papers/2407.12173) for more information.
|
127
136
|
lambda_min_clipped (`float`, defaults to `-inf`):
|
128
137
|
Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the
|
129
138
|
cosine (`squaredcos_cap_v2`) noise schedule.
|
@@ -158,11 +167,21 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
158
167
|
lower_order_final: bool = True,
|
159
168
|
euler_at_final: bool = False,
|
160
169
|
use_karras_sigmas: Optional[bool] = False,
|
170
|
+
use_exponential_sigmas: Optional[bool] = False,
|
171
|
+
use_beta_sigmas: Optional[bool] = False,
|
172
|
+
use_flow_sigmas: Optional[bool] = False,
|
173
|
+
flow_shift: Optional[float] = 1.0,
|
161
174
|
lambda_min_clipped: float = -float("inf"),
|
162
175
|
variance_type: Optional[str] = None,
|
163
176
|
timestep_spacing: str = "linspace",
|
164
177
|
steps_offset: int = 0,
|
165
178
|
):
|
179
|
+
if self.config.use_beta_sigmas and not is_scipy_available():
|
180
|
+
raise ImportError("Make sure to install scipy if you want to use beta sigmas.")
|
181
|
+
if sum([self.config.use_beta_sigmas, self.config.use_exponential_sigmas, self.config.use_karras_sigmas]) > 1:
|
182
|
+
raise ValueError(
|
183
|
+
"Only one of `config.use_beta_sigmas`, `config.use_exponential_sigmas`, `config.use_karras_sigmas` can be used."
|
184
|
+
)
|
166
185
|
if algorithm_type in ["dpmsolver", "sde-dpmsolver"]:
|
167
186
|
deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead"
|
168
187
|
deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0", deprecation_message)
|
@@ -178,7 +197,7 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
178
197
|
# Glide cosine schedule
|
179
198
|
self.betas = betas_for_alpha_bar(num_train_timesteps)
|
180
199
|
else:
|
181
|
-
raise NotImplementedError(f"{beta_schedule}
|
200
|
+
raise NotImplementedError(f"{beta_schedule} is not implemented for {self.__class__}")
|
182
201
|
|
183
202
|
self.alphas = 1.0 - self.betas
|
184
203
|
self.alphas_cumprod = torch.cumprod(self.alphas, dim=0)
|
@@ -196,13 +215,13 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
196
215
|
if algorithm_type == "deis":
|
197
216
|
self.register_to_config(algorithm_type="dpmsolver++")
|
198
217
|
else:
|
199
|
-
raise NotImplementedError(f"{algorithm_type}
|
218
|
+
raise NotImplementedError(f"{algorithm_type} is not implemented for {self.__class__}")
|
200
219
|
|
201
220
|
if solver_type not in ["midpoint", "heun"]:
|
202
221
|
if solver_type in ["logrho", "bh1", "bh2"]:
|
203
222
|
self.register_to_config(solver_type="midpoint")
|
204
223
|
else:
|
205
|
-
raise NotImplementedError(f"{solver_type}
|
224
|
+
raise NotImplementedError(f"{solver_type} is not implemented for {self.__class__}")
|
206
225
|
|
207
226
|
# setable values
|
208
227
|
self.num_inference_steps = None
|
@@ -213,11 +232,13 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
213
232
|
self._step_index = None
|
214
233
|
self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication
|
215
234
|
self.use_karras_sigmas = use_karras_sigmas
|
235
|
+
self.use_exponential_sigmas = use_exponential_sigmas
|
236
|
+
self.use_beta_sigmas = use_beta_sigmas
|
216
237
|
|
217
238
|
@property
|
218
239
|
def step_index(self):
|
219
240
|
"""
|
220
|
-
The index counter for current timestep. It will
|
241
|
+
The index counter for current timestep. It will increase 1 after each scheduler step.
|
221
242
|
"""
|
222
243
|
return self._step_index
|
223
244
|
|
@@ -233,7 +254,7 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
233
254
|
"""
|
234
255
|
# Clipping the minimum of all lambda(t) for numerical stability.
|
235
256
|
# This is critical for cosine (squaredcos_cap_v2) noise schedule.
|
236
|
-
clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.lambda_min_clipped).item()
|
257
|
+
clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.config.lambda_min_clipped).item()
|
237
258
|
self.noisiest_timestep = self.config.num_train_timesteps - 1 - clipped_idx
|
238
259
|
|
239
260
|
# "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891
|
@@ -267,6 +288,20 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
267
288
|
timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round()
|
268
289
|
timesteps = timesteps.copy().astype(np.int64)
|
269
290
|
sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32)
|
291
|
+
elif self.config.use_exponential_sigmas:
|
292
|
+
sigmas = self._convert_to_exponential(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
|
293
|
+
timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas])
|
294
|
+
sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32)
|
295
|
+
elif self.config.use_beta_sigmas:
|
296
|
+
sigmas = self._convert_to_beta(in_sigmas=sigmas, num_inference_steps=num_inference_steps)
|
297
|
+
timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas])
|
298
|
+
sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32)
|
299
|
+
elif self.config.use_flow_sigmas:
|
300
|
+
alphas = np.linspace(1, 1 / self.config.num_train_timesteps, num_inference_steps + 1)
|
301
|
+
sigmas = 1.0 - alphas
|
302
|
+
sigmas = np.flip(self.config.flow_shift * sigmas / (1 + (self.config.flow_shift - 1) * sigmas))[:-1].copy()
|
303
|
+
timesteps = (sigmas * self.config.num_train_timesteps).copy()
|
304
|
+
sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32)
|
270
305
|
else:
|
271
306
|
sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas)
|
272
307
|
sigma_max = (
|
@@ -295,7 +330,7 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
295
330
|
self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication
|
296
331
|
|
297
332
|
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample
|
298
|
-
def _threshold_sample(self, sample: torch.
|
333
|
+
def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:
|
299
334
|
"""
|
300
335
|
"Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the
|
301
336
|
prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by
|
@@ -354,13 +389,17 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
354
389
|
|
355
390
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t
|
356
391
|
def _sigma_to_alpha_sigma_t(self, sigma):
|
357
|
-
|
358
|
-
|
392
|
+
if self.config.use_flow_sigmas:
|
393
|
+
alpha_t = 1 - sigma
|
394
|
+
sigma_t = sigma
|
395
|
+
else:
|
396
|
+
alpha_t = 1 / ((sigma**2 + 1) ** 0.5)
|
397
|
+
sigma_t = sigma * alpha_t
|
359
398
|
|
360
399
|
return alpha_t, sigma_t
|
361
400
|
|
362
401
|
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras
|
363
|
-
def _convert_to_karras(self, in_sigmas: torch.
|
402
|
+
def _convert_to_karras(self, in_sigmas: torch.Tensor, num_inference_steps) -> torch.Tensor:
|
364
403
|
"""Constructs the noise schedule of Karras et al. (2022)."""
|
365
404
|
|
366
405
|
# Hack to make sure that other schedulers which copy this function don't break
|
@@ -385,14 +424,68 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
385
424
|
sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho
|
386
425
|
return sigmas
|
387
426
|
|
427
|
+
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_exponential
|
428
|
+
def _convert_to_exponential(self, in_sigmas: torch.Tensor, num_inference_steps: int) -> torch.Tensor:
|
429
|
+
"""Constructs an exponential noise schedule."""
|
430
|
+
|
431
|
+
# Hack to make sure that other schedulers which copy this function don't break
|
432
|
+
# TODO: Add this logic to the other schedulers
|
433
|
+
if hasattr(self.config, "sigma_min"):
|
434
|
+
sigma_min = self.config.sigma_min
|
435
|
+
else:
|
436
|
+
sigma_min = None
|
437
|
+
|
438
|
+
if hasattr(self.config, "sigma_max"):
|
439
|
+
sigma_max = self.config.sigma_max
|
440
|
+
else:
|
441
|
+
sigma_max = None
|
442
|
+
|
443
|
+
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
444
|
+
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
445
|
+
|
446
|
+
sigmas = np.exp(np.linspace(math.log(sigma_max), math.log(sigma_min), num_inference_steps))
|
447
|
+
return sigmas
|
448
|
+
|
449
|
+
# Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_beta
|
450
|
+
def _convert_to_beta(
|
451
|
+
self, in_sigmas: torch.Tensor, num_inference_steps: int, alpha: float = 0.6, beta: float = 0.6
|
452
|
+
) -> torch.Tensor:
|
453
|
+
"""From "Beta Sampling is All You Need" [arXiv:2407.12173] (Lee et. al, 2024)"""
|
454
|
+
|
455
|
+
# Hack to make sure that other schedulers which copy this function don't break
|
456
|
+
# TODO: Add this logic to the other schedulers
|
457
|
+
if hasattr(self.config, "sigma_min"):
|
458
|
+
sigma_min = self.config.sigma_min
|
459
|
+
else:
|
460
|
+
sigma_min = None
|
461
|
+
|
462
|
+
if hasattr(self.config, "sigma_max"):
|
463
|
+
sigma_max = self.config.sigma_max
|
464
|
+
else:
|
465
|
+
sigma_max = None
|
466
|
+
|
467
|
+
sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item()
|
468
|
+
sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item()
|
469
|
+
|
470
|
+
sigmas = np.array(
|
471
|
+
[
|
472
|
+
sigma_min + (ppf * (sigma_max - sigma_min))
|
473
|
+
for ppf in [
|
474
|
+
scipy.stats.beta.ppf(timestep, alpha, beta)
|
475
|
+
for timestep in 1 - np.linspace(0, 1, num_inference_steps)
|
476
|
+
]
|
477
|
+
]
|
478
|
+
)
|
479
|
+
return sigmas
|
480
|
+
|
388
481
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.convert_model_output
|
389
482
|
def convert_model_output(
|
390
483
|
self,
|
391
|
-
model_output: torch.
|
484
|
+
model_output: torch.Tensor,
|
392
485
|
*args,
|
393
|
-
sample: torch.
|
486
|
+
sample: torch.Tensor = None,
|
394
487
|
**kwargs,
|
395
|
-
) -> torch.
|
488
|
+
) -> torch.Tensor:
|
396
489
|
"""
|
397
490
|
Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is
|
398
491
|
designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an
|
@@ -406,13 +499,13 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
406
499
|
</Tip>
|
407
500
|
|
408
501
|
Args:
|
409
|
-
model_output (`torch.
|
502
|
+
model_output (`torch.Tensor`):
|
410
503
|
The direct output from the learned diffusion model.
|
411
|
-
sample (`torch.
|
504
|
+
sample (`torch.Tensor`):
|
412
505
|
A current instance of a sample created by the diffusion process.
|
413
506
|
|
414
507
|
Returns:
|
415
|
-
`torch.
|
508
|
+
`torch.Tensor`:
|
416
509
|
The converted model output.
|
417
510
|
"""
|
418
511
|
timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None)
|
@@ -443,10 +536,13 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
443
536
|
sigma = self.sigmas[self.step_index]
|
444
537
|
alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)
|
445
538
|
x0_pred = alpha_t * sample - sigma_t * model_output
|
539
|
+
elif self.config.prediction_type == "flow_prediction":
|
540
|
+
sigma_t = self.sigmas[self.step_index]
|
541
|
+
x0_pred = sample - sigma_t * model_output
|
446
542
|
else:
|
447
543
|
raise ValueError(
|
448
|
-
f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`,
|
449
|
-
"
|
544
|
+
f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, "
|
545
|
+
"`v_prediction`, or `flow_prediction` for the DPMSolverMultistepScheduler."
|
450
546
|
)
|
451
547
|
|
452
548
|
if self.config.thresholding:
|
@@ -488,23 +584,23 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
488
584
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.dpm_solver_first_order_update
|
489
585
|
def dpm_solver_first_order_update(
|
490
586
|
self,
|
491
|
-
model_output: torch.
|
587
|
+
model_output: torch.Tensor,
|
492
588
|
*args,
|
493
|
-
sample: torch.
|
494
|
-
noise: Optional[torch.
|
589
|
+
sample: torch.Tensor = None,
|
590
|
+
noise: Optional[torch.Tensor] = None,
|
495
591
|
**kwargs,
|
496
|
-
) -> torch.
|
592
|
+
) -> torch.Tensor:
|
497
593
|
"""
|
498
594
|
One step for the first-order DPMSolver (equivalent to DDIM).
|
499
595
|
|
500
596
|
Args:
|
501
|
-
model_output (`torch.
|
597
|
+
model_output (`torch.Tensor`):
|
502
598
|
The direct output from the learned diffusion model.
|
503
|
-
sample (`torch.
|
599
|
+
sample (`torch.Tensor`):
|
504
600
|
A current instance of a sample created by the diffusion process.
|
505
601
|
|
506
602
|
Returns:
|
507
|
-
`torch.
|
603
|
+
`torch.Tensor`:
|
508
604
|
The sample tensor at the previous timestep.
|
509
605
|
"""
|
510
606
|
timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None)
|
@@ -558,23 +654,23 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
558
654
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_second_order_update
|
559
655
|
def multistep_dpm_solver_second_order_update(
|
560
656
|
self,
|
561
|
-
model_output_list: List[torch.
|
657
|
+
model_output_list: List[torch.Tensor],
|
562
658
|
*args,
|
563
|
-
sample: torch.
|
564
|
-
noise: Optional[torch.
|
659
|
+
sample: torch.Tensor = None,
|
660
|
+
noise: Optional[torch.Tensor] = None,
|
565
661
|
**kwargs,
|
566
|
-
) -> torch.
|
662
|
+
) -> torch.Tensor:
|
567
663
|
"""
|
568
664
|
One step for the second-order multistep DPMSolver.
|
569
665
|
|
570
666
|
Args:
|
571
|
-
model_output_list (`List[torch.
|
667
|
+
model_output_list (`List[torch.Tensor]`):
|
572
668
|
The direct outputs from learned diffusion model at current and latter timesteps.
|
573
|
-
sample (`torch.
|
669
|
+
sample (`torch.Tensor`):
|
574
670
|
A current instance of a sample created by the diffusion process.
|
575
671
|
|
576
672
|
Returns:
|
577
|
-
`torch.
|
673
|
+
`torch.Tensor`:
|
578
674
|
The sample tensor at the previous timestep.
|
579
675
|
"""
|
580
676
|
timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None)
|
@@ -682,22 +778,23 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
682
778
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_third_order_update
|
683
779
|
def multistep_dpm_solver_third_order_update(
|
684
780
|
self,
|
685
|
-
model_output_list: List[torch.
|
781
|
+
model_output_list: List[torch.Tensor],
|
686
782
|
*args,
|
687
|
-
sample: torch.
|
783
|
+
sample: torch.Tensor = None,
|
784
|
+
noise: Optional[torch.Tensor] = None,
|
688
785
|
**kwargs,
|
689
|
-
) -> torch.
|
786
|
+
) -> torch.Tensor:
|
690
787
|
"""
|
691
788
|
One step for the third-order multistep DPMSolver.
|
692
789
|
|
693
790
|
Args:
|
694
|
-
model_output_list (`List[torch.
|
791
|
+
model_output_list (`List[torch.Tensor]`):
|
695
792
|
The direct outputs from learned diffusion model at current and latter timesteps.
|
696
|
-
sample (`torch.
|
793
|
+
sample (`torch.Tensor`):
|
697
794
|
A current instance of a sample created by diffusion process.
|
698
795
|
|
699
796
|
Returns:
|
700
|
-
`torch.
|
797
|
+
`torch.Tensor`:
|
701
798
|
The sample tensor at the previous timestep.
|
702
799
|
"""
|
703
800
|
|
@@ -763,6 +860,15 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
763
860
|
- (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1
|
764
861
|
- (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2
|
765
862
|
)
|
863
|
+
elif self.config.algorithm_type == "sde-dpmsolver++":
|
864
|
+
assert noise is not None
|
865
|
+
x_t = (
|
866
|
+
(sigma_t / sigma_s0 * torch.exp(-h)) * sample
|
867
|
+
+ (alpha_t * (1.0 - torch.exp(-2.0 * h))) * D0
|
868
|
+
+ (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / (-2.0 * h) + 1.0)) * D1
|
869
|
+
+ (alpha_t * ((1.0 - torch.exp(-2.0 * h) - 2.0 * h) / (2.0 * h) ** 2 - 0.5)) * D2
|
870
|
+
+ sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise
|
871
|
+
)
|
766
872
|
return x_t
|
767
873
|
|
768
874
|
def _init_step_index(self, timestep):
|
@@ -786,11 +892,11 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
786
892
|
|
787
893
|
def step(
|
788
894
|
self,
|
789
|
-
model_output: torch.
|
790
|
-
timestep: int,
|
791
|
-
sample: torch.
|
895
|
+
model_output: torch.Tensor,
|
896
|
+
timestep: Union[int, torch.Tensor],
|
897
|
+
sample: torch.Tensor,
|
792
898
|
generator=None,
|
793
|
-
variance_noise: Optional[torch.
|
899
|
+
variance_noise: Optional[torch.Tensor] = None,
|
794
900
|
return_dict: bool = True,
|
795
901
|
) -> Union[SchedulerOutput, Tuple]:
|
796
902
|
"""
|
@@ -798,15 +904,15 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
798
904
|
the multistep DPMSolver.
|
799
905
|
|
800
906
|
Args:
|
801
|
-
model_output (`torch.
|
907
|
+
model_output (`torch.Tensor`):
|
802
908
|
The direct output from learned diffusion model.
|
803
909
|
timestep (`int`):
|
804
910
|
The current discrete timestep in the diffusion chain.
|
805
|
-
sample (`torch.
|
911
|
+
sample (`torch.Tensor`):
|
806
912
|
A current instance of a sample created by the diffusion process.
|
807
913
|
generator (`torch.Generator`, *optional*):
|
808
914
|
A random number generator.
|
809
|
-
variance_noise (`torch.
|
915
|
+
variance_noise (`torch.Tensor`):
|
810
916
|
Alternative to generating noise with `generator` by directly providing the noise for the variance
|
811
917
|
itself. Useful for methods such as [`CycleDiffusion`].
|
812
918
|
return_dict (`bool`):
|
@@ -867,27 +973,27 @@ class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin):
|
|
867
973
|
return SchedulerOutput(prev_sample=prev_sample)
|
868
974
|
|
869
975
|
# Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input
|
870
|
-
def scale_model_input(self, sample: torch.
|
976
|
+
def scale_model_input(self, sample: torch.Tensor, *args, **kwargs) -> torch.Tensor:
|
871
977
|
"""
|
872
978
|
Ensures interchangeability with schedulers that need to scale the denoising model input depending on the
|
873
979
|
current timestep.
|
874
980
|
|
875
981
|
Args:
|
876
|
-
sample (`torch.
|
982
|
+
sample (`torch.Tensor`):
|
877
983
|
The input sample.
|
878
984
|
|
879
985
|
Returns:
|
880
|
-
`torch.
|
986
|
+
`torch.Tensor`:
|
881
987
|
A scaled input sample.
|
882
988
|
"""
|
883
989
|
return sample
|
884
990
|
|
885
991
|
def add_noise(
|
886
992
|
self,
|
887
|
-
original_samples: torch.
|
888
|
-
noise: torch.
|
993
|
+
original_samples: torch.Tensor,
|
994
|
+
noise: torch.Tensor,
|
889
995
|
timesteps: torch.IntTensor,
|
890
|
-
) -> torch.
|
996
|
+
) -> torch.Tensor:
|
891
997
|
# Make sure sigmas and timesteps have the same device and dtype as original_samples
|
892
998
|
sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype)
|
893
999
|
if original_samples.device.type == "mps" and torch.is_floating_point(timesteps):
|