diffusers 0.27.1__py3-none-any.whl → 0.32.2__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- diffusers/__init__.py +233 -6
- diffusers/callbacks.py +209 -0
- diffusers/commands/env.py +102 -6
- diffusers/configuration_utils.py +45 -16
- diffusers/dependency_versions_table.py +4 -3
- diffusers/image_processor.py +434 -110
- diffusers/loaders/__init__.py +42 -9
- diffusers/loaders/ip_adapter.py +626 -36
- diffusers/loaders/lora_base.py +900 -0
- diffusers/loaders/lora_conversion_utils.py +991 -125
- diffusers/loaders/lora_pipeline.py +3812 -0
- diffusers/loaders/peft.py +571 -7
- diffusers/loaders/single_file.py +405 -173
- diffusers/loaders/single_file_model.py +385 -0
- diffusers/loaders/single_file_utils.py +1783 -713
- diffusers/loaders/textual_inversion.py +41 -23
- diffusers/loaders/transformer_flux.py +181 -0
- diffusers/loaders/transformer_sd3.py +89 -0
- diffusers/loaders/unet.py +464 -540
- diffusers/loaders/unet_loader_utils.py +163 -0
- diffusers/models/__init__.py +76 -7
- diffusers/models/activations.py +65 -10
- diffusers/models/adapter.py +53 -53
- diffusers/models/attention.py +605 -18
- diffusers/models/attention_flax.py +1 -1
- diffusers/models/attention_processor.py +4304 -687
- diffusers/models/autoencoders/__init__.py +8 -0
- diffusers/models/autoencoders/autoencoder_asym_kl.py +15 -17
- diffusers/models/autoencoders/autoencoder_dc.py +620 -0
- diffusers/models/autoencoders/autoencoder_kl.py +110 -28
- diffusers/models/autoencoders/autoencoder_kl_allegro.py +1149 -0
- diffusers/models/autoencoders/autoencoder_kl_cogvideox.py +1482 -0
- diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +1176 -0
- diffusers/models/autoencoders/autoencoder_kl_ltx.py +1338 -0
- diffusers/models/autoencoders/autoencoder_kl_mochi.py +1166 -0
- diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +19 -24
- diffusers/models/autoencoders/autoencoder_oobleck.py +464 -0
- diffusers/models/autoencoders/autoencoder_tiny.py +21 -18
- diffusers/models/autoencoders/consistency_decoder_vae.py +45 -20
- diffusers/models/autoencoders/vae.py +41 -29
- diffusers/models/autoencoders/vq_model.py +182 -0
- diffusers/models/controlnet.py +47 -800
- diffusers/models/controlnet_flux.py +70 -0
- diffusers/models/controlnet_sd3.py +68 -0
- diffusers/models/controlnet_sparsectrl.py +116 -0
- diffusers/models/controlnets/__init__.py +23 -0
- diffusers/models/controlnets/controlnet.py +872 -0
- diffusers/models/{controlnet_flax.py → controlnets/controlnet_flax.py} +9 -9
- diffusers/models/controlnets/controlnet_flux.py +536 -0
- diffusers/models/controlnets/controlnet_hunyuan.py +401 -0
- diffusers/models/controlnets/controlnet_sd3.py +489 -0
- diffusers/models/controlnets/controlnet_sparsectrl.py +788 -0
- diffusers/models/controlnets/controlnet_union.py +832 -0
- diffusers/models/controlnets/controlnet_xs.py +1946 -0
- diffusers/models/controlnets/multicontrolnet.py +183 -0
- diffusers/models/downsampling.py +85 -18
- diffusers/models/embeddings.py +1856 -158
- diffusers/models/embeddings_flax.py +23 -9
- diffusers/models/model_loading_utils.py +480 -0
- diffusers/models/modeling_flax_pytorch_utils.py +2 -1
- diffusers/models/modeling_flax_utils.py +2 -7
- diffusers/models/modeling_outputs.py +14 -0
- diffusers/models/modeling_pytorch_flax_utils.py +1 -1
- diffusers/models/modeling_utils.py +611 -146
- diffusers/models/normalization.py +361 -20
- diffusers/models/resnet.py +18 -23
- diffusers/models/transformers/__init__.py +16 -0
- diffusers/models/transformers/auraflow_transformer_2d.py +544 -0
- diffusers/models/transformers/cogvideox_transformer_3d.py +542 -0
- diffusers/models/transformers/dit_transformer_2d.py +240 -0
- diffusers/models/transformers/dual_transformer_2d.py +9 -8
- diffusers/models/transformers/hunyuan_transformer_2d.py +578 -0
- diffusers/models/transformers/latte_transformer_3d.py +327 -0
- diffusers/models/transformers/lumina_nextdit2d.py +340 -0
- diffusers/models/transformers/pixart_transformer_2d.py +445 -0
- diffusers/models/transformers/prior_transformer.py +13 -13
- diffusers/models/transformers/sana_transformer.py +488 -0
- diffusers/models/transformers/stable_audio_transformer.py +458 -0
- diffusers/models/transformers/t5_film_transformer.py +17 -19
- diffusers/models/transformers/transformer_2d.py +297 -187
- diffusers/models/transformers/transformer_allegro.py +422 -0
- diffusers/models/transformers/transformer_cogview3plus.py +386 -0
- diffusers/models/transformers/transformer_flux.py +593 -0
- diffusers/models/transformers/transformer_hunyuan_video.py +791 -0
- diffusers/models/transformers/transformer_ltx.py +469 -0
- diffusers/models/transformers/transformer_mochi.py +499 -0
- diffusers/models/transformers/transformer_sd3.py +461 -0
- diffusers/models/transformers/transformer_temporal.py +21 -19
- diffusers/models/unets/unet_1d.py +8 -8
- diffusers/models/unets/unet_1d_blocks.py +31 -31
- diffusers/models/unets/unet_2d.py +17 -10
- diffusers/models/unets/unet_2d_blocks.py +225 -149
- diffusers/models/unets/unet_2d_condition.py +41 -40
- diffusers/models/unets/unet_2d_condition_flax.py +6 -5
- diffusers/models/unets/unet_3d_blocks.py +192 -1057
- diffusers/models/unets/unet_3d_condition.py +22 -27
- diffusers/models/unets/unet_i2vgen_xl.py +22 -18
- diffusers/models/unets/unet_kandinsky3.py +2 -2
- diffusers/models/unets/unet_motion_model.py +1413 -89
- diffusers/models/unets/unet_spatio_temporal_condition.py +40 -16
- diffusers/models/unets/unet_stable_cascade.py +19 -18
- diffusers/models/unets/uvit_2d.py +2 -2
- diffusers/models/upsampling.py +95 -26
- diffusers/models/vq_model.py +12 -164
- diffusers/optimization.py +1 -1
- diffusers/pipelines/__init__.py +202 -3
- diffusers/pipelines/allegro/__init__.py +48 -0
- diffusers/pipelines/allegro/pipeline_allegro.py +938 -0
- diffusers/pipelines/allegro/pipeline_output.py +23 -0
- diffusers/pipelines/amused/pipeline_amused.py +12 -12
- diffusers/pipelines/amused/pipeline_amused_img2img.py +14 -12
- diffusers/pipelines/amused/pipeline_amused_inpaint.py +13 -11
- diffusers/pipelines/animatediff/__init__.py +8 -0
- diffusers/pipelines/animatediff/pipeline_animatediff.py +122 -109
- diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +1106 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +1288 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +1010 -0
- diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +236 -180
- diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +1341 -0
- diffusers/pipelines/animatediff/pipeline_output.py +3 -2
- diffusers/pipelines/audioldm/pipeline_audioldm.py +14 -14
- diffusers/pipelines/audioldm2/modeling_audioldm2.py +58 -39
- diffusers/pipelines/audioldm2/pipeline_audioldm2.py +121 -36
- diffusers/pipelines/aura_flow/__init__.py +48 -0
- diffusers/pipelines/aura_flow/pipeline_aura_flow.py +584 -0
- diffusers/pipelines/auto_pipeline.py +196 -28
- diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
- diffusers/pipelines/blip_diffusion/modeling_blip2.py +6 -6
- diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +1 -1
- diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +2 -2
- diffusers/pipelines/cogvideo/__init__.py +54 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox.py +772 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py +825 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py +885 -0
- diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py +851 -0
- diffusers/pipelines/cogvideo/pipeline_output.py +20 -0
- diffusers/pipelines/cogview3/__init__.py +47 -0
- diffusers/pipelines/cogview3/pipeline_cogview3plus.py +674 -0
- diffusers/pipelines/cogview3/pipeline_output.py +21 -0
- diffusers/pipelines/consistency_models/pipeline_consistency_models.py +6 -6
- diffusers/pipelines/controlnet/__init__.py +86 -80
- diffusers/pipelines/controlnet/multicontrolnet.py +7 -182
- diffusers/pipelines/controlnet/pipeline_controlnet.py +134 -87
- diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +2 -2
- diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +93 -77
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +88 -197
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +136 -90
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +176 -80
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +125 -89
- diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +1790 -0
- diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +1501 -0
- diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +1627 -0
- diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +2 -2
- diffusers/pipelines/controlnet_hunyuandit/__init__.py +48 -0
- diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +1060 -0
- diffusers/pipelines/controlnet_sd3/__init__.py +57 -0
- diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +1133 -0
- diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +1153 -0
- diffusers/pipelines/controlnet_xs/__init__.py +68 -0
- diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py +916 -0
- diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +1111 -0
- diffusers/pipelines/ddpm/pipeline_ddpm.py +2 -2
- diffusers/pipelines/deepfloyd_if/pipeline_if.py +16 -30
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +20 -35
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +23 -41
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +22 -38
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +25 -41
- diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +19 -34
- diffusers/pipelines/deepfloyd_if/pipeline_output.py +6 -5
- diffusers/pipelines/deepfloyd_if/watermark.py +1 -1
- diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py +11 -11
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +70 -30
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +48 -25
- diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +2 -2
- diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +7 -7
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +21 -20
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +27 -29
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +33 -27
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +33 -23
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +36 -30
- diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +102 -69
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py +13 -13
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +10 -5
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +11 -6
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +10 -5
- diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +5 -5
- diffusers/pipelines/dit/pipeline_dit.py +7 -4
- diffusers/pipelines/flux/__init__.py +69 -0
- diffusers/pipelines/flux/modeling_flux.py +47 -0
- diffusers/pipelines/flux/pipeline_flux.py +957 -0
- diffusers/pipelines/flux/pipeline_flux_control.py +889 -0
- diffusers/pipelines/flux/pipeline_flux_control_img2img.py +945 -0
- diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +1141 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet.py +1006 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +998 -0
- diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +1204 -0
- diffusers/pipelines/flux/pipeline_flux_fill.py +969 -0
- diffusers/pipelines/flux/pipeline_flux_img2img.py +856 -0
- diffusers/pipelines/flux/pipeline_flux_inpaint.py +1022 -0
- diffusers/pipelines/flux/pipeline_flux_prior_redux.py +492 -0
- diffusers/pipelines/flux/pipeline_output.py +37 -0
- diffusers/pipelines/free_init_utils.py +41 -38
- diffusers/pipelines/free_noise_utils.py +596 -0
- diffusers/pipelines/hunyuan_video/__init__.py +48 -0
- diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +687 -0
- diffusers/pipelines/hunyuan_video/pipeline_output.py +20 -0
- diffusers/pipelines/hunyuandit/__init__.py +48 -0
- diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +916 -0
- diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +33 -48
- diffusers/pipelines/kandinsky/pipeline_kandinsky.py +8 -8
- diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +32 -29
- diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +11 -11
- diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +12 -12
- diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +6 -6
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +34 -31
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +10 -10
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +6 -6
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +8 -8
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +7 -7
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +6 -6
- diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py +3 -3
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +22 -35
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +26 -37
- diffusers/pipelines/kolors/__init__.py +54 -0
- diffusers/pipelines/kolors/pipeline_kolors.py +1070 -0
- diffusers/pipelines/kolors/pipeline_kolors_img2img.py +1250 -0
- diffusers/pipelines/kolors/pipeline_output.py +21 -0
- diffusers/pipelines/kolors/text_encoder.py +889 -0
- diffusers/pipelines/kolors/tokenizer.py +338 -0
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +82 -62
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +77 -60
- diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +12 -12
- diffusers/pipelines/latte/__init__.py +48 -0
- diffusers/pipelines/latte/pipeline_latte.py +881 -0
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +80 -74
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +85 -76
- diffusers/pipelines/ledits_pp/pipeline_output.py +2 -2
- diffusers/pipelines/ltx/__init__.py +50 -0
- diffusers/pipelines/ltx/pipeline_ltx.py +789 -0
- diffusers/pipelines/ltx/pipeline_ltx_image2video.py +885 -0
- diffusers/pipelines/ltx/pipeline_output.py +20 -0
- diffusers/pipelines/lumina/__init__.py +48 -0
- diffusers/pipelines/lumina/pipeline_lumina.py +890 -0
- diffusers/pipelines/marigold/__init__.py +50 -0
- diffusers/pipelines/marigold/marigold_image_processing.py +576 -0
- diffusers/pipelines/marigold/pipeline_marigold_depth.py +813 -0
- diffusers/pipelines/marigold/pipeline_marigold_normals.py +690 -0
- diffusers/pipelines/mochi/__init__.py +48 -0
- diffusers/pipelines/mochi/pipeline_mochi.py +748 -0
- diffusers/pipelines/mochi/pipeline_output.py +20 -0
- diffusers/pipelines/musicldm/pipeline_musicldm.py +14 -14
- diffusers/pipelines/pag/__init__.py +80 -0
- diffusers/pipelines/pag/pag_utils.py +243 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +1328 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +1543 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +1610 -0
- diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +1683 -0
- diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +969 -0
- diffusers/pipelines/pag/pipeline_pag_kolors.py +1136 -0
- diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py +865 -0
- diffusers/pipelines/pag/pipeline_pag_sana.py +886 -0
- diffusers/pipelines/pag/pipeline_pag_sd.py +1062 -0
- diffusers/pipelines/pag/pipeline_pag_sd_3.py +994 -0
- diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +1058 -0
- diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +866 -0
- diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +1094 -0
- diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +1356 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl.py +1345 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +1544 -0
- diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +1776 -0
- diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +17 -12
- diffusers/pipelines/pia/pipeline_pia.py +74 -164
- diffusers/pipelines/pipeline_flax_utils.py +5 -10
- diffusers/pipelines/pipeline_loading_utils.py +515 -53
- diffusers/pipelines/pipeline_utils.py +411 -222
- diffusers/pipelines/pixart_alpha/__init__.py +8 -1
- diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +76 -93
- diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py +873 -0
- diffusers/pipelines/sana/__init__.py +47 -0
- diffusers/pipelines/sana/pipeline_output.py +21 -0
- diffusers/pipelines/sana/pipeline_sana.py +884 -0
- diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +27 -23
- diffusers/pipelines/shap_e/pipeline_shap_e.py +3 -3
- diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +14 -14
- diffusers/pipelines/shap_e/renderer.py +1 -1
- diffusers/pipelines/stable_audio/__init__.py +50 -0
- diffusers/pipelines/stable_audio/modeling_stable_audio.py +158 -0
- diffusers/pipelines/stable_audio/pipeline_stable_audio.py +756 -0
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +71 -25
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +23 -19
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +35 -34
- diffusers/pipelines/stable_diffusion/__init__.py +0 -1
- diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +20 -11
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +2 -2
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +6 -6
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +145 -79
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +43 -28
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +13 -8
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +100 -68
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +109 -201
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +131 -32
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +247 -87
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +30 -29
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +35 -27
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +49 -42
- diffusers/pipelines/stable_diffusion/safety_checker.py +2 -1
- diffusers/pipelines/stable_diffusion_3/__init__.py +54 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_output.py +21 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +1140 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +1036 -0
- diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +1250 -0
- diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +29 -20
- diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +59 -58
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +31 -25
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +38 -22
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +30 -24
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +24 -23
- diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +107 -67
- diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +316 -69
- diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +10 -5
- diffusers/pipelines/stable_diffusion_safe/safety_checker.py +1 -1
- diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +98 -30
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +121 -83
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +161 -105
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +142 -218
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +45 -29
- diffusers/pipelines/stable_diffusion_xl/watermark.py +9 -3
- diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +110 -57
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +69 -39
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +105 -74
- diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +3 -2
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +29 -49
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +32 -93
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +37 -25
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +54 -40
- diffusers/pipelines/unclip/pipeline_unclip.py +6 -6
- diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +6 -6
- diffusers/pipelines/unidiffuser/modeling_text_decoder.py +1 -1
- diffusers/pipelines/unidiffuser/modeling_uvit.py +12 -12
- diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +29 -28
- diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +5 -5
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +5 -10
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +6 -8
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +4 -4
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +12 -12
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +15 -14
- diffusers/{models/dual_transformer_2d.py → quantizers/__init__.py} +2 -6
- diffusers/quantizers/auto.py +139 -0
- diffusers/quantizers/base.py +233 -0
- diffusers/quantizers/bitsandbytes/__init__.py +2 -0
- diffusers/quantizers/bitsandbytes/bnb_quantizer.py +561 -0
- diffusers/quantizers/bitsandbytes/utils.py +306 -0
- diffusers/quantizers/gguf/__init__.py +1 -0
- diffusers/quantizers/gguf/gguf_quantizer.py +159 -0
- diffusers/quantizers/gguf/utils.py +456 -0
- diffusers/quantizers/quantization_config.py +669 -0
- diffusers/quantizers/torchao/__init__.py +15 -0
- diffusers/quantizers/torchao/torchao_quantizer.py +292 -0
- diffusers/schedulers/__init__.py +12 -2
- diffusers/schedulers/deprecated/__init__.py +1 -1
- diffusers/schedulers/deprecated/scheduling_karras_ve.py +25 -25
- diffusers/schedulers/scheduling_amused.py +5 -5
- diffusers/schedulers/scheduling_consistency_decoder.py +11 -11
- diffusers/schedulers/scheduling_consistency_models.py +23 -25
- diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py +572 -0
- diffusers/schedulers/scheduling_ddim.py +27 -26
- diffusers/schedulers/scheduling_ddim_cogvideox.py +452 -0
- diffusers/schedulers/scheduling_ddim_flax.py +2 -1
- diffusers/schedulers/scheduling_ddim_inverse.py +16 -16
- diffusers/schedulers/scheduling_ddim_parallel.py +32 -31
- diffusers/schedulers/scheduling_ddpm.py +27 -30
- diffusers/schedulers/scheduling_ddpm_flax.py +7 -3
- diffusers/schedulers/scheduling_ddpm_parallel.py +33 -36
- diffusers/schedulers/scheduling_ddpm_wuerstchen.py +14 -14
- diffusers/schedulers/scheduling_deis_multistep.py +150 -50
- diffusers/schedulers/scheduling_dpm_cogvideox.py +489 -0
- diffusers/schedulers/scheduling_dpmsolver_multistep.py +221 -84
- diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +2 -2
- diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +158 -52
- diffusers/schedulers/scheduling_dpmsolver_sde.py +153 -34
- diffusers/schedulers/scheduling_dpmsolver_singlestep.py +275 -86
- diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +81 -57
- diffusers/schedulers/scheduling_edm_euler.py +62 -39
- diffusers/schedulers/scheduling_euler_ancestral_discrete.py +30 -29
- diffusers/schedulers/scheduling_euler_discrete.py +255 -74
- diffusers/schedulers/scheduling_flow_match_euler_discrete.py +458 -0
- diffusers/schedulers/scheduling_flow_match_heun_discrete.py +320 -0
- diffusers/schedulers/scheduling_heun_discrete.py +174 -46
- diffusers/schedulers/scheduling_ipndm.py +9 -9
- diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +138 -29
- diffusers/schedulers/scheduling_k_dpm_2_discrete.py +132 -26
- diffusers/schedulers/scheduling_karras_ve_flax.py +6 -6
- diffusers/schedulers/scheduling_lcm.py +23 -29
- diffusers/schedulers/scheduling_lms_discrete.py +105 -28
- diffusers/schedulers/scheduling_pndm.py +20 -20
- diffusers/schedulers/scheduling_repaint.py +21 -21
- diffusers/schedulers/scheduling_sasolver.py +157 -60
- diffusers/schedulers/scheduling_sde_ve.py +19 -19
- diffusers/schedulers/scheduling_tcd.py +41 -36
- diffusers/schedulers/scheduling_unclip.py +19 -16
- diffusers/schedulers/scheduling_unipc_multistep.py +243 -47
- diffusers/schedulers/scheduling_utils.py +12 -5
- diffusers/schedulers/scheduling_utils_flax.py +1 -3
- diffusers/schedulers/scheduling_vq_diffusion.py +10 -10
- diffusers/training_utils.py +214 -30
- diffusers/utils/__init__.py +17 -1
- diffusers/utils/constants.py +3 -0
- diffusers/utils/doc_utils.py +1 -0
- diffusers/utils/dummy_pt_objects.py +592 -7
- diffusers/utils/dummy_torch_and_torchsde_objects.py +15 -0
- diffusers/utils/dummy_torch_and_transformers_and_sentencepiece_objects.py +47 -0
- diffusers/utils/dummy_torch_and_transformers_objects.py +1001 -71
- diffusers/utils/dynamic_modules_utils.py +34 -29
- diffusers/utils/export_utils.py +50 -6
- diffusers/utils/hub_utils.py +131 -17
- diffusers/utils/import_utils.py +210 -8
- diffusers/utils/loading_utils.py +118 -5
- diffusers/utils/logging.py +4 -2
- diffusers/utils/peft_utils.py +37 -7
- diffusers/utils/state_dict_utils.py +13 -2
- diffusers/utils/testing_utils.py +193 -11
- diffusers/utils/torch_utils.py +4 -0
- diffusers/video_processor.py +113 -0
- {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/METADATA +82 -91
- diffusers-0.32.2.dist-info/RECORD +550 -0
- {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/WHEEL +1 -1
- diffusers/loaders/autoencoder.py +0 -146
- diffusers/loaders/controlnet.py +0 -136
- diffusers/loaders/lora.py +0 -1349
- diffusers/models/prior_transformer.py +0 -12
- diffusers/models/t5_film_transformer.py +0 -70
- diffusers/models/transformer_2d.py +0 -25
- diffusers/models/transformer_temporal.py +0 -34
- diffusers/models/unet_1d.py +0 -26
- diffusers/models/unet_1d_blocks.py +0 -203
- diffusers/models/unet_2d.py +0 -27
- diffusers/models/unet_2d_blocks.py +0 -375
- diffusers/models/unet_2d_condition.py +0 -25
- diffusers-0.27.1.dist-info/RECORD +0 -399
- {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/LICENSE +0 -0
- {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/entry_points.txt +0 -0
- {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,969 @@
|
|
1
|
+
# Copyright 2024 HunyuanDiT Authors and The HuggingFace Team. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
import inspect
|
16
|
+
from typing import Callable, Dict, List, Optional, Tuple, Union
|
17
|
+
|
18
|
+
import numpy as np
|
19
|
+
import torch
|
20
|
+
from transformers import BertModel, BertTokenizer, CLIPImageProcessor, MT5Tokenizer, T5EncoderModel
|
21
|
+
|
22
|
+
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
|
23
|
+
|
24
|
+
from ...callbacks import MultiPipelineCallbacks, PipelineCallback
|
25
|
+
from ...image_processor import VaeImageProcessor
|
26
|
+
from ...models import AutoencoderKL, HunyuanDiT2DModel
|
27
|
+
from ...models.attention_processor import PAGCFGHunyuanAttnProcessor2_0, PAGHunyuanAttnProcessor2_0
|
28
|
+
from ...models.embeddings import get_2d_rotary_pos_embed
|
29
|
+
from ...pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
|
30
|
+
from ...schedulers import DDPMScheduler
|
31
|
+
from ...utils import (
|
32
|
+
is_torch_xla_available,
|
33
|
+
logging,
|
34
|
+
replace_example_docstring,
|
35
|
+
)
|
36
|
+
from ...utils.torch_utils import randn_tensor
|
37
|
+
from ..pipeline_utils import DiffusionPipeline
|
38
|
+
from .pag_utils import PAGMixin
|
39
|
+
|
40
|
+
|
41
|
+
if is_torch_xla_available():
|
42
|
+
import torch_xla.core.xla_model as xm
|
43
|
+
|
44
|
+
XLA_AVAILABLE = True
|
45
|
+
else:
|
46
|
+
XLA_AVAILABLE = False
|
47
|
+
|
48
|
+
|
49
|
+
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
50
|
+
|
51
|
+
EXAMPLE_DOC_STRING = """
|
52
|
+
Examples:
|
53
|
+
```python
|
54
|
+
>>> import torch
|
55
|
+
>>> from diffusers import AutoPipelineForText2Image
|
56
|
+
|
57
|
+
>>> pipe = AutoPipelineForText2Image.from_pretrained(
|
58
|
+
... "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers",
|
59
|
+
... torch_dtype=torch.float16,
|
60
|
+
... enable_pag=True,
|
61
|
+
... pag_applied_layers=[14],
|
62
|
+
... ).to("cuda")
|
63
|
+
|
64
|
+
>>> # prompt = "an astronaut riding a horse"
|
65
|
+
>>> prompt = "一个宇航员在骑马"
|
66
|
+
>>> image = pipe(prompt, guidance_scale=4, pag_scale=3).images[0]
|
67
|
+
```
|
68
|
+
"""
|
69
|
+
|
70
|
+
STANDARD_RATIO = np.array(
|
71
|
+
[
|
72
|
+
1.0, # 1:1
|
73
|
+
4.0 / 3.0, # 4:3
|
74
|
+
3.0 / 4.0, # 3:4
|
75
|
+
16.0 / 9.0, # 16:9
|
76
|
+
9.0 / 16.0, # 9:16
|
77
|
+
]
|
78
|
+
)
|
79
|
+
STANDARD_SHAPE = [
|
80
|
+
[(1024, 1024), (1280, 1280)], # 1:1
|
81
|
+
[(1024, 768), (1152, 864), (1280, 960)], # 4:3
|
82
|
+
[(768, 1024), (864, 1152), (960, 1280)], # 3:4
|
83
|
+
[(1280, 768)], # 16:9
|
84
|
+
[(768, 1280)], # 9:16
|
85
|
+
]
|
86
|
+
STANDARD_AREA = [np.array([w * h for w, h in shapes]) for shapes in STANDARD_SHAPE]
|
87
|
+
SUPPORTED_SHAPE = [
|
88
|
+
(1024, 1024),
|
89
|
+
(1280, 1280), # 1:1
|
90
|
+
(1024, 768),
|
91
|
+
(1152, 864),
|
92
|
+
(1280, 960), # 4:3
|
93
|
+
(768, 1024),
|
94
|
+
(864, 1152),
|
95
|
+
(960, 1280), # 3:4
|
96
|
+
(1280, 768), # 16:9
|
97
|
+
(768, 1280), # 9:16
|
98
|
+
]
|
99
|
+
|
100
|
+
|
101
|
+
def map_to_standard_shapes(target_width, target_height):
|
102
|
+
target_ratio = target_width / target_height
|
103
|
+
closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio))
|
104
|
+
closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height))
|
105
|
+
width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx]
|
106
|
+
return width, height
|
107
|
+
|
108
|
+
|
109
|
+
def get_resize_crop_region_for_grid(src, tgt_size):
|
110
|
+
th = tw = tgt_size
|
111
|
+
h, w = src
|
112
|
+
|
113
|
+
r = h / w
|
114
|
+
|
115
|
+
# resize
|
116
|
+
if r > 1:
|
117
|
+
resize_height = th
|
118
|
+
resize_width = int(round(th / h * w))
|
119
|
+
else:
|
120
|
+
resize_width = tw
|
121
|
+
resize_height = int(round(tw / w * h))
|
122
|
+
|
123
|
+
crop_top = int(round((th - resize_height) / 2.0))
|
124
|
+
crop_left = int(round((tw - resize_width) / 2.0))
|
125
|
+
|
126
|
+
return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width)
|
127
|
+
|
128
|
+
|
129
|
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
|
130
|
+
def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
|
131
|
+
r"""
|
132
|
+
Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
|
133
|
+
Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
|
134
|
+
Flawed](https://arxiv.org/pdf/2305.08891.pdf).
|
135
|
+
|
136
|
+
Args:
|
137
|
+
noise_cfg (`torch.Tensor`):
|
138
|
+
The predicted noise tensor for the guided diffusion process.
|
139
|
+
noise_pred_text (`torch.Tensor`):
|
140
|
+
The predicted noise tensor for the text-guided diffusion process.
|
141
|
+
guidance_rescale (`float`, *optional*, defaults to 0.0):
|
142
|
+
A rescale factor applied to the noise predictions.
|
143
|
+
|
144
|
+
Returns:
|
145
|
+
noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
|
146
|
+
"""
|
147
|
+
std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
|
148
|
+
std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
|
149
|
+
# rescale the results from guidance (fixes overexposure)
|
150
|
+
noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
|
151
|
+
# mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
|
152
|
+
noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
|
153
|
+
return noise_cfg
|
154
|
+
|
155
|
+
|
156
|
+
class HunyuanDiTPAGPipeline(DiffusionPipeline, PAGMixin):
|
157
|
+
r"""
|
158
|
+
Pipeline for English/Chinese-to-image generation using HunyuanDiT and [Perturbed Attention
|
159
|
+
Guidance](https://huggingface.co/docs/diffusers/en/using-diffusers/pag).
|
160
|
+
|
161
|
+
This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
|
162
|
+
library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
|
163
|
+
|
164
|
+
HunyuanDiT uses two text encoders: [mT5](https://huggingface.co/google/mt5-base) and [bilingual CLIP](fine-tuned by
|
165
|
+
ourselves)
|
166
|
+
|
167
|
+
Args:
|
168
|
+
vae ([`AutoencoderKL`]):
|
169
|
+
Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. We use
|
170
|
+
`sdxl-vae-fp16-fix`.
|
171
|
+
text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]):
|
172
|
+
Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
|
173
|
+
HunyuanDiT uses a fine-tuned [bilingual CLIP].
|
174
|
+
tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]):
|
175
|
+
A `BertTokenizer` or `CLIPTokenizer` to tokenize text.
|
176
|
+
transformer ([`HunyuanDiT2DModel`]):
|
177
|
+
The HunyuanDiT model designed by Tencent Hunyuan.
|
178
|
+
text_encoder_2 (`T5EncoderModel`):
|
179
|
+
The mT5 embedder. Specifically, it is 't5-v1_1-xxl'.
|
180
|
+
tokenizer_2 (`MT5Tokenizer`):
|
181
|
+
The tokenizer for the mT5 embedder.
|
182
|
+
scheduler ([`DDPMScheduler`]):
|
183
|
+
A scheduler to be used in combination with HunyuanDiT to denoise the encoded image latents.
|
184
|
+
"""
|
185
|
+
|
186
|
+
model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
|
187
|
+
_optional_components = [
|
188
|
+
"safety_checker",
|
189
|
+
"feature_extractor",
|
190
|
+
"text_encoder_2",
|
191
|
+
"tokenizer_2",
|
192
|
+
"text_encoder",
|
193
|
+
"tokenizer",
|
194
|
+
]
|
195
|
+
_exclude_from_cpu_offload = ["safety_checker"]
|
196
|
+
_callback_tensor_inputs = [
|
197
|
+
"latents",
|
198
|
+
"prompt_embeds",
|
199
|
+
"negative_prompt_embeds",
|
200
|
+
"prompt_embeds_2",
|
201
|
+
"negative_prompt_embeds_2",
|
202
|
+
]
|
203
|
+
|
204
|
+
def __init__(
|
205
|
+
self,
|
206
|
+
vae: AutoencoderKL,
|
207
|
+
text_encoder: BertModel,
|
208
|
+
tokenizer: BertTokenizer,
|
209
|
+
transformer: HunyuanDiT2DModel,
|
210
|
+
scheduler: DDPMScheduler,
|
211
|
+
safety_checker: Optional[StableDiffusionSafetyChecker] = None,
|
212
|
+
feature_extractor: Optional[CLIPImageProcessor] = None,
|
213
|
+
requires_safety_checker: bool = True,
|
214
|
+
text_encoder_2: Optional[T5EncoderModel] = None,
|
215
|
+
tokenizer_2: Optional[MT5Tokenizer] = None,
|
216
|
+
pag_applied_layers: Union[str, List[str]] = "blocks.1", # "blocks.16.attn1", "blocks.16", "16", 16
|
217
|
+
):
|
218
|
+
super().__init__()
|
219
|
+
|
220
|
+
self.register_modules(
|
221
|
+
vae=vae,
|
222
|
+
text_encoder=text_encoder,
|
223
|
+
tokenizer=tokenizer,
|
224
|
+
tokenizer_2=tokenizer_2,
|
225
|
+
transformer=transformer,
|
226
|
+
scheduler=scheduler,
|
227
|
+
safety_checker=safety_checker,
|
228
|
+
feature_extractor=feature_extractor,
|
229
|
+
text_encoder_2=text_encoder_2,
|
230
|
+
)
|
231
|
+
|
232
|
+
if safety_checker is None and requires_safety_checker:
|
233
|
+
logger.warning(
|
234
|
+
f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
|
235
|
+
" that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
|
236
|
+
" results in services or applications open to the public. Both the diffusers team and Hugging Face"
|
237
|
+
" strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
|
238
|
+
" it only for use-cases that involve analyzing network behavior or auditing its results. For more"
|
239
|
+
" information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
|
240
|
+
)
|
241
|
+
|
242
|
+
if safety_checker is not None and feature_extractor is None:
|
243
|
+
raise ValueError(
|
244
|
+
"Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
|
245
|
+
" checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
|
246
|
+
)
|
247
|
+
|
248
|
+
self.vae_scale_factor = (
|
249
|
+
2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
|
250
|
+
)
|
251
|
+
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
|
252
|
+
self.register_to_config(requires_safety_checker=requires_safety_checker)
|
253
|
+
self.default_sample_size = (
|
254
|
+
self.transformer.config.sample_size
|
255
|
+
if hasattr(self, "transformer") and self.transformer is not None
|
256
|
+
else 128
|
257
|
+
)
|
258
|
+
|
259
|
+
self.set_pag_applied_layers(
|
260
|
+
pag_applied_layers, pag_attn_processors=(PAGCFGHunyuanAttnProcessor2_0(), PAGHunyuanAttnProcessor2_0())
|
261
|
+
)
|
262
|
+
|
263
|
+
# Copied from diffusers.pipelines.hunyuandit.pipeline_hunyuandit.HunyuanDiTPipeline.encode_prompt
|
264
|
+
def encode_prompt(
|
265
|
+
self,
|
266
|
+
prompt: str,
|
267
|
+
device: torch.device = None,
|
268
|
+
dtype: torch.dtype = None,
|
269
|
+
num_images_per_prompt: int = 1,
|
270
|
+
do_classifier_free_guidance: bool = True,
|
271
|
+
negative_prompt: Optional[str] = None,
|
272
|
+
prompt_embeds: Optional[torch.Tensor] = None,
|
273
|
+
negative_prompt_embeds: Optional[torch.Tensor] = None,
|
274
|
+
prompt_attention_mask: Optional[torch.Tensor] = None,
|
275
|
+
negative_prompt_attention_mask: Optional[torch.Tensor] = None,
|
276
|
+
max_sequence_length: Optional[int] = None,
|
277
|
+
text_encoder_index: int = 0,
|
278
|
+
):
|
279
|
+
r"""
|
280
|
+
Encodes the prompt into text encoder hidden states.
|
281
|
+
|
282
|
+
Args:
|
283
|
+
prompt (`str` or `List[str]`, *optional*):
|
284
|
+
prompt to be encoded
|
285
|
+
device: (`torch.device`):
|
286
|
+
torch device
|
287
|
+
dtype (`torch.dtype`):
|
288
|
+
torch dtype
|
289
|
+
num_images_per_prompt (`int`):
|
290
|
+
number of images that should be generated per prompt
|
291
|
+
do_classifier_free_guidance (`bool`):
|
292
|
+
whether to use classifier free guidance or not
|
293
|
+
negative_prompt (`str` or `List[str]`, *optional*):
|
294
|
+
The prompt or prompts not to guide the image generation. If not defined, one has to pass
|
295
|
+
`negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
|
296
|
+
less than `1`).
|
297
|
+
prompt_embeds (`torch.Tensor`, *optional*):
|
298
|
+
Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
|
299
|
+
provided, text embeddings will be generated from `prompt` input argument.
|
300
|
+
negative_prompt_embeds (`torch.Tensor`, *optional*):
|
301
|
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
|
302
|
+
weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
|
303
|
+
argument.
|
304
|
+
prompt_attention_mask (`torch.Tensor`, *optional*):
|
305
|
+
Attention mask for the prompt. Required when `prompt_embeds` is passed directly.
|
306
|
+
negative_prompt_attention_mask (`torch.Tensor`, *optional*):
|
307
|
+
Attention mask for the negative prompt. Required when `negative_prompt_embeds` is passed directly.
|
308
|
+
max_sequence_length (`int`, *optional*): maximum sequence length to use for the prompt.
|
309
|
+
text_encoder_index (`int`, *optional*):
|
310
|
+
Index of the text encoder to use. `0` for clip and `1` for T5.
|
311
|
+
"""
|
312
|
+
if dtype is None:
|
313
|
+
if self.text_encoder_2 is not None:
|
314
|
+
dtype = self.text_encoder_2.dtype
|
315
|
+
elif self.transformer is not None:
|
316
|
+
dtype = self.transformer.dtype
|
317
|
+
else:
|
318
|
+
dtype = None
|
319
|
+
|
320
|
+
if device is None:
|
321
|
+
device = self._execution_device
|
322
|
+
|
323
|
+
tokenizers = [self.tokenizer, self.tokenizer_2]
|
324
|
+
text_encoders = [self.text_encoder, self.text_encoder_2]
|
325
|
+
|
326
|
+
tokenizer = tokenizers[text_encoder_index]
|
327
|
+
text_encoder = text_encoders[text_encoder_index]
|
328
|
+
|
329
|
+
if max_sequence_length is None:
|
330
|
+
if text_encoder_index == 0:
|
331
|
+
max_length = 77
|
332
|
+
if text_encoder_index == 1:
|
333
|
+
max_length = 256
|
334
|
+
else:
|
335
|
+
max_length = max_sequence_length
|
336
|
+
|
337
|
+
if prompt is not None and isinstance(prompt, str):
|
338
|
+
batch_size = 1
|
339
|
+
elif prompt is not None and isinstance(prompt, list):
|
340
|
+
batch_size = len(prompt)
|
341
|
+
else:
|
342
|
+
batch_size = prompt_embeds.shape[0]
|
343
|
+
|
344
|
+
if prompt_embeds is None:
|
345
|
+
text_inputs = tokenizer(
|
346
|
+
prompt,
|
347
|
+
padding="max_length",
|
348
|
+
max_length=max_length,
|
349
|
+
truncation=True,
|
350
|
+
return_attention_mask=True,
|
351
|
+
return_tensors="pt",
|
352
|
+
)
|
353
|
+
text_input_ids = text_inputs.input_ids
|
354
|
+
untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
|
355
|
+
|
356
|
+
if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
|
357
|
+
text_input_ids, untruncated_ids
|
358
|
+
):
|
359
|
+
removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
|
360
|
+
logger.warning(
|
361
|
+
"The following part of your input was truncated because CLIP can only handle sequences up to"
|
362
|
+
f" {tokenizer.model_max_length} tokens: {removed_text}"
|
363
|
+
)
|
364
|
+
|
365
|
+
prompt_attention_mask = text_inputs.attention_mask.to(device)
|
366
|
+
prompt_embeds = text_encoder(
|
367
|
+
text_input_ids.to(device),
|
368
|
+
attention_mask=prompt_attention_mask,
|
369
|
+
)
|
370
|
+
prompt_embeds = prompt_embeds[0]
|
371
|
+
prompt_attention_mask = prompt_attention_mask.repeat(num_images_per_prompt, 1)
|
372
|
+
|
373
|
+
prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
|
374
|
+
|
375
|
+
bs_embed, seq_len, _ = prompt_embeds.shape
|
376
|
+
# duplicate text embeddings for each generation per prompt, using mps friendly method
|
377
|
+
prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
378
|
+
prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
|
379
|
+
|
380
|
+
# get unconditional embeddings for classifier free guidance
|
381
|
+
if do_classifier_free_guidance and negative_prompt_embeds is None:
|
382
|
+
uncond_tokens: List[str]
|
383
|
+
if negative_prompt is None:
|
384
|
+
uncond_tokens = [""] * batch_size
|
385
|
+
elif prompt is not None and type(prompt) is not type(negative_prompt):
|
386
|
+
raise TypeError(
|
387
|
+
f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
|
388
|
+
f" {type(prompt)}."
|
389
|
+
)
|
390
|
+
elif isinstance(negative_prompt, str):
|
391
|
+
uncond_tokens = [negative_prompt]
|
392
|
+
elif batch_size != len(negative_prompt):
|
393
|
+
raise ValueError(
|
394
|
+
f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
|
395
|
+
f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
|
396
|
+
" the batch size of `prompt`."
|
397
|
+
)
|
398
|
+
else:
|
399
|
+
uncond_tokens = negative_prompt
|
400
|
+
|
401
|
+
max_length = prompt_embeds.shape[1]
|
402
|
+
uncond_input = tokenizer(
|
403
|
+
uncond_tokens,
|
404
|
+
padding="max_length",
|
405
|
+
max_length=max_length,
|
406
|
+
truncation=True,
|
407
|
+
return_tensors="pt",
|
408
|
+
)
|
409
|
+
|
410
|
+
negative_prompt_attention_mask = uncond_input.attention_mask.to(device)
|
411
|
+
negative_prompt_embeds = text_encoder(
|
412
|
+
uncond_input.input_ids.to(device),
|
413
|
+
attention_mask=negative_prompt_attention_mask,
|
414
|
+
)
|
415
|
+
negative_prompt_embeds = negative_prompt_embeds[0]
|
416
|
+
negative_prompt_attention_mask = negative_prompt_attention_mask.repeat(num_images_per_prompt, 1)
|
417
|
+
|
418
|
+
if do_classifier_free_guidance:
|
419
|
+
# duplicate unconditional embeddings for each generation per prompt, using mps friendly method
|
420
|
+
seq_len = negative_prompt_embeds.shape[1]
|
421
|
+
|
422
|
+
negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
|
423
|
+
|
424
|
+
negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
|
425
|
+
negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
|
426
|
+
|
427
|
+
return prompt_embeds, negative_prompt_embeds, prompt_attention_mask, negative_prompt_attention_mask
|
428
|
+
|
429
|
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
|
430
|
+
def run_safety_checker(self, image, device, dtype):
|
431
|
+
if self.safety_checker is None:
|
432
|
+
has_nsfw_concept = None
|
433
|
+
else:
|
434
|
+
if torch.is_tensor(image):
|
435
|
+
feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")
|
436
|
+
else:
|
437
|
+
feature_extractor_input = self.image_processor.numpy_to_pil(image)
|
438
|
+
safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)
|
439
|
+
image, has_nsfw_concept = self.safety_checker(
|
440
|
+
images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
|
441
|
+
)
|
442
|
+
return image, has_nsfw_concept
|
443
|
+
|
444
|
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
|
445
|
+
def prepare_extra_step_kwargs(self, generator, eta):
|
446
|
+
# prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
|
447
|
+
# eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
|
448
|
+
# eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
|
449
|
+
# and should be between [0, 1]
|
450
|
+
|
451
|
+
accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
|
452
|
+
extra_step_kwargs = {}
|
453
|
+
if accepts_eta:
|
454
|
+
extra_step_kwargs["eta"] = eta
|
455
|
+
|
456
|
+
# check if the scheduler accepts generator
|
457
|
+
accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
|
458
|
+
if accepts_generator:
|
459
|
+
extra_step_kwargs["generator"] = generator
|
460
|
+
return extra_step_kwargs
|
461
|
+
|
462
|
+
# Copied from diffusers.pipelines.hunyuandit.pipeline_hunyuandit.HunyuanDiTPipeline.check_inputs
|
463
|
+
def check_inputs(
|
464
|
+
self,
|
465
|
+
prompt,
|
466
|
+
height,
|
467
|
+
width,
|
468
|
+
negative_prompt=None,
|
469
|
+
prompt_embeds=None,
|
470
|
+
negative_prompt_embeds=None,
|
471
|
+
prompt_attention_mask=None,
|
472
|
+
negative_prompt_attention_mask=None,
|
473
|
+
prompt_embeds_2=None,
|
474
|
+
negative_prompt_embeds_2=None,
|
475
|
+
prompt_attention_mask_2=None,
|
476
|
+
negative_prompt_attention_mask_2=None,
|
477
|
+
callback_on_step_end_tensor_inputs=None,
|
478
|
+
):
|
479
|
+
if height % 8 != 0 or width % 8 != 0:
|
480
|
+
raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
|
481
|
+
|
482
|
+
if callback_on_step_end_tensor_inputs is not None and not all(
|
483
|
+
k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
|
484
|
+
):
|
485
|
+
raise ValueError(
|
486
|
+
f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
|
487
|
+
)
|
488
|
+
|
489
|
+
if prompt is not None and prompt_embeds is not None:
|
490
|
+
raise ValueError(
|
491
|
+
f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
492
|
+
" only forward one of the two."
|
493
|
+
)
|
494
|
+
elif prompt is None and prompt_embeds is None:
|
495
|
+
raise ValueError(
|
496
|
+
"Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
|
497
|
+
)
|
498
|
+
elif prompt is None and prompt_embeds_2 is None:
|
499
|
+
raise ValueError(
|
500
|
+
"Provide either `prompt` or `prompt_embeds_2`. Cannot leave both `prompt` and `prompt_embeds_2` undefined."
|
501
|
+
)
|
502
|
+
elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
|
503
|
+
raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
|
504
|
+
|
505
|
+
if prompt_embeds is not None and prompt_attention_mask is None:
|
506
|
+
raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.")
|
507
|
+
|
508
|
+
if prompt_embeds_2 is not None and prompt_attention_mask_2 is None:
|
509
|
+
raise ValueError("Must provide `prompt_attention_mask_2` when specifying `prompt_embeds_2`.")
|
510
|
+
|
511
|
+
if negative_prompt is not None and negative_prompt_embeds is not None:
|
512
|
+
raise ValueError(
|
513
|
+
f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
|
514
|
+
f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
|
515
|
+
)
|
516
|
+
|
517
|
+
if negative_prompt_embeds is not None and negative_prompt_attention_mask is None:
|
518
|
+
raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.")
|
519
|
+
|
520
|
+
if negative_prompt_embeds_2 is not None and negative_prompt_attention_mask_2 is None:
|
521
|
+
raise ValueError(
|
522
|
+
"Must provide `negative_prompt_attention_mask_2` when specifying `negative_prompt_embeds_2`."
|
523
|
+
)
|
524
|
+
if prompt_embeds is not None and negative_prompt_embeds is not None:
|
525
|
+
if prompt_embeds.shape != negative_prompt_embeds.shape:
|
526
|
+
raise ValueError(
|
527
|
+
"`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
|
528
|
+
f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
|
529
|
+
f" {negative_prompt_embeds.shape}."
|
530
|
+
)
|
531
|
+
if prompt_embeds_2 is not None and negative_prompt_embeds_2 is not None:
|
532
|
+
if prompt_embeds_2.shape != negative_prompt_embeds_2.shape:
|
533
|
+
raise ValueError(
|
534
|
+
"`prompt_embeds_2` and `negative_prompt_embeds_2` must have the same shape when passed directly, but"
|
535
|
+
f" got: `prompt_embeds_2` {prompt_embeds_2.shape} != `negative_prompt_embeds_2`"
|
536
|
+
f" {negative_prompt_embeds_2.shape}."
|
537
|
+
)
|
538
|
+
|
539
|
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
|
540
|
+
def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
|
541
|
+
shape = (
|
542
|
+
batch_size,
|
543
|
+
num_channels_latents,
|
544
|
+
int(height) // self.vae_scale_factor,
|
545
|
+
int(width) // self.vae_scale_factor,
|
546
|
+
)
|
547
|
+
if isinstance(generator, list) and len(generator) != batch_size:
|
548
|
+
raise ValueError(
|
549
|
+
f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
|
550
|
+
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
|
551
|
+
)
|
552
|
+
|
553
|
+
if latents is None:
|
554
|
+
latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
|
555
|
+
else:
|
556
|
+
latents = latents.to(device)
|
557
|
+
|
558
|
+
# scale the initial noise by the standard deviation required by the scheduler
|
559
|
+
latents = latents * self.scheduler.init_noise_sigma
|
560
|
+
return latents
|
561
|
+
|
562
|
+
@property
|
563
|
+
def guidance_scale(self):
|
564
|
+
return self._guidance_scale
|
565
|
+
|
566
|
+
@property
|
567
|
+
def guidance_rescale(self):
|
568
|
+
return self._guidance_rescale
|
569
|
+
|
570
|
+
# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
|
571
|
+
# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
|
572
|
+
# corresponds to doing no classifier free guidance.
|
573
|
+
@property
|
574
|
+
def do_classifier_free_guidance(self):
|
575
|
+
return self._guidance_scale > 1
|
576
|
+
|
577
|
+
@property
|
578
|
+
def num_timesteps(self):
|
579
|
+
return self._num_timesteps
|
580
|
+
|
581
|
+
@property
|
582
|
+
def interrupt(self):
|
583
|
+
return self._interrupt
|
584
|
+
|
585
|
+
@torch.no_grad()
|
586
|
+
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
587
|
+
def __call__(
|
588
|
+
self,
|
589
|
+
prompt: Union[str, List[str]] = None,
|
590
|
+
height: Optional[int] = None,
|
591
|
+
width: Optional[int] = None,
|
592
|
+
num_inference_steps: Optional[int] = 50,
|
593
|
+
guidance_scale: Optional[float] = 5.0,
|
594
|
+
negative_prompt: Optional[Union[str, List[str]]] = None,
|
595
|
+
num_images_per_prompt: Optional[int] = 1,
|
596
|
+
eta: Optional[float] = 0.0,
|
597
|
+
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
|
598
|
+
latents: Optional[torch.Tensor] = None,
|
599
|
+
prompt_embeds: Optional[torch.Tensor] = None,
|
600
|
+
prompt_embeds_2: Optional[torch.Tensor] = None,
|
601
|
+
negative_prompt_embeds: Optional[torch.Tensor] = None,
|
602
|
+
negative_prompt_embeds_2: Optional[torch.Tensor] = None,
|
603
|
+
prompt_attention_mask: Optional[torch.Tensor] = None,
|
604
|
+
prompt_attention_mask_2: Optional[torch.Tensor] = None,
|
605
|
+
negative_prompt_attention_mask: Optional[torch.Tensor] = None,
|
606
|
+
negative_prompt_attention_mask_2: Optional[torch.Tensor] = None,
|
607
|
+
output_type: Optional[str] = "pil",
|
608
|
+
return_dict: bool = True,
|
609
|
+
callback_on_step_end: Optional[
|
610
|
+
Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
|
611
|
+
] = None,
|
612
|
+
callback_on_step_end_tensor_inputs: List[str] = ["latents"],
|
613
|
+
guidance_rescale: float = 0.0,
|
614
|
+
original_size: Optional[Tuple[int, int]] = (1024, 1024),
|
615
|
+
target_size: Optional[Tuple[int, int]] = None,
|
616
|
+
crops_coords_top_left: Tuple[int, int] = (0, 0),
|
617
|
+
use_resolution_binning: bool = True,
|
618
|
+
pag_scale: float = 3.0,
|
619
|
+
pag_adaptive_scale: float = 0.0,
|
620
|
+
):
|
621
|
+
r"""
|
622
|
+
The call function to the pipeline for generation with HunyuanDiT.
|
623
|
+
|
624
|
+
Args:
|
625
|
+
prompt (`str` or `List[str]`, *optional*):
|
626
|
+
The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
|
627
|
+
height (`int`):
|
628
|
+
The height in pixels of the generated image.
|
629
|
+
width (`int`):
|
630
|
+
The width in pixels of the generated image.
|
631
|
+
num_inference_steps (`int`, *optional*, defaults to 50):
|
632
|
+
The number of denoising steps. More denoising steps usually lead to a higher quality image at the
|
633
|
+
expense of slower inference. This parameter is modulated by `strength`.
|
634
|
+
guidance_scale (`float`, *optional*, defaults to 7.5):
|
635
|
+
A higher guidance scale value encourages the model to generate images closely linked to the text
|
636
|
+
`prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
|
637
|
+
negative_prompt (`str` or `List[str]`, *optional*):
|
638
|
+
The prompt or prompts to guide what to not include in image generation. If not defined, you need to
|
639
|
+
pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
|
640
|
+
num_images_per_prompt (`int`, *optional*, defaults to 1):
|
641
|
+
The number of images to generate per prompt.
|
642
|
+
eta (`float`, *optional*, defaults to 0.0):
|
643
|
+
Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies
|
644
|
+
to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
|
645
|
+
generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
|
646
|
+
A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
|
647
|
+
generation deterministic.
|
648
|
+
prompt_embeds (`torch.Tensor`, *optional*):
|
649
|
+
Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
|
650
|
+
provided, text embeddings are generated from the `prompt` input argument.
|
651
|
+
prompt_embeds_2 (`torch.Tensor`, *optional*):
|
652
|
+
Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
|
653
|
+
provided, text embeddings are generated from the `prompt` input argument.
|
654
|
+
negative_prompt_embeds (`torch.Tensor`, *optional*):
|
655
|
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
|
656
|
+
not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
|
657
|
+
negative_prompt_embeds_2 (`torch.Tensor`, *optional*):
|
658
|
+
Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
|
659
|
+
not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
|
660
|
+
prompt_attention_mask (`torch.Tensor`, *optional*):
|
661
|
+
Attention mask for the prompt. Required when `prompt_embeds` is passed directly.
|
662
|
+
prompt_attention_mask_2 (`torch.Tensor`, *optional*):
|
663
|
+
Attention mask for the prompt. Required when `prompt_embeds_2` is passed directly.
|
664
|
+
negative_prompt_attention_mask (`torch.Tensor`, *optional*):
|
665
|
+
Attention mask for the negative prompt. Required when `negative_prompt_embeds` is passed directly.
|
666
|
+
negative_prompt_attention_mask_2 (`torch.Tensor`, *optional*):
|
667
|
+
Attention mask for the negative prompt. Required when `negative_prompt_embeds_2` is passed directly.
|
668
|
+
output_type (`str`, *optional*, defaults to `"pil"`):
|
669
|
+
The output format of the generated image. Choose between `PIL.Image` or `np.array`.
|
670
|
+
return_dict (`bool`, *optional*, defaults to `True`):
|
671
|
+
Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
|
672
|
+
plain tuple.
|
673
|
+
callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
|
674
|
+
A callback function or a list of callback functions to be called at the end of each denoising step.
|
675
|
+
callback_on_step_end_tensor_inputs (`List[str]`, *optional*):
|
676
|
+
A list of tensor inputs that should be passed to the callback function. If not defined, all tensor
|
677
|
+
inputs will be passed.
|
678
|
+
guidance_rescale (`float`, *optional*, defaults to 0.0):
|
679
|
+
Rescale the noise_cfg according to `guidance_rescale`. Based on findings of [Common Diffusion Noise
|
680
|
+
Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
|
681
|
+
original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`):
|
682
|
+
The original size of the image. Used to calculate the time ids.
|
683
|
+
target_size (`Tuple[int, int]`, *optional*):
|
684
|
+
The target size of the image. Used to calculate the time ids.
|
685
|
+
crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`):
|
686
|
+
The top left coordinates of the crop. Used to calculate the time ids.
|
687
|
+
use_resolution_binning (`bool`, *optional*, defaults to `True`):
|
688
|
+
Whether to use resolution binning or not. If `True`, the input resolution will be mapped to the closest
|
689
|
+
standard resolution. Supported resolutions are 1024x1024, 1280x1280, 1024x768, 1152x864, 1280x960,
|
690
|
+
768x1024, 864x1152, 960x1280, 1280x768, and 768x1280. It is recommended to set this to `True`.
|
691
|
+
pag_scale (`float`, *optional*, defaults to 3.0):
|
692
|
+
The scale factor for the perturbed attention guidance. If it is set to 0.0, the perturbed attention
|
693
|
+
guidance will not be used.
|
694
|
+
pag_adaptive_scale (`float`, *optional*, defaults to 0.0):
|
695
|
+
The adaptive scale factor for the perturbed attention guidance. If it is set to 0.0, `pag_scale` is
|
696
|
+
used.
|
697
|
+
|
698
|
+
Examples:
|
699
|
+
|
700
|
+
Returns:
|
701
|
+
[`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
|
702
|
+
If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,
|
703
|
+
otherwise a `tuple` is returned where the first element is a list with the generated images and the
|
704
|
+
second element is a list of `bool`s indicating whether the corresponding generated image contains
|
705
|
+
"not-safe-for-work" (nsfw) content.
|
706
|
+
"""
|
707
|
+
|
708
|
+
if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
|
709
|
+
callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
|
710
|
+
|
711
|
+
# 0. Default height and width
|
712
|
+
height = height or self.default_sample_size * self.vae_scale_factor
|
713
|
+
width = width or self.default_sample_size * self.vae_scale_factor
|
714
|
+
height = int((height // 16) * 16)
|
715
|
+
width = int((width // 16) * 16)
|
716
|
+
|
717
|
+
if use_resolution_binning and (height, width) not in SUPPORTED_SHAPE:
|
718
|
+
width, height = map_to_standard_shapes(width, height)
|
719
|
+
height = int(height)
|
720
|
+
width = int(width)
|
721
|
+
logger.warning(f"Reshaped to (height, width)=({height}, {width}), Supported shapes are {SUPPORTED_SHAPE}")
|
722
|
+
|
723
|
+
# 1. Check inputs. Raise error if not correct
|
724
|
+
self.check_inputs(
|
725
|
+
prompt,
|
726
|
+
height,
|
727
|
+
width,
|
728
|
+
negative_prompt,
|
729
|
+
prompt_embeds,
|
730
|
+
negative_prompt_embeds,
|
731
|
+
prompt_attention_mask,
|
732
|
+
negative_prompt_attention_mask,
|
733
|
+
prompt_embeds_2,
|
734
|
+
negative_prompt_embeds_2,
|
735
|
+
prompt_attention_mask_2,
|
736
|
+
negative_prompt_attention_mask_2,
|
737
|
+
callback_on_step_end_tensor_inputs,
|
738
|
+
)
|
739
|
+
self._guidance_scale = guidance_scale
|
740
|
+
self._guidance_rescale = guidance_rescale
|
741
|
+
self._interrupt = False
|
742
|
+
self._pag_scale = pag_scale
|
743
|
+
self._pag_adaptive_scale = pag_adaptive_scale
|
744
|
+
|
745
|
+
# 2. Define call parameters
|
746
|
+
if prompt is not None and isinstance(prompt, str):
|
747
|
+
batch_size = 1
|
748
|
+
elif prompt is not None and isinstance(prompt, list):
|
749
|
+
batch_size = len(prompt)
|
750
|
+
else:
|
751
|
+
batch_size = prompt_embeds.shape[0]
|
752
|
+
|
753
|
+
device = self._execution_device
|
754
|
+
|
755
|
+
# 3. Encode input prompt
|
756
|
+
(
|
757
|
+
prompt_embeds,
|
758
|
+
negative_prompt_embeds,
|
759
|
+
prompt_attention_mask,
|
760
|
+
negative_prompt_attention_mask,
|
761
|
+
) = self.encode_prompt(
|
762
|
+
prompt=prompt,
|
763
|
+
device=device,
|
764
|
+
dtype=self.transformer.dtype,
|
765
|
+
num_images_per_prompt=num_images_per_prompt,
|
766
|
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
767
|
+
negative_prompt=negative_prompt,
|
768
|
+
prompt_embeds=prompt_embeds,
|
769
|
+
negative_prompt_embeds=negative_prompt_embeds,
|
770
|
+
prompt_attention_mask=prompt_attention_mask,
|
771
|
+
negative_prompt_attention_mask=negative_prompt_attention_mask,
|
772
|
+
max_sequence_length=77,
|
773
|
+
text_encoder_index=0,
|
774
|
+
)
|
775
|
+
(
|
776
|
+
prompt_embeds_2,
|
777
|
+
negative_prompt_embeds_2,
|
778
|
+
prompt_attention_mask_2,
|
779
|
+
negative_prompt_attention_mask_2,
|
780
|
+
) = self.encode_prompt(
|
781
|
+
prompt=prompt,
|
782
|
+
device=device,
|
783
|
+
dtype=self.transformer.dtype,
|
784
|
+
num_images_per_prompt=num_images_per_prompt,
|
785
|
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
786
|
+
negative_prompt=negative_prompt,
|
787
|
+
prompt_embeds=prompt_embeds_2,
|
788
|
+
negative_prompt_embeds=negative_prompt_embeds_2,
|
789
|
+
prompt_attention_mask=prompt_attention_mask_2,
|
790
|
+
negative_prompt_attention_mask=negative_prompt_attention_mask_2,
|
791
|
+
max_sequence_length=256,
|
792
|
+
text_encoder_index=1,
|
793
|
+
)
|
794
|
+
|
795
|
+
# 4. Prepare timesteps
|
796
|
+
self.scheduler.set_timesteps(num_inference_steps, device=device)
|
797
|
+
timesteps = self.scheduler.timesteps
|
798
|
+
|
799
|
+
# 5. Prepare latent variables
|
800
|
+
num_channels_latents = self.transformer.config.in_channels
|
801
|
+
latents = self.prepare_latents(
|
802
|
+
batch_size * num_images_per_prompt,
|
803
|
+
num_channels_latents,
|
804
|
+
height,
|
805
|
+
width,
|
806
|
+
prompt_embeds.dtype,
|
807
|
+
device,
|
808
|
+
generator,
|
809
|
+
latents,
|
810
|
+
)
|
811
|
+
|
812
|
+
# 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
|
813
|
+
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
|
814
|
+
|
815
|
+
# 7. Create image_rotary_emb, style embedding & time ids
|
816
|
+
grid_height = height // 8 // self.transformer.config.patch_size
|
817
|
+
grid_width = width // 8 // self.transformer.config.patch_size
|
818
|
+
base_size = 512 // 8 // self.transformer.config.patch_size
|
819
|
+
grid_crops_coords = get_resize_crop_region_for_grid((grid_height, grid_width), base_size)
|
820
|
+
image_rotary_emb = get_2d_rotary_pos_embed(
|
821
|
+
self.transformer.inner_dim // self.transformer.num_heads,
|
822
|
+
grid_crops_coords,
|
823
|
+
(grid_height, grid_width),
|
824
|
+
device=device,
|
825
|
+
output_type="pt",
|
826
|
+
)
|
827
|
+
|
828
|
+
style = torch.tensor([0], device=device)
|
829
|
+
|
830
|
+
target_size = target_size or (height, width)
|
831
|
+
add_time_ids = list(original_size + target_size + crops_coords_top_left)
|
832
|
+
add_time_ids = torch.tensor([add_time_ids], dtype=prompt_embeds.dtype)
|
833
|
+
|
834
|
+
# For classifier free guidance, we need to do two forward passes.
|
835
|
+
# Here we concatenate the unconditional and text embeddings into a single batch
|
836
|
+
# to avoid doing two forward passes
|
837
|
+
if self.do_perturbed_attention_guidance:
|
838
|
+
prompt_embeds = self._prepare_perturbed_attention_guidance(
|
839
|
+
prompt_embeds, negative_prompt_embeds, self.do_classifier_free_guidance
|
840
|
+
)
|
841
|
+
prompt_attention_mask = self._prepare_perturbed_attention_guidance(
|
842
|
+
prompt_attention_mask, negative_prompt_attention_mask, self.do_classifier_free_guidance
|
843
|
+
)
|
844
|
+
prompt_embeds_2 = self._prepare_perturbed_attention_guidance(
|
845
|
+
prompt_embeds_2, negative_prompt_embeds_2, self.do_classifier_free_guidance
|
846
|
+
)
|
847
|
+
prompt_attention_mask_2 = self._prepare_perturbed_attention_guidance(
|
848
|
+
prompt_attention_mask_2, negative_prompt_attention_mask_2, self.do_classifier_free_guidance
|
849
|
+
)
|
850
|
+
add_time_ids = torch.cat([add_time_ids] * 3, dim=0)
|
851
|
+
style = torch.cat([style] * 3, dim=0)
|
852
|
+
elif self.do_classifier_free_guidance:
|
853
|
+
prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
|
854
|
+
prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask])
|
855
|
+
prompt_embeds_2 = torch.cat([negative_prompt_embeds_2, prompt_embeds_2])
|
856
|
+
prompt_attention_mask_2 = torch.cat([negative_prompt_attention_mask_2, prompt_attention_mask_2])
|
857
|
+
add_time_ids = torch.cat([add_time_ids] * 2, dim=0)
|
858
|
+
style = torch.cat([style] * 2, dim=0)
|
859
|
+
|
860
|
+
prompt_embeds = prompt_embeds.to(device=device)
|
861
|
+
prompt_attention_mask = prompt_attention_mask.to(device=device)
|
862
|
+
prompt_embeds_2 = prompt_embeds_2.to(device=device)
|
863
|
+
prompt_attention_mask_2 = prompt_attention_mask_2.to(device=device)
|
864
|
+
add_time_ids = add_time_ids.to(dtype=prompt_embeds.dtype, device=device).repeat(
|
865
|
+
batch_size * num_images_per_prompt, 1
|
866
|
+
)
|
867
|
+
style = style.to(device=device).repeat(batch_size * num_images_per_prompt)
|
868
|
+
|
869
|
+
# 8. Denoising loop
|
870
|
+
num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
|
871
|
+
self._num_timesteps = len(timesteps)
|
872
|
+
|
873
|
+
if self.do_perturbed_attention_guidance:
|
874
|
+
original_attn_proc = self.transformer.attn_processors
|
875
|
+
self._set_pag_attn_processor(
|
876
|
+
pag_applied_layers=self.pag_applied_layers,
|
877
|
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
878
|
+
)
|
879
|
+
|
880
|
+
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
881
|
+
for i, t in enumerate(timesteps):
|
882
|
+
if self.interrupt:
|
883
|
+
continue
|
884
|
+
|
885
|
+
# expand the latents if we are doing classifier free guidance
|
886
|
+
latent_model_input = torch.cat([latents] * (prompt_embeds.shape[0] // latents.shape[0]))
|
887
|
+
latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
|
888
|
+
|
889
|
+
# expand scalar t to 1-D tensor to match the 1st dim of latent_model_input
|
890
|
+
t_expand = torch.tensor([t] * latent_model_input.shape[0], device=device).to(
|
891
|
+
dtype=latent_model_input.dtype
|
892
|
+
)
|
893
|
+
|
894
|
+
# predict the noise residual
|
895
|
+
noise_pred = self.transformer(
|
896
|
+
latent_model_input,
|
897
|
+
t_expand,
|
898
|
+
encoder_hidden_states=prompt_embeds,
|
899
|
+
text_embedding_mask=prompt_attention_mask,
|
900
|
+
encoder_hidden_states_t5=prompt_embeds_2,
|
901
|
+
text_embedding_mask_t5=prompt_attention_mask_2,
|
902
|
+
image_meta_size=add_time_ids,
|
903
|
+
style=style,
|
904
|
+
image_rotary_emb=image_rotary_emb,
|
905
|
+
return_dict=False,
|
906
|
+
)[0]
|
907
|
+
|
908
|
+
noise_pred, _ = noise_pred.chunk(2, dim=1)
|
909
|
+
|
910
|
+
# perform guidance
|
911
|
+
if self.do_perturbed_attention_guidance:
|
912
|
+
noise_pred, noise_pred_text = self._apply_perturbed_attention_guidance(
|
913
|
+
noise_pred, self.do_classifier_free_guidance, self.guidance_scale, t, True
|
914
|
+
)
|
915
|
+
elif self.do_classifier_free_guidance:
|
916
|
+
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
917
|
+
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
918
|
+
|
919
|
+
if self.do_classifier_free_guidance and guidance_rescale > 0.0:
|
920
|
+
# Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
|
921
|
+
noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
|
922
|
+
|
923
|
+
# compute the previous noisy sample x_t -> x_t-1
|
924
|
+
latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
|
925
|
+
|
926
|
+
if callback_on_step_end is not None:
|
927
|
+
callback_kwargs = {}
|
928
|
+
for k in callback_on_step_end_tensor_inputs:
|
929
|
+
callback_kwargs[k] = locals()[k]
|
930
|
+
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
931
|
+
|
932
|
+
latents = callback_outputs.pop("latents", latents)
|
933
|
+
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
934
|
+
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
|
935
|
+
prompt_embeds_2 = callback_outputs.pop("prompt_embeds_2", prompt_embeds_2)
|
936
|
+
negative_prompt_embeds_2 = callback_outputs.pop(
|
937
|
+
"negative_prompt_embeds_2", negative_prompt_embeds_2
|
938
|
+
)
|
939
|
+
|
940
|
+
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
941
|
+
progress_bar.update()
|
942
|
+
|
943
|
+
if XLA_AVAILABLE:
|
944
|
+
xm.mark_step()
|
945
|
+
|
946
|
+
if not output_type == "latent":
|
947
|
+
image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
|
948
|
+
image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
|
949
|
+
else:
|
950
|
+
image = latents
|
951
|
+
has_nsfw_concept = None
|
952
|
+
|
953
|
+
if has_nsfw_concept is None:
|
954
|
+
do_denormalize = [True] * image.shape[0]
|
955
|
+
else:
|
956
|
+
do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]
|
957
|
+
|
958
|
+
image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
|
959
|
+
|
960
|
+
# 9. Offload all models
|
961
|
+
self.maybe_free_model_hooks()
|
962
|
+
|
963
|
+
if self.do_perturbed_attention_guidance:
|
964
|
+
self.transformer.set_attn_processor(original_attn_proc)
|
965
|
+
|
966
|
+
if not return_dict:
|
967
|
+
return (image, has_nsfw_concept)
|
968
|
+
|
969
|
+
return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)
|