diffusers 0.26.2__py3-none-any.whl → 0.27.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- diffusers/__init__.py +20 -1
- diffusers/commands/__init__.py +1 -1
- diffusers/commands/diffusers_cli.py +1 -1
- diffusers/commands/env.py +1 -1
- diffusers/commands/fp16_safetensors.py +1 -1
- diffusers/configuration_utils.py +7 -3
- diffusers/dependency_versions_check.py +1 -1
- diffusers/dependency_versions_table.py +2 -2
- diffusers/experimental/rl/value_guided_sampling.py +1 -1
- diffusers/image_processor.py +110 -4
- diffusers/loaders/autoencoder.py +28 -8
- diffusers/loaders/controlnet.py +17 -8
- diffusers/loaders/ip_adapter.py +86 -23
- diffusers/loaders/lora.py +105 -310
- diffusers/loaders/lora_conversion_utils.py +1 -1
- diffusers/loaders/peft.py +1 -1
- diffusers/loaders/single_file.py +51 -12
- diffusers/loaders/single_file_utils.py +278 -49
- diffusers/loaders/textual_inversion.py +23 -4
- diffusers/loaders/unet.py +195 -41
- diffusers/loaders/utils.py +1 -1
- diffusers/models/__init__.py +3 -1
- diffusers/models/activations.py +9 -9
- diffusers/models/attention.py +26 -36
- diffusers/models/attention_flax.py +1 -1
- diffusers/models/attention_processor.py +171 -114
- diffusers/models/autoencoders/autoencoder_asym_kl.py +1 -1
- diffusers/models/autoencoders/autoencoder_kl.py +3 -1
- diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +1 -1
- diffusers/models/autoencoders/autoencoder_tiny.py +4 -2
- diffusers/models/autoencoders/consistency_decoder_vae.py +1 -1
- diffusers/models/autoencoders/vae.py +1 -1
- diffusers/models/controlnet.py +1 -1
- diffusers/models/controlnet_flax.py +1 -1
- diffusers/models/downsampling.py +8 -12
- diffusers/models/dual_transformer_2d.py +1 -1
- diffusers/models/embeddings.py +3 -4
- diffusers/models/embeddings_flax.py +1 -1
- diffusers/models/lora.py +33 -10
- diffusers/models/modeling_flax_pytorch_utils.py +1 -1
- diffusers/models/modeling_flax_utils.py +1 -1
- diffusers/models/modeling_pytorch_flax_utils.py +1 -1
- diffusers/models/modeling_utils.py +4 -6
- diffusers/models/normalization.py +1 -1
- diffusers/models/resnet.py +31 -58
- diffusers/models/resnet_flax.py +1 -1
- diffusers/models/t5_film_transformer.py +1 -1
- diffusers/models/transformer_2d.py +1 -1
- diffusers/models/transformer_temporal.py +1 -1
- diffusers/models/transformers/dual_transformer_2d.py +1 -1
- diffusers/models/transformers/t5_film_transformer.py +1 -1
- diffusers/models/transformers/transformer_2d.py +29 -31
- diffusers/models/transformers/transformer_temporal.py +1 -1
- diffusers/models/unet_1d.py +1 -1
- diffusers/models/unet_1d_blocks.py +1 -1
- diffusers/models/unet_2d.py +1 -1
- diffusers/models/unet_2d_blocks.py +1 -1
- diffusers/models/unet_2d_condition.py +1 -1
- diffusers/models/unets/__init__.py +1 -0
- diffusers/models/unets/unet_1d.py +1 -1
- diffusers/models/unets/unet_1d_blocks.py +1 -1
- diffusers/models/unets/unet_2d.py +4 -4
- diffusers/models/unets/unet_2d_blocks.py +238 -98
- diffusers/models/unets/unet_2d_blocks_flax.py +1 -1
- diffusers/models/unets/unet_2d_condition.py +420 -323
- diffusers/models/unets/unet_2d_condition_flax.py +21 -12
- diffusers/models/unets/unet_3d_blocks.py +50 -40
- diffusers/models/unets/unet_3d_condition.py +47 -8
- diffusers/models/unets/unet_i2vgen_xl.py +75 -30
- diffusers/models/unets/unet_kandinsky3.py +1 -1
- diffusers/models/unets/unet_motion_model.py +48 -8
- diffusers/models/unets/unet_spatio_temporal_condition.py +1 -1
- diffusers/models/unets/unet_stable_cascade.py +610 -0
- diffusers/models/unets/uvit_2d.py +1 -1
- diffusers/models/upsampling.py +10 -16
- diffusers/models/vae_flax.py +1 -1
- diffusers/models/vq_model.py +1 -1
- diffusers/optimization.py +1 -1
- diffusers/pipelines/__init__.py +26 -0
- diffusers/pipelines/amused/pipeline_amused.py +1 -1
- diffusers/pipelines/amused/pipeline_amused_img2img.py +1 -1
- diffusers/pipelines/amused/pipeline_amused_inpaint.py +1 -1
- diffusers/pipelines/animatediff/pipeline_animatediff.py +162 -417
- diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +165 -137
- diffusers/pipelines/animatediff/pipeline_output.py +7 -6
- diffusers/pipelines/audioldm/pipeline_audioldm.py +3 -19
- diffusers/pipelines/audioldm2/modeling_audioldm2.py +1 -1
- diffusers/pipelines/audioldm2/pipeline_audioldm2.py +3 -3
- diffusers/pipelines/auto_pipeline.py +7 -16
- diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
- diffusers/pipelines/blip_diffusion/modeling_blip2.py +1 -1
- diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +2 -2
- diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +2 -2
- diffusers/pipelines/consistency_models/pipeline_consistency_models.py +1 -1
- diffusers/pipelines/controlnet/pipeline_controlnet.py +90 -90
- diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +2 -2
- diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +98 -90
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +92 -90
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +145 -70
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +126 -89
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +108 -96
- diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +2 -2
- diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +1 -1
- diffusers/pipelines/ddim/pipeline_ddim.py +1 -1
- diffusers/pipelines/ddpm/pipeline_ddpm.py +1 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if.py +4 -4
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +4 -4
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +5 -5
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +4 -4
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +5 -5
- diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +5 -5
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +10 -120
- diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +10 -91
- diffusers/pipelines/deprecated/audio_diffusion/mel.py +1 -1
- diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py +1 -1
- diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +1 -1
- diffusers/pipelines/deprecated/pndm/pipeline_pndm.py +1 -1
- diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +1 -1
- diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py +1 -1
- diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py +1 -1
- diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py +1 -1
- diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py +1 -1
- diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +1 -1
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +5 -4
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +5 -4
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +7 -22
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +5 -39
- diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +5 -5
- diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py +1 -1
- diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +31 -22
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +1 -1
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +1 -1
- diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +1 -2
- diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +1 -1
- diffusers/pipelines/dit/pipeline_dit.py +1 -1
- diffusers/pipelines/free_init_utils.py +184 -0
- diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +22 -104
- diffusers/pipelines/kandinsky/pipeline_kandinsky.py +1 -1
- diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +1 -1
- diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +1 -1
- diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +2 -2
- diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +2 -2
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +104 -93
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +112 -74
- diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +1 -1
- diffusers/pipelines/ledits_pp/__init__.py +55 -0
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +1505 -0
- diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +1797 -0
- diffusers/pipelines/ledits_pp/pipeline_output.py +43 -0
- diffusers/pipelines/musicldm/pipeline_musicldm.py +3 -19
- diffusers/pipelines/onnx_utils.py +1 -1
- diffusers/pipelines/paint_by_example/image_encoder.py +1 -1
- diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +3 -3
- diffusers/pipelines/pia/pipeline_pia.py +168 -327
- diffusers/pipelines/pipeline_flax_utils.py +1 -1
- diffusers/pipelines/pipeline_loading_utils.py +508 -0
- diffusers/pipelines/pipeline_utils.py +188 -534
- diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +56 -10
- diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +3 -3
- diffusers/pipelines/shap_e/camera.py +1 -1
- diffusers/pipelines/shap_e/pipeline_shap_e.py +1 -1
- diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +1 -1
- diffusers/pipelines/shap_e/renderer.py +1 -1
- diffusers/pipelines/stable_cascade/__init__.py +50 -0
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +482 -0
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +311 -0
- diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +638 -0
- diffusers/pipelines/stable_diffusion/clip_image_project_model.py +1 -1
- diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +4 -1
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py +2 -2
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +1 -1
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +90 -146
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +5 -4
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +4 -32
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +92 -119
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +92 -119
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +13 -59
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +3 -31
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +5 -33
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +5 -21
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +7 -21
- diffusers/pipelines/stable_diffusion/safety_checker.py +1 -1
- diffusers/pipelines/stable_diffusion/safety_checker_flax.py +1 -1
- diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py +1 -1
- diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +5 -21
- diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +9 -38
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +5 -34
- diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +6 -35
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +7 -6
- diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +4 -124
- diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +282 -80
- diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +94 -46
- diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +3 -3
- diffusers/pipelines/stable_diffusion_safe/safety_checker.py +1 -1
- diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +6 -22
- diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py +1 -1
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +96 -148
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +98 -154
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +98 -153
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +25 -87
- diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +89 -80
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +5 -49
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +80 -88
- diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +8 -6
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +15 -86
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +20 -93
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +5 -5
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +3 -19
- diffusers/pipelines/unclip/pipeline_unclip.py +1 -1
- diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +1 -1
- diffusers/pipelines/unclip/text_proj.py +1 -1
- diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +35 -35
- diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +1 -1
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +4 -21
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py +2 -2
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +4 -5
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +8 -8
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +1 -1
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +2 -2
- diffusers/schedulers/__init__.py +7 -1
- diffusers/schedulers/deprecated/scheduling_karras_ve.py +1 -1
- diffusers/schedulers/deprecated/scheduling_sde_vp.py +1 -1
- diffusers/schedulers/scheduling_consistency_models.py +42 -19
- diffusers/schedulers/scheduling_ddim.py +2 -4
- diffusers/schedulers/scheduling_ddim_flax.py +13 -5
- diffusers/schedulers/scheduling_ddim_inverse.py +2 -4
- diffusers/schedulers/scheduling_ddim_parallel.py +2 -4
- diffusers/schedulers/scheduling_ddpm.py +2 -4
- diffusers/schedulers/scheduling_ddpm_flax.py +1 -1
- diffusers/schedulers/scheduling_ddpm_parallel.py +2 -4
- diffusers/schedulers/scheduling_ddpm_wuerstchen.py +1 -1
- diffusers/schedulers/scheduling_deis_multistep.py +46 -19
- diffusers/schedulers/scheduling_dpmsolver_multistep.py +107 -21
- diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +1 -1
- diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +9 -7
- diffusers/schedulers/scheduling_dpmsolver_sde.py +35 -35
- diffusers/schedulers/scheduling_dpmsolver_singlestep.py +52 -21
- diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +683 -0
- diffusers/schedulers/scheduling_edm_euler.py +381 -0
- diffusers/schedulers/scheduling_euler_ancestral_discrete.py +43 -15
- diffusers/schedulers/scheduling_euler_discrete.py +42 -17
- diffusers/schedulers/scheduling_euler_discrete_flax.py +1 -1
- diffusers/schedulers/scheduling_heun_discrete.py +35 -35
- diffusers/schedulers/scheduling_ipndm.py +37 -11
- diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +44 -44
- diffusers/schedulers/scheduling_k_dpm_2_discrete.py +44 -44
- diffusers/schedulers/scheduling_karras_ve_flax.py +1 -1
- diffusers/schedulers/scheduling_lcm.py +38 -14
- diffusers/schedulers/scheduling_lms_discrete.py +43 -15
- diffusers/schedulers/scheduling_lms_discrete_flax.py +1 -1
- diffusers/schedulers/scheduling_pndm.py +2 -4
- diffusers/schedulers/scheduling_pndm_flax.py +2 -4
- diffusers/schedulers/scheduling_repaint.py +1 -1
- diffusers/schedulers/scheduling_sasolver.py +41 -9
- diffusers/schedulers/scheduling_sde_ve.py +1 -1
- diffusers/schedulers/scheduling_sde_ve_flax.py +1 -1
- diffusers/schedulers/scheduling_tcd.py +686 -0
- diffusers/schedulers/scheduling_unclip.py +1 -1
- diffusers/schedulers/scheduling_unipc_multistep.py +46 -19
- diffusers/schedulers/scheduling_utils.py +2 -1
- diffusers/schedulers/scheduling_utils_flax.py +1 -1
- diffusers/schedulers/scheduling_vq_diffusion.py +1 -1
- diffusers/training_utils.py +9 -2
- diffusers/utils/__init__.py +2 -1
- diffusers/utils/accelerate_utils.py +1 -1
- diffusers/utils/constants.py +1 -1
- diffusers/utils/doc_utils.py +1 -1
- diffusers/utils/dummy_pt_objects.py +60 -0
- diffusers/utils/dummy_torch_and_transformers_objects.py +75 -0
- diffusers/utils/dynamic_modules_utils.py +1 -1
- diffusers/utils/export_utils.py +3 -3
- diffusers/utils/hub_utils.py +60 -16
- diffusers/utils/import_utils.py +15 -1
- diffusers/utils/loading_utils.py +2 -0
- diffusers/utils/logging.py +1 -1
- diffusers/utils/model_card_template.md +24 -0
- diffusers/utils/outputs.py +14 -7
- diffusers/utils/peft_utils.py +1 -1
- diffusers/utils/state_dict_utils.py +1 -1
- diffusers/utils/testing_utils.py +2 -0
- diffusers/utils/torch_utils.py +1 -1
- {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/METADATA +5 -5
- diffusers-0.27.0.dist-info/RECORD +399 -0
- diffusers-0.26.2.dist-info/RECORD +0 -384
- {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/LICENSE +0 -0
- {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/WHEEL +0 -0
- {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/entry_points.txt +0 -0
- {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright
|
1
|
+
# Copyright 2024 TencentARC and The HuggingFace Team. All rights reserved.
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -51,7 +51,7 @@ from ...utils import (
|
|
51
51
|
unscale_lora_layers,
|
52
52
|
)
|
53
53
|
from ...utils.torch_utils import randn_tensor
|
54
|
-
from ..pipeline_utils import DiffusionPipeline
|
54
|
+
from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
|
55
55
|
from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
|
56
56
|
|
57
57
|
|
@@ -181,6 +181,7 @@ def retrieve_timesteps(
|
|
181
181
|
|
182
182
|
class StableDiffusionXLAdapterPipeline(
|
183
183
|
DiffusionPipeline,
|
184
|
+
StableDiffusionMixin,
|
184
185
|
TextualInversionLoaderMixin,
|
185
186
|
StableDiffusionXLLoraLoaderMixin,
|
186
187
|
IPAdapterMixin,
|
@@ -270,39 +271,6 @@ class StableDiffusionXLAdapterPipeline(
|
|
270
271
|
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
|
271
272
|
self.default_sample_size = self.unet.config.sample_size
|
272
273
|
|
273
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing
|
274
|
-
def enable_vae_slicing(self):
|
275
|
-
r"""
|
276
|
-
Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
|
277
|
-
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
|
278
|
-
"""
|
279
|
-
self.vae.enable_slicing()
|
280
|
-
|
281
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing
|
282
|
-
def disable_vae_slicing(self):
|
283
|
-
r"""
|
284
|
-
Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
|
285
|
-
computing decoding in one step.
|
286
|
-
"""
|
287
|
-
self.vae.disable_slicing()
|
288
|
-
|
289
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling
|
290
|
-
def enable_vae_tiling(self):
|
291
|
-
r"""
|
292
|
-
Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
|
293
|
-
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
|
294
|
-
processing larger images.
|
295
|
-
"""
|
296
|
-
self.vae.enable_tiling()
|
297
|
-
|
298
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling
|
299
|
-
def disable_vae_tiling(self):
|
300
|
-
r"""
|
301
|
-
Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
|
302
|
-
computing decoding in one step.
|
303
|
-
"""
|
304
|
-
self.vae.disable_tiling()
|
305
|
-
|
306
274
|
# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
|
307
275
|
def encode_prompt(
|
308
276
|
self,
|
@@ -399,7 +367,7 @@ class StableDiffusionXLAdapterPipeline(
|
|
399
367
|
prompt_2 = prompt_2 or prompt
|
400
368
|
prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
|
401
369
|
|
402
|
-
# textual inversion:
|
370
|
+
# textual inversion: process multi-vector tokens if necessary
|
403
371
|
prompt_embeds_list = []
|
404
372
|
prompts = [prompt, prompt_2]
|
405
373
|
for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
|
@@ -564,31 +532,54 @@ class StableDiffusionXLAdapterPipeline(
|
|
564
532
|
return image_embeds, uncond_image_embeds
|
565
533
|
|
566
534
|
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
|
567
|
-
def prepare_ip_adapter_image_embeds(
|
568
|
-
|
569
|
-
|
535
|
+
def prepare_ip_adapter_image_embeds(
|
536
|
+
self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
|
537
|
+
):
|
538
|
+
if ip_adapter_image_embeds is None:
|
539
|
+
if not isinstance(ip_adapter_image, list):
|
540
|
+
ip_adapter_image = [ip_adapter_image]
|
570
541
|
|
571
|
-
|
572
|
-
|
573
|
-
|
574
|
-
|
542
|
+
if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
|
543
|
+
raise ValueError(
|
544
|
+
f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
|
545
|
+
)
|
575
546
|
|
576
|
-
|
577
|
-
|
578
|
-
|
579
|
-
|
580
|
-
|
581
|
-
|
582
|
-
|
583
|
-
|
584
|
-
|
585
|
-
|
547
|
+
image_embeds = []
|
548
|
+
for single_ip_adapter_image, image_proj_layer in zip(
|
549
|
+
ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
|
550
|
+
):
|
551
|
+
output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
|
552
|
+
single_image_embeds, single_negative_image_embeds = self.encode_image(
|
553
|
+
single_ip_adapter_image, device, 1, output_hidden_state
|
554
|
+
)
|
555
|
+
single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0)
|
556
|
+
single_negative_image_embeds = torch.stack(
|
557
|
+
[single_negative_image_embeds] * num_images_per_prompt, dim=0
|
558
|
+
)
|
586
559
|
|
587
|
-
|
588
|
-
|
589
|
-
|
560
|
+
if do_classifier_free_guidance:
|
561
|
+
single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
|
562
|
+
single_image_embeds = single_image_embeds.to(device)
|
590
563
|
|
591
|
-
|
564
|
+
image_embeds.append(single_image_embeds)
|
565
|
+
else:
|
566
|
+
repeat_dims = [1]
|
567
|
+
image_embeds = []
|
568
|
+
for single_image_embeds in ip_adapter_image_embeds:
|
569
|
+
if do_classifier_free_guidance:
|
570
|
+
single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
|
571
|
+
single_image_embeds = single_image_embeds.repeat(
|
572
|
+
num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
|
573
|
+
)
|
574
|
+
single_negative_image_embeds = single_negative_image_embeds.repeat(
|
575
|
+
num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:]))
|
576
|
+
)
|
577
|
+
single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
|
578
|
+
else:
|
579
|
+
single_image_embeds = single_image_embeds.repeat(
|
580
|
+
num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
|
581
|
+
)
|
582
|
+
image_embeds.append(single_image_embeds)
|
592
583
|
|
593
584
|
return image_embeds
|
594
585
|
|
@@ -624,6 +615,8 @@ class StableDiffusionXLAdapterPipeline(
|
|
624
615
|
negative_prompt_embeds=None,
|
625
616
|
pooled_prompt_embeds=None,
|
626
617
|
negative_pooled_prompt_embeds=None,
|
618
|
+
ip_adapter_image=None,
|
619
|
+
ip_adapter_image_embeds=None,
|
627
620
|
callback_on_step_end_tensor_inputs=None,
|
628
621
|
):
|
629
622
|
if height % 8 != 0 or width % 8 != 0:
|
@@ -690,6 +683,21 @@ class StableDiffusionXLAdapterPipeline(
|
|
690
683
|
"If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
|
691
684
|
)
|
692
685
|
|
686
|
+
if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
|
687
|
+
raise ValueError(
|
688
|
+
"Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
|
689
|
+
)
|
690
|
+
|
691
|
+
if ip_adapter_image_embeds is not None:
|
692
|
+
if not isinstance(ip_adapter_image_embeds, list):
|
693
|
+
raise ValueError(
|
694
|
+
f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
|
695
|
+
)
|
696
|
+
elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
|
697
|
+
raise ValueError(
|
698
|
+
f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
|
699
|
+
)
|
700
|
+
|
693
701
|
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
|
694
702
|
def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
|
695
703
|
shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)
|
@@ -775,34 +783,6 @@ class StableDiffusionXLAdapterPipeline(
|
|
775
783
|
|
776
784
|
return height, width
|
777
785
|
|
778
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu
|
779
|
-
def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):
|
780
|
-
r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.
|
781
|
-
|
782
|
-
The suffixes after the scaling factors represent the stages where they are being applied.
|
783
|
-
|
784
|
-
Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values
|
785
|
-
that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
|
786
|
-
|
787
|
-
Args:
|
788
|
-
s1 (`float`):
|
789
|
-
Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
|
790
|
-
mitigate "oversmoothing effect" in the enhanced denoising process.
|
791
|
-
s2 (`float`):
|
792
|
-
Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
|
793
|
-
mitigate "oversmoothing effect" in the enhanced denoising process.
|
794
|
-
b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
|
795
|
-
b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
|
796
|
-
"""
|
797
|
-
if not hasattr(self, "unet"):
|
798
|
-
raise ValueError("The pipeline must have `unet` for using FreeU.")
|
799
|
-
self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)
|
800
|
-
|
801
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu
|
802
|
-
def disable_freeu(self):
|
803
|
-
"""Disables the FreeU mechanism if enabled."""
|
804
|
-
self.unet.disable_freeu()
|
805
|
-
|
806
786
|
# Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
|
807
787
|
def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):
|
808
788
|
"""
|
@@ -867,6 +847,7 @@ class StableDiffusionXLAdapterPipeline(
|
|
867
847
|
pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
|
868
848
|
negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
|
869
849
|
ip_adapter_image: Optional[PipelineImageInput] = None,
|
850
|
+
ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None,
|
870
851
|
output_type: Optional[str] = "pil",
|
871
852
|
return_dict: bool = True,
|
872
853
|
callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
|
@@ -959,6 +940,11 @@ class StableDiffusionXLAdapterPipeline(
|
|
959
940
|
weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
|
960
941
|
input argument.
|
961
942
|
ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
|
943
|
+
ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*):
|
944
|
+
Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters.
|
945
|
+
Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding
|
946
|
+
if `do_classifier_free_guidance` is set to `True`.
|
947
|
+
If not provided, embeddings are computed from the `ip_adapter_image` input argument.
|
962
948
|
output_type (`str`, *optional*, defaults to `"pil"`):
|
963
949
|
The output format of the generate image. Choose between
|
964
950
|
[PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
|
@@ -1060,6 +1046,8 @@ class StableDiffusionXLAdapterPipeline(
|
|
1060
1046
|
negative_prompt_embeds,
|
1061
1047
|
pooled_prompt_embeds,
|
1062
1048
|
negative_pooled_prompt_embeds,
|
1049
|
+
ip_adapter_image,
|
1050
|
+
ip_adapter_image_embeds,
|
1063
1051
|
)
|
1064
1052
|
|
1065
1053
|
self._guidance_scale = guidance_scale
|
@@ -1096,9 +1084,13 @@ class StableDiffusionXLAdapterPipeline(
|
|
1096
1084
|
)
|
1097
1085
|
|
1098
1086
|
# 3.2 Encode ip_adapter_image
|
1099
|
-
if ip_adapter_image is not None:
|
1087
|
+
if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
|
1100
1088
|
image_embeds = self.prepare_ip_adapter_image_embeds(
|
1101
|
-
ip_adapter_image,
|
1089
|
+
ip_adapter_image,
|
1090
|
+
ip_adapter_image_embeds,
|
1091
|
+
device,
|
1092
|
+
batch_size * num_images_per_prompt,
|
1093
|
+
self.do_classifier_free_guidance,
|
1102
1094
|
)
|
1103
1095
|
|
1104
1096
|
# 4. Prepare timesteps
|
@@ -1199,7 +1191,7 @@ class StableDiffusionXLAdapterPipeline(
|
|
1199
1191
|
|
1200
1192
|
added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
|
1201
1193
|
|
1202
|
-
if ip_adapter_image is not None:
|
1194
|
+
if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
|
1203
1195
|
added_cond_kwargs["image_embeds"] = image_embeds
|
1204
1196
|
|
1205
1197
|
# predict the noise residual
|
@@ -2,6 +2,7 @@ from dataclasses import dataclass
|
|
2
2
|
from typing import List, Union
|
3
3
|
|
4
4
|
import numpy as np
|
5
|
+
import PIL
|
5
6
|
import torch
|
6
7
|
|
7
8
|
from ...utils import (
|
@@ -12,12 +13,13 @@ from ...utils import (
|
|
12
13
|
@dataclass
|
13
14
|
class TextToVideoSDPipelineOutput(BaseOutput):
|
14
15
|
"""
|
15
|
-
|
16
|
+
Output class for text-to-video pipelines.
|
16
17
|
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
18
|
+
Args:
|
19
|
+
frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]):
|
20
|
+
List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised
|
21
|
+
PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape
|
22
|
+
`(batch_size, num_frames, channels, height, width)`
|
21
23
|
"""
|
22
24
|
|
23
|
-
frames: Union[
|
25
|
+
frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]]
|
@@ -1,4 +1,4 @@
|
|
1
|
-
# Copyright
|
1
|
+
# Copyright 2024 The HuggingFace Team. All rights reserved.
|
2
2
|
#
|
3
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
4
|
# you may not use this file except in compliance with the License.
|
@@ -33,7 +33,7 @@ from ...utils import (
|
|
33
33
|
unscale_lora_layers,
|
34
34
|
)
|
35
35
|
from ...utils.torch_utils import randn_tensor
|
36
|
-
from ..pipeline_utils import DiffusionPipeline
|
36
|
+
from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
|
37
37
|
from . import TextToVideoSDPipelineOutput
|
38
38
|
|
39
39
|
|
@@ -52,7 +52,7 @@ EXAMPLE_DOC_STRING = """
|
|
52
52
|
>>> pipe.enable_model_cpu_offload()
|
53
53
|
|
54
54
|
>>> prompt = "Spiderman is surfing"
|
55
|
-
>>> video_frames = pipe(prompt).frames
|
55
|
+
>>> video_frames = pipe(prompt).frames[0]
|
56
56
|
>>> video_path = export_to_video(video_frames)
|
57
57
|
>>> video_path
|
58
58
|
```
|
@@ -76,12 +76,12 @@ def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type:
|
|
76
76
|
outputs = torch.stack(outputs)
|
77
77
|
|
78
78
|
elif not output_type == "pil":
|
79
|
-
raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil]")
|
79
|
+
raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']")
|
80
80
|
|
81
81
|
return outputs
|
82
82
|
|
83
83
|
|
84
|
-
class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin):
|
84
|
+
class TextToVideoSDPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin):
|
85
85
|
r"""
|
86
86
|
Pipeline for text-to-video generation.
|
87
87
|
|
@@ -129,39 +129,6 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
129
129
|
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
|
130
130
|
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
|
131
131
|
|
132
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing
|
133
|
-
def enable_vae_slicing(self):
|
134
|
-
r"""
|
135
|
-
Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
|
136
|
-
compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
|
137
|
-
"""
|
138
|
-
self.vae.enable_slicing()
|
139
|
-
|
140
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing
|
141
|
-
def disable_vae_slicing(self):
|
142
|
-
r"""
|
143
|
-
Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
|
144
|
-
computing decoding in one step.
|
145
|
-
"""
|
146
|
-
self.vae.disable_slicing()
|
147
|
-
|
148
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling
|
149
|
-
def enable_vae_tiling(self):
|
150
|
-
r"""
|
151
|
-
Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
|
152
|
-
compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
|
153
|
-
processing larger images.
|
154
|
-
"""
|
155
|
-
self.vae.enable_tiling()
|
156
|
-
|
157
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling
|
158
|
-
def disable_vae_tiling(self):
|
159
|
-
r"""
|
160
|
-
Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
|
161
|
-
computing decoding in one step.
|
162
|
-
"""
|
163
|
-
self.vae.disable_tiling()
|
164
|
-
|
165
132
|
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt
|
166
133
|
def _encode_prompt(
|
167
134
|
self,
|
@@ -256,7 +223,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
256
223
|
batch_size = prompt_embeds.shape[0]
|
257
224
|
|
258
225
|
if prompt_embeds is None:
|
259
|
-
# textual inversion:
|
226
|
+
# textual inversion: process multi-vector tokens if necessary
|
260
227
|
if isinstance(self, TextualInversionLoaderMixin):
|
261
228
|
prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
|
262
229
|
|
@@ -338,7 +305,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
338
305
|
else:
|
339
306
|
uncond_tokens = negative_prompt
|
340
307
|
|
341
|
-
# textual inversion:
|
308
|
+
# textual inversion: process multi-vector tokens if necessary
|
342
309
|
if isinstance(self, TextualInversionLoaderMixin):
|
343
310
|
uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)
|
344
311
|
|
@@ -384,18 +351,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
384
351
|
latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width)
|
385
352
|
|
386
353
|
image = self.vae.decode(latents).sample
|
387
|
-
video = (
|
388
|
-
image[None, :]
|
389
|
-
.reshape(
|
390
|
-
(
|
391
|
-
batch_size,
|
392
|
-
num_frames,
|
393
|
-
-1,
|
394
|
-
)
|
395
|
-
+ image.shape[2:]
|
396
|
-
)
|
397
|
-
.permute(0, 2, 1, 3, 4)
|
398
|
-
)
|
354
|
+
video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4)
|
399
355
|
# we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
|
400
356
|
video = video.float()
|
401
357
|
return video
|
@@ -418,7 +374,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
418
374
|
extra_step_kwargs["generator"] = generator
|
419
375
|
return extra_step_kwargs
|
420
376
|
|
421
|
-
# Copied from diffusers.pipelines.
|
377
|
+
# Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs
|
422
378
|
def check_inputs(
|
423
379
|
self,
|
424
380
|
prompt,
|
@@ -496,34 +452,6 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
496
452
|
latents = latents * self.scheduler.init_noise_sigma
|
497
453
|
return latents
|
498
454
|
|
499
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu
|
500
|
-
def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):
|
501
|
-
r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.
|
502
|
-
|
503
|
-
The suffixes after the scaling factors represent the stages where they are being applied.
|
504
|
-
|
505
|
-
Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values
|
506
|
-
that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
|
507
|
-
|
508
|
-
Args:
|
509
|
-
s1 (`float`):
|
510
|
-
Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
|
511
|
-
mitigate "oversmoothing effect" in the enhanced denoising process.
|
512
|
-
s2 (`float`):
|
513
|
-
Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
|
514
|
-
mitigate "oversmoothing effect" in the enhanced denoising process.
|
515
|
-
b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
|
516
|
-
b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
|
517
|
-
"""
|
518
|
-
if not hasattr(self, "unet"):
|
519
|
-
raise ValueError("The pipeline must have `unet` for using FreeU.")
|
520
|
-
self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)
|
521
|
-
|
522
|
-
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu
|
523
|
-
def disable_freeu(self):
|
524
|
-
"""Disables the FreeU mechanism if enabled."""
|
525
|
-
self.unet.disable_freeu()
|
526
|
-
|
527
455
|
@torch.no_grad()
|
528
456
|
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
529
457
|
def __call__(
|
@@ -719,13 +647,14 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
|
|
719
647
|
step_idx = i // getattr(self.scheduler, "order", 1)
|
720
648
|
callback(step_idx, t, latents)
|
721
649
|
|
650
|
+
# 8. Post processing
|
722
651
|
if output_type == "latent":
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
652
|
+
video = latents
|
653
|
+
else:
|
654
|
+
video_tensor = self.decode_latents(latents)
|
655
|
+
video = tensor2vid(video_tensor, self.image_processor, output_type)
|
727
656
|
|
728
|
-
# Offload all models
|
657
|
+
# 9. Offload all models
|
729
658
|
self.maybe_free_model_hooks()
|
730
659
|
|
731
660
|
if not return_dict:
|