diffusers 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl
Sign up to get free protection for your applications and to get access to all the features.
- diffusers/__init__.py +26 -2
- diffusers/commands/fp16_safetensors.py +10 -11
- diffusers/configuration_utils.py +13 -8
- diffusers/dependency_versions_check.py +0 -1
- diffusers/dependency_versions_table.py +5 -5
- diffusers/experimental/rl/value_guided_sampling.py +1 -1
- diffusers/image_processor.py +463 -51
- diffusers/loaders/__init__.py +82 -0
- diffusers/loaders/ip_adapter.py +159 -0
- diffusers/loaders/lora.py +1553 -0
- diffusers/loaders/lora_conversion_utils.py +284 -0
- diffusers/loaders/single_file.py +637 -0
- diffusers/loaders/textual_inversion.py +455 -0
- diffusers/loaders/unet.py +828 -0
- diffusers/loaders/utils.py +59 -0
- diffusers/models/__init__.py +26 -9
- diffusers/models/activations.py +9 -6
- diffusers/models/attention.py +301 -29
- diffusers/models/attention_flax.py +9 -1
- diffusers/models/attention_processor.py +378 -6
- diffusers/models/autoencoders/__init__.py +5 -0
- diffusers/models/{autoencoder_asym_kl.py → autoencoders/autoencoder_asym_kl.py} +17 -12
- diffusers/models/{autoencoder_kl.py → autoencoders/autoencoder_kl.py} +47 -23
- diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +402 -0
- diffusers/models/{autoencoder_tiny.py → autoencoders/autoencoder_tiny.py} +24 -28
- diffusers/models/{consistency_decoder_vae.py → autoencoders/consistency_decoder_vae.py} +51 -44
- diffusers/models/{vae.py → autoencoders/vae.py} +71 -17
- diffusers/models/controlnet.py +59 -39
- diffusers/models/controlnet_flax.py +19 -18
- diffusers/models/downsampling.py +338 -0
- diffusers/models/embeddings.py +112 -29
- diffusers/models/embeddings_flax.py +2 -0
- diffusers/models/lora.py +131 -1
- diffusers/models/modeling_flax_utils.py +14 -8
- diffusers/models/modeling_outputs.py +17 -0
- diffusers/models/modeling_utils.py +37 -29
- diffusers/models/normalization.py +110 -4
- diffusers/models/resnet.py +299 -652
- diffusers/models/transformer_2d.py +22 -5
- diffusers/models/transformer_temporal.py +183 -1
- diffusers/models/unet_2d_blocks_flax.py +5 -0
- diffusers/models/unet_2d_condition.py +46 -0
- diffusers/models/unet_2d_condition_flax.py +13 -13
- diffusers/models/unet_3d_blocks.py +957 -173
- diffusers/models/unet_3d_condition.py +16 -8
- diffusers/models/unet_kandinsky3.py +535 -0
- diffusers/models/unet_motion_model.py +48 -33
- diffusers/models/unet_spatio_temporal_condition.py +489 -0
- diffusers/models/upsampling.py +454 -0
- diffusers/models/uvit_2d.py +471 -0
- diffusers/models/vae_flax.py +7 -0
- diffusers/models/vq_model.py +12 -3
- diffusers/optimization.py +16 -9
- diffusers/pipelines/__init__.py +137 -76
- diffusers/pipelines/amused/__init__.py +62 -0
- diffusers/pipelines/amused/pipeline_amused.py +328 -0
- diffusers/pipelines/amused/pipeline_amused_img2img.py +347 -0
- diffusers/pipelines/amused/pipeline_amused_inpaint.py +378 -0
- diffusers/pipelines/animatediff/pipeline_animatediff.py +66 -8
- diffusers/pipelines/audioldm/pipeline_audioldm.py +1 -0
- diffusers/pipelines/auto_pipeline.py +23 -13
- diffusers/pipelines/consistency_models/pipeline_consistency_models.py +1 -0
- diffusers/pipelines/controlnet/pipeline_controlnet.py +238 -35
- diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +148 -37
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +155 -41
- diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +123 -43
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +216 -39
- diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +106 -34
- diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +1 -0
- diffusers/pipelines/ddim/pipeline_ddim.py +1 -0
- diffusers/pipelines/ddpm/pipeline_ddpm.py +1 -0
- diffusers/pipelines/deepfloyd_if/pipeline_if.py +13 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +13 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +13 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +13 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +13 -1
- diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +13 -1
- diffusers/pipelines/deprecated/__init__.py +153 -0
- diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/__init__.py +3 -3
- diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_alt_diffusion.py +177 -34
- diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_alt_diffusion_img2img.py +182 -37
- diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_output.py +1 -1
- diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/__init__.py +1 -1
- diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/mel.py +2 -2
- diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/pipeline_audio_diffusion.py +4 -4
- diffusers/pipelines/{latent_diffusion_uncond → deprecated/latent_diffusion_uncond}/__init__.py +1 -1
- diffusers/pipelines/{latent_diffusion_uncond → deprecated/latent_diffusion_uncond}/pipeline_latent_diffusion_uncond.py +4 -4
- diffusers/pipelines/{pndm → deprecated/pndm}/__init__.py +1 -1
- diffusers/pipelines/{pndm → deprecated/pndm}/pipeline_pndm.py +4 -4
- diffusers/pipelines/{repaint → deprecated/repaint}/__init__.py +1 -1
- diffusers/pipelines/{repaint → deprecated/repaint}/pipeline_repaint.py +5 -5
- diffusers/pipelines/{score_sde_ve → deprecated/score_sde_ve}/__init__.py +1 -1
- diffusers/pipelines/{score_sde_ve → deprecated/score_sde_ve}/pipeline_score_sde_ve.py +5 -4
- diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/__init__.py +6 -6
- diffusers/pipelines/{spectrogram_diffusion/continous_encoder.py → deprecated/spectrogram_diffusion/continuous_encoder.py} +2 -2
- diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/midi_utils.py +1 -1
- diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/notes_encoder.py +2 -2
- diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/pipeline_spectrogram_diffusion.py +8 -7
- diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py +55 -0
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_cycle_diffusion.py +34 -13
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_onnx_stable_diffusion_inpaint_legacy.py +7 -6
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_inpaint_legacy.py +12 -11
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_model_editing.py +17 -11
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_paradigms.py +11 -10
- diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_pix2pix_zero.py +14 -13
- diffusers/pipelines/{stochastic_karras_ve → deprecated/stochastic_karras_ve}/__init__.py +1 -1
- diffusers/pipelines/{stochastic_karras_ve → deprecated/stochastic_karras_ve}/pipeline_stochastic_karras_ve.py +4 -4
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/__init__.py +3 -3
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/modeling_text_unet.py +83 -51
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion.py +4 -4
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_dual_guided.py +7 -6
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_image_variation.py +7 -6
- diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_text_to_image.py +7 -6
- diffusers/pipelines/{vq_diffusion → deprecated/vq_diffusion}/__init__.py +3 -3
- diffusers/pipelines/{vq_diffusion → deprecated/vq_diffusion}/pipeline_vq_diffusion.py +5 -5
- diffusers/pipelines/dit/pipeline_dit.py +1 -0
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +3 -3
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +1 -1
- diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +1 -1
- diffusers/pipelines/kandinsky3/__init__.py +49 -0
- diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py +98 -0
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +589 -0
- diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +654 -0
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +111 -11
- diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +102 -9
- diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +1 -0
- diffusers/pipelines/musicldm/pipeline_musicldm.py +1 -1
- diffusers/pipelines/onnx_utils.py +8 -5
- diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +7 -2
- diffusers/pipelines/pipeline_flax_utils.py +11 -8
- diffusers/pipelines/pipeline_utils.py +63 -42
- diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +247 -38
- diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +3 -3
- diffusers/pipelines/stable_diffusion/__init__.py +37 -65
- diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +75 -78
- diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +2 -2
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +2 -4
- diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +1 -0
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +174 -11
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +8 -3
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +1 -0
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +178 -11
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +224 -13
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +74 -20
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +4 -0
- diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +7 -0
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +5 -0
- diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +5 -0
- diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py +48 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_attend_and_excite}/pipeline_stable_diffusion_attend_and_excite.py +6 -2
- diffusers/pipelines/stable_diffusion_diffedit/__init__.py +48 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_diffedit}/pipeline_stable_diffusion_diffedit.py +3 -3
- diffusers/pipelines/stable_diffusion_gligen/__init__.py +50 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_gligen}/pipeline_stable_diffusion_gligen.py +3 -2
- diffusers/pipelines/{stable_diffusion → stable_diffusion_gligen}/pipeline_stable_diffusion_gligen_text_image.py +4 -3
- diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py +60 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_k_diffusion}/pipeline_stable_diffusion_k_diffusion.py +7 -1
- diffusers/pipelines/stable_diffusion_ldm3d/__init__.py +48 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_ldm3d}/pipeline_stable_diffusion_ldm3d.py +51 -7
- diffusers/pipelines/stable_diffusion_panorama/__init__.py +48 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_panorama}/pipeline_stable_diffusion_panorama.py +57 -8
- diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +58 -6
- diffusers/pipelines/stable_diffusion_sag/__init__.py +48 -0
- diffusers/pipelines/{stable_diffusion → stable_diffusion_sag}/pipeline_stable_diffusion_sag.py +68 -10
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +194 -17
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +205 -16
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +206 -17
- diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +23 -17
- diffusers/pipelines/stable_video_diffusion/__init__.py +58 -0
- diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +652 -0
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +108 -12
- diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +115 -14
- diffusers/pipelines/text_to_video_synthesis/__init__.py +2 -0
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +6 -0
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +23 -3
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +334 -10
- diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +1331 -0
- diffusers/pipelines/unclip/pipeline_unclip.py +2 -1
- diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +1 -0
- diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +1 -1
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +14 -4
- diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +9 -5
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +1 -1
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +2 -2
- diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +5 -1
- diffusers/schedulers/__init__.py +4 -4
- diffusers/schedulers/deprecated/__init__.py +50 -0
- diffusers/schedulers/{scheduling_karras_ve.py → deprecated/scheduling_karras_ve.py} +4 -4
- diffusers/schedulers/{scheduling_sde_vp.py → deprecated/scheduling_sde_vp.py} +4 -6
- diffusers/schedulers/scheduling_amused.py +162 -0
- diffusers/schedulers/scheduling_consistency_models.py +2 -0
- diffusers/schedulers/scheduling_ddim.py +1 -3
- diffusers/schedulers/scheduling_ddim_inverse.py +2 -7
- diffusers/schedulers/scheduling_ddim_parallel.py +1 -3
- diffusers/schedulers/scheduling_ddpm.py +47 -3
- diffusers/schedulers/scheduling_ddpm_parallel.py +47 -3
- diffusers/schedulers/scheduling_deis_multistep.py +28 -6
- diffusers/schedulers/scheduling_dpmsolver_multistep.py +28 -6
- diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +28 -6
- diffusers/schedulers/scheduling_dpmsolver_sde.py +3 -3
- diffusers/schedulers/scheduling_dpmsolver_singlestep.py +28 -6
- diffusers/schedulers/scheduling_euler_ancestral_discrete.py +59 -3
- diffusers/schedulers/scheduling_euler_discrete.py +102 -16
- diffusers/schedulers/scheduling_heun_discrete.py +17 -5
- diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +17 -5
- diffusers/schedulers/scheduling_k_dpm_2_discrete.py +17 -5
- diffusers/schedulers/scheduling_lcm.py +123 -29
- diffusers/schedulers/scheduling_lms_discrete.py +3 -3
- diffusers/schedulers/scheduling_pndm.py +1 -3
- diffusers/schedulers/scheduling_repaint.py +1 -3
- diffusers/schedulers/scheduling_unipc_multistep.py +28 -6
- diffusers/schedulers/scheduling_utils.py +3 -1
- diffusers/schedulers/scheduling_utils_flax.py +3 -1
- diffusers/training_utils.py +1 -1
- diffusers/utils/__init__.py +1 -2
- diffusers/utils/constants.py +10 -12
- diffusers/utils/dummy_pt_objects.py +75 -0
- diffusers/utils/dummy_torch_and_transformers_objects.py +105 -0
- diffusers/utils/dynamic_modules_utils.py +18 -22
- diffusers/utils/export_utils.py +8 -3
- diffusers/utils/hub_utils.py +24 -36
- diffusers/utils/logging.py +11 -11
- diffusers/utils/outputs.py +5 -5
- diffusers/utils/peft_utils.py +88 -44
- diffusers/utils/state_dict_utils.py +8 -0
- diffusers/utils/testing_utils.py +199 -1
- diffusers/utils/torch_utils.py +4 -4
- {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/METADATA +86 -69
- diffusers-0.25.0.dist-info/RECORD +360 -0
- {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/WHEEL +1 -1
- {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/entry_points.txt +0 -1
- diffusers/loaders.py +0 -3336
- diffusers-0.23.1.dist-info/RECORD +0 -323
- /diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/modeling_roberta_series.py +0 -0
- {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/LICENSE +0 -0
- {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/top_level.txt +0 -0
@@ -19,10 +19,10 @@ import numpy as np
|
|
19
19
|
import PIL.Image
|
20
20
|
import torch
|
21
21
|
import torch.nn.functional as F
|
22
|
-
from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
|
22
|
+
from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection
|
23
23
|
|
24
24
|
from ...image_processor import PipelineImageInput, VaeImageProcessor
|
25
|
-
from ...loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin
|
25
|
+
from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin
|
26
26
|
from ...models import AutoencoderKL, ControlNetModel, UNet2DConditionModel
|
27
27
|
from ...models.lora import adjust_lora_scale_text_encoder
|
28
28
|
from ...schedulers import KarrasDiffusionSchedulers
|
@@ -92,9 +92,13 @@ EXAMPLE_DOC_STRING = """
|
|
92
92
|
|
93
93
|
|
94
94
|
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
|
95
|
-
def retrieve_latents(
|
96
|
-
|
95
|
+
def retrieve_latents(
|
96
|
+
encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
|
97
|
+
):
|
98
|
+
if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
|
97
99
|
return encoder_output.latent_dist.sample(generator)
|
100
|
+
elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
|
101
|
+
return encoder_output.latent_dist.mode()
|
98
102
|
elif hasattr(encoder_output, "latents"):
|
99
103
|
return encoder_output.latents
|
100
104
|
else:
|
@@ -126,7 +130,7 @@ def prepare_image(image):
|
|
126
130
|
|
127
131
|
|
128
132
|
class StableDiffusionControlNetImg2ImgPipeline(
|
129
|
-
DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin
|
133
|
+
DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin
|
130
134
|
):
|
131
135
|
r"""
|
132
136
|
Pipeline for image-to-image generation using Stable Diffusion with ControlNet guidance.
|
@@ -136,6 +140,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
136
140
|
|
137
141
|
The pipeline also inherits the following loading methods:
|
138
142
|
- [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
|
143
|
+
- [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights
|
144
|
+
- [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights
|
145
|
+
- [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
|
146
|
+
- [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
|
139
147
|
|
140
148
|
Args:
|
141
149
|
vae ([`AutoencoderKL`]):
|
@@ -160,9 +168,11 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
160
168
|
feature_extractor ([`~transformers.CLIPImageProcessor`]):
|
161
169
|
A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.
|
162
170
|
"""
|
171
|
+
|
163
172
|
model_cpu_offload_seq = "text_encoder->unet->vae"
|
164
|
-
_optional_components = ["safety_checker", "feature_extractor"]
|
173
|
+
_optional_components = ["safety_checker", "feature_extractor", "image_encoder"]
|
165
174
|
_exclude_from_cpu_offload = ["safety_checker"]
|
175
|
+
_callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
|
166
176
|
|
167
177
|
def __init__(
|
168
178
|
self,
|
@@ -174,6 +184,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
174
184
|
scheduler: KarrasDiffusionSchedulers,
|
175
185
|
safety_checker: StableDiffusionSafetyChecker,
|
176
186
|
feature_extractor: CLIPImageProcessor,
|
187
|
+
image_encoder: CLIPVisionModelWithProjection = None,
|
177
188
|
requires_safety_checker: bool = True,
|
178
189
|
):
|
179
190
|
super().__init__()
|
@@ -206,6 +217,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
206
217
|
scheduler=scheduler,
|
207
218
|
safety_checker=safety_checker,
|
208
219
|
feature_extractor=feature_extractor,
|
220
|
+
image_encoder=image_encoder,
|
209
221
|
)
|
210
222
|
self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
|
211
223
|
self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True)
|
@@ -462,6 +474,31 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
462
474
|
|
463
475
|
return prompt_embeds, negative_prompt_embeds
|
464
476
|
|
477
|
+
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
|
478
|
+
def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
|
479
|
+
dtype = next(self.image_encoder.parameters()).dtype
|
480
|
+
|
481
|
+
if not isinstance(image, torch.Tensor):
|
482
|
+
image = self.feature_extractor(image, return_tensors="pt").pixel_values
|
483
|
+
|
484
|
+
image = image.to(device=device, dtype=dtype)
|
485
|
+
if output_hidden_states:
|
486
|
+
image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
|
487
|
+
image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
|
488
|
+
uncond_image_enc_hidden_states = self.image_encoder(
|
489
|
+
torch.zeros_like(image), output_hidden_states=True
|
490
|
+
).hidden_states[-2]
|
491
|
+
uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
|
492
|
+
num_images_per_prompt, dim=0
|
493
|
+
)
|
494
|
+
return image_enc_hidden_states, uncond_image_enc_hidden_states
|
495
|
+
else:
|
496
|
+
image_embeds = self.image_encoder(image).image_embeds
|
497
|
+
image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
|
498
|
+
uncond_image_embeds = torch.zeros_like(image_embeds)
|
499
|
+
|
500
|
+
return image_embeds, uncond_image_embeds
|
501
|
+
|
465
502
|
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
|
466
503
|
def run_safety_checker(self, image, device, dtype):
|
467
504
|
if self.safety_checker is None:
|
@@ -518,15 +555,21 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
518
555
|
controlnet_conditioning_scale=1.0,
|
519
556
|
control_guidance_start=0.0,
|
520
557
|
control_guidance_end=1.0,
|
558
|
+
callback_on_step_end_tensor_inputs=None,
|
521
559
|
):
|
522
|
-
if
|
523
|
-
callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
|
524
|
-
):
|
560
|
+
if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
|
525
561
|
raise ValueError(
|
526
562
|
f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
|
527
563
|
f" {type(callback_steps)}."
|
528
564
|
)
|
529
565
|
|
566
|
+
if callback_on_step_end_tensor_inputs is not None and not all(
|
567
|
+
k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
|
568
|
+
):
|
569
|
+
raise ValueError(
|
570
|
+
f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
|
571
|
+
)
|
572
|
+
|
530
573
|
if prompt is not None and prompt_embeds is not None:
|
531
574
|
raise ValueError(
|
532
575
|
f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
|
@@ -807,6 +850,29 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
807
850
|
"""Disables the FreeU mechanism if enabled."""
|
808
851
|
self.unet.disable_freeu()
|
809
852
|
|
853
|
+
@property
|
854
|
+
def guidance_scale(self):
|
855
|
+
return self._guidance_scale
|
856
|
+
|
857
|
+
@property
|
858
|
+
def clip_skip(self):
|
859
|
+
return self._clip_skip
|
860
|
+
|
861
|
+
# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
|
862
|
+
# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
|
863
|
+
# corresponds to doing no classifier free guidance.
|
864
|
+
@property
|
865
|
+
def do_classifier_free_guidance(self):
|
866
|
+
return self._guidance_scale > 1
|
867
|
+
|
868
|
+
@property
|
869
|
+
def cross_attention_kwargs(self):
|
870
|
+
return self._cross_attention_kwargs
|
871
|
+
|
872
|
+
@property
|
873
|
+
def num_timesteps(self):
|
874
|
+
return self._num_timesteps
|
875
|
+
|
810
876
|
@torch.no_grad()
|
811
877
|
@replace_example_docstring(EXAMPLE_DOC_STRING)
|
812
878
|
def __call__(
|
@@ -826,16 +892,18 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
826
892
|
latents: Optional[torch.FloatTensor] = None,
|
827
893
|
prompt_embeds: Optional[torch.FloatTensor] = None,
|
828
894
|
negative_prompt_embeds: Optional[torch.FloatTensor] = None,
|
895
|
+
ip_adapter_image: Optional[PipelineImageInput] = None,
|
829
896
|
output_type: Optional[str] = "pil",
|
830
897
|
return_dict: bool = True,
|
831
|
-
callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
|
832
|
-
callback_steps: int = 1,
|
833
898
|
cross_attention_kwargs: Optional[Dict[str, Any]] = None,
|
834
899
|
controlnet_conditioning_scale: Union[float, List[float]] = 0.8,
|
835
900
|
guess_mode: bool = False,
|
836
901
|
control_guidance_start: Union[float, List[float]] = 0.0,
|
837
902
|
control_guidance_end: Union[float, List[float]] = 1.0,
|
838
903
|
clip_skip: Optional[int] = None,
|
904
|
+
callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
|
905
|
+
callback_on_step_end_tensor_inputs: List[str] = ["latents"],
|
906
|
+
**kwargs,
|
839
907
|
):
|
840
908
|
r"""
|
841
909
|
The call function to the pipeline for generation.
|
@@ -886,17 +954,12 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
886
954
|
negative_prompt_embeds (`torch.FloatTensor`, *optional*):
|
887
955
|
Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
|
888
956
|
not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
|
957
|
+
ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
|
889
958
|
output_type (`str`, *optional*, defaults to `"pil"`):
|
890
959
|
The output format of the generated image. Choose between `PIL.Image` or `np.array`.
|
891
960
|
return_dict (`bool`, *optional*, defaults to `True`):
|
892
961
|
Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
|
893
962
|
plain tuple.
|
894
|
-
callback (`Callable`, *optional*):
|
895
|
-
A function that calls every `callback_steps` steps during inference. The function is called with the
|
896
|
-
following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
|
897
|
-
callback_steps (`int`, *optional*, defaults to 1):
|
898
|
-
The frequency at which the `callback` function is called. If not specified, the callback is called at
|
899
|
-
every step.
|
900
963
|
cross_attention_kwargs (`dict`, *optional*):
|
901
964
|
A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
|
902
965
|
[`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
|
@@ -914,6 +977,15 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
914
977
|
clip_skip (`int`, *optional*):
|
915
978
|
Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
|
916
979
|
the output of the pre-final layer will be used for computing the prompt embeddings.
|
980
|
+
callback_on_step_end (`Callable`, *optional*):
|
981
|
+
A function that calls at the end of each denoising steps during the inference. The function is called
|
982
|
+
with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
|
983
|
+
callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
|
984
|
+
`callback_on_step_end_tensor_inputs`.
|
985
|
+
callback_on_step_end_tensor_inputs (`List`, *optional*):
|
986
|
+
The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
|
987
|
+
will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
|
988
|
+
`._callback_tensor_inputs` attribute of your pipeine class.
|
917
989
|
|
918
990
|
Examples:
|
919
991
|
|
@@ -924,6 +996,23 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
924
996
|
second element is a list of `bool`s indicating whether the corresponding generated image contains
|
925
997
|
"not-safe-for-work" (nsfw) content.
|
926
998
|
"""
|
999
|
+
|
1000
|
+
callback = kwargs.pop("callback", None)
|
1001
|
+
callback_steps = kwargs.pop("callback_steps", None)
|
1002
|
+
|
1003
|
+
if callback is not None:
|
1004
|
+
deprecate(
|
1005
|
+
"callback",
|
1006
|
+
"1.0.0",
|
1007
|
+
"Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
|
1008
|
+
)
|
1009
|
+
if callback_steps is not None:
|
1010
|
+
deprecate(
|
1011
|
+
"callback_steps",
|
1012
|
+
"1.0.0",
|
1013
|
+
"Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
|
1014
|
+
)
|
1015
|
+
|
927
1016
|
controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet
|
928
1017
|
|
929
1018
|
# align format for control guidance
|
@@ -933,9 +1022,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
933
1022
|
control_guidance_end = len(control_guidance_start) * [control_guidance_end]
|
934
1023
|
elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):
|
935
1024
|
mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1
|
936
|
-
control_guidance_start, control_guidance_end =
|
937
|
-
|
938
|
-
|
1025
|
+
control_guidance_start, control_guidance_end = (
|
1026
|
+
mult * [control_guidance_start],
|
1027
|
+
mult * [control_guidance_end],
|
1028
|
+
)
|
939
1029
|
|
940
1030
|
# 1. Check inputs. Raise error if not correct
|
941
1031
|
self.check_inputs(
|
@@ -948,8 +1038,13 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
948
1038
|
controlnet_conditioning_scale,
|
949
1039
|
control_guidance_start,
|
950
1040
|
control_guidance_end,
|
1041
|
+
callback_on_step_end_tensor_inputs,
|
951
1042
|
)
|
952
1043
|
|
1044
|
+
self._guidance_scale = guidance_scale
|
1045
|
+
self._clip_skip = clip_skip
|
1046
|
+
self._cross_attention_kwargs = cross_attention_kwargs
|
1047
|
+
|
953
1048
|
# 2. Define call parameters
|
954
1049
|
if prompt is not None and isinstance(prompt, str):
|
955
1050
|
batch_size = 1
|
@@ -959,10 +1054,6 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
959
1054
|
batch_size = prompt_embeds.shape[0]
|
960
1055
|
|
961
1056
|
device = self._execution_device
|
962
|
-
# here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
|
963
|
-
# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
|
964
|
-
# corresponds to doing no classifier free guidance.
|
965
|
-
do_classifier_free_guidance = guidance_scale > 1.0
|
966
1057
|
|
967
1058
|
if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
|
968
1059
|
controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)
|
@@ -976,27 +1067,32 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
976
1067
|
|
977
1068
|
# 3. Encode input prompt
|
978
1069
|
text_encoder_lora_scale = (
|
979
|
-
cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
|
1070
|
+
self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
|
980
1071
|
)
|
981
1072
|
prompt_embeds, negative_prompt_embeds = self.encode_prompt(
|
982
1073
|
prompt,
|
983
1074
|
device,
|
984
1075
|
num_images_per_prompt,
|
985
|
-
do_classifier_free_guidance,
|
1076
|
+
self.do_classifier_free_guidance,
|
986
1077
|
negative_prompt,
|
987
1078
|
prompt_embeds=prompt_embeds,
|
988
1079
|
negative_prompt_embeds=negative_prompt_embeds,
|
989
1080
|
lora_scale=text_encoder_lora_scale,
|
990
|
-
clip_skip=clip_skip,
|
1081
|
+
clip_skip=self.clip_skip,
|
991
1082
|
)
|
992
1083
|
# For classifier free guidance, we need to do two forward passes.
|
993
1084
|
# Here we concatenate the unconditional and text embeddings into a single batch
|
994
1085
|
# to avoid doing two forward passes
|
995
|
-
if do_classifier_free_guidance:
|
1086
|
+
if self.do_classifier_free_guidance:
|
996
1087
|
prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
|
997
1088
|
|
1089
|
+
if ip_adapter_image is not None:
|
1090
|
+
image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt)
|
1091
|
+
if self.do_classifier_free_guidance:
|
1092
|
+
image_embeds = torch.cat([negative_image_embeds, image_embeds])
|
1093
|
+
|
998
1094
|
# 4. Prepare image
|
999
|
-
image = self.image_processor.preprocess(image).to(dtype=torch.float32)
|
1095
|
+
image = self.image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32)
|
1000
1096
|
|
1001
1097
|
# 5. Prepare controlnet_conditioning_image
|
1002
1098
|
if isinstance(controlnet, ControlNetModel):
|
@@ -1008,7 +1104,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1008
1104
|
num_images_per_prompt=num_images_per_prompt,
|
1009
1105
|
device=device,
|
1010
1106
|
dtype=controlnet.dtype,
|
1011
|
-
do_classifier_free_guidance=do_classifier_free_guidance,
|
1107
|
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
1012
1108
|
guess_mode=guess_mode,
|
1013
1109
|
)
|
1014
1110
|
elif isinstance(controlnet, MultiControlNetModel):
|
@@ -1023,7 +1119,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1023
1119
|
num_images_per_prompt=num_images_per_prompt,
|
1024
1120
|
device=device,
|
1025
1121
|
dtype=controlnet.dtype,
|
1026
|
-
do_classifier_free_guidance=do_classifier_free_guidance,
|
1122
|
+
do_classifier_free_guidance=self.do_classifier_free_guidance,
|
1027
1123
|
guess_mode=guess_mode,
|
1028
1124
|
)
|
1029
1125
|
|
@@ -1037,6 +1133,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1037
1133
|
self.scheduler.set_timesteps(num_inference_steps, device=device)
|
1038
1134
|
timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
|
1039
1135
|
latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
|
1136
|
+
self._num_timesteps = len(timesteps)
|
1040
1137
|
|
1041
1138
|
# 6. Prepare latent variables
|
1042
1139
|
latents = self.prepare_latents(
|
@@ -1052,7 +1149,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1052
1149
|
# 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
|
1053
1150
|
extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
|
1054
1151
|
|
1055
|
-
# 7.1
|
1152
|
+
# 7.1 Add image embeds for IP-Adapter
|
1153
|
+
added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None
|
1154
|
+
|
1155
|
+
# 7.2 Create tensor stating which controlnets to keep
|
1056
1156
|
controlnet_keep = []
|
1057
1157
|
for i in range(len(timesteps)):
|
1058
1158
|
keeps = [
|
@@ -1066,11 +1166,11 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1066
1166
|
with self.progress_bar(total=num_inference_steps) as progress_bar:
|
1067
1167
|
for i, t in enumerate(timesteps):
|
1068
1168
|
# expand the latents if we are doing classifier free guidance
|
1069
|
-
latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
|
1169
|
+
latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
|
1070
1170
|
latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
|
1071
1171
|
|
1072
1172
|
# controlnet(s) inference
|
1073
|
-
if guess_mode and do_classifier_free_guidance:
|
1173
|
+
if guess_mode and self.do_classifier_free_guidance:
|
1074
1174
|
# Infer ControlNet only for the conditional batch.
|
1075
1175
|
control_model_input = latents
|
1076
1176
|
control_model_input = self.scheduler.scale_model_input(control_model_input, t)
|
@@ -1097,7 +1197,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1097
1197
|
return_dict=False,
|
1098
1198
|
)
|
1099
1199
|
|
1100
|
-
if guess_mode and do_classifier_free_guidance:
|
1200
|
+
if guess_mode and self.do_classifier_free_guidance:
|
1101
1201
|
# Infered ControlNet only for the conditional batch.
|
1102
1202
|
# To apply the output of ControlNet to both the unconditional and conditional batches,
|
1103
1203
|
# add 0 to the unconditional batch to keep it unchanged.
|
@@ -1109,20 +1209,31 @@ class StableDiffusionControlNetImg2ImgPipeline(
|
|
1109
1209
|
latent_model_input,
|
1110
1210
|
t,
|
1111
1211
|
encoder_hidden_states=prompt_embeds,
|
1112
|
-
cross_attention_kwargs=cross_attention_kwargs,
|
1212
|
+
cross_attention_kwargs=self.cross_attention_kwargs,
|
1113
1213
|
down_block_additional_residuals=down_block_res_samples,
|
1114
1214
|
mid_block_additional_residual=mid_block_res_sample,
|
1215
|
+
added_cond_kwargs=added_cond_kwargs,
|
1115
1216
|
return_dict=False,
|
1116
1217
|
)[0]
|
1117
1218
|
|
1118
1219
|
# perform guidance
|
1119
|
-
if do_classifier_free_guidance:
|
1220
|
+
if self.do_classifier_free_guidance:
|
1120
1221
|
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
1121
1222
|
noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
1122
1223
|
|
1123
1224
|
# compute the previous noisy sample x_t -> x_t-1
|
1124
1225
|
latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
|
1125
1226
|
|
1227
|
+
if callback_on_step_end is not None:
|
1228
|
+
callback_kwargs = {}
|
1229
|
+
for k in callback_on_step_end_tensor_inputs:
|
1230
|
+
callback_kwargs[k] = locals()[k]
|
1231
|
+
callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
|
1232
|
+
|
1233
|
+
latents = callback_outputs.pop("latents", latents)
|
1234
|
+
prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
|
1235
|
+
negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
|
1236
|
+
|
1126
1237
|
# call the callback, if provided
|
1127
1238
|
if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
|
1128
1239
|
progress_bar.update()
|