diffusers 0.23.1__py3-none-any.whl → 0.25.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (238) hide show
  1. diffusers/__init__.py +26 -2
  2. diffusers/commands/fp16_safetensors.py +10 -11
  3. diffusers/configuration_utils.py +13 -8
  4. diffusers/dependency_versions_check.py +0 -1
  5. diffusers/dependency_versions_table.py +5 -5
  6. diffusers/experimental/rl/value_guided_sampling.py +1 -1
  7. diffusers/image_processor.py +463 -51
  8. diffusers/loaders/__init__.py +82 -0
  9. diffusers/loaders/ip_adapter.py +159 -0
  10. diffusers/loaders/lora.py +1553 -0
  11. diffusers/loaders/lora_conversion_utils.py +284 -0
  12. diffusers/loaders/single_file.py +637 -0
  13. diffusers/loaders/textual_inversion.py +455 -0
  14. diffusers/loaders/unet.py +828 -0
  15. diffusers/loaders/utils.py +59 -0
  16. diffusers/models/__init__.py +26 -9
  17. diffusers/models/activations.py +9 -6
  18. diffusers/models/attention.py +301 -29
  19. diffusers/models/attention_flax.py +9 -1
  20. diffusers/models/attention_processor.py +378 -6
  21. diffusers/models/autoencoders/__init__.py +5 -0
  22. diffusers/models/{autoencoder_asym_kl.py → autoencoders/autoencoder_asym_kl.py} +17 -12
  23. diffusers/models/{autoencoder_kl.py → autoencoders/autoencoder_kl.py} +47 -23
  24. diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +402 -0
  25. diffusers/models/{autoencoder_tiny.py → autoencoders/autoencoder_tiny.py} +24 -28
  26. diffusers/models/{consistency_decoder_vae.py → autoencoders/consistency_decoder_vae.py} +51 -44
  27. diffusers/models/{vae.py → autoencoders/vae.py} +71 -17
  28. diffusers/models/controlnet.py +59 -39
  29. diffusers/models/controlnet_flax.py +19 -18
  30. diffusers/models/downsampling.py +338 -0
  31. diffusers/models/embeddings.py +112 -29
  32. diffusers/models/embeddings_flax.py +2 -0
  33. diffusers/models/lora.py +131 -1
  34. diffusers/models/modeling_flax_utils.py +14 -8
  35. diffusers/models/modeling_outputs.py +17 -0
  36. diffusers/models/modeling_utils.py +37 -29
  37. diffusers/models/normalization.py +110 -4
  38. diffusers/models/resnet.py +299 -652
  39. diffusers/models/transformer_2d.py +22 -5
  40. diffusers/models/transformer_temporal.py +183 -1
  41. diffusers/models/unet_2d_blocks_flax.py +5 -0
  42. diffusers/models/unet_2d_condition.py +46 -0
  43. diffusers/models/unet_2d_condition_flax.py +13 -13
  44. diffusers/models/unet_3d_blocks.py +957 -173
  45. diffusers/models/unet_3d_condition.py +16 -8
  46. diffusers/models/unet_kandinsky3.py +535 -0
  47. diffusers/models/unet_motion_model.py +48 -33
  48. diffusers/models/unet_spatio_temporal_condition.py +489 -0
  49. diffusers/models/upsampling.py +454 -0
  50. diffusers/models/uvit_2d.py +471 -0
  51. diffusers/models/vae_flax.py +7 -0
  52. diffusers/models/vq_model.py +12 -3
  53. diffusers/optimization.py +16 -9
  54. diffusers/pipelines/__init__.py +137 -76
  55. diffusers/pipelines/amused/__init__.py +62 -0
  56. diffusers/pipelines/amused/pipeline_amused.py +328 -0
  57. diffusers/pipelines/amused/pipeline_amused_img2img.py +347 -0
  58. diffusers/pipelines/amused/pipeline_amused_inpaint.py +378 -0
  59. diffusers/pipelines/animatediff/pipeline_animatediff.py +66 -8
  60. diffusers/pipelines/audioldm/pipeline_audioldm.py +1 -0
  61. diffusers/pipelines/auto_pipeline.py +23 -13
  62. diffusers/pipelines/consistency_models/pipeline_consistency_models.py +1 -0
  63. diffusers/pipelines/controlnet/pipeline_controlnet.py +238 -35
  64. diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +148 -37
  65. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +155 -41
  66. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +123 -43
  67. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +216 -39
  68. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +106 -34
  69. diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +1 -0
  70. diffusers/pipelines/ddim/pipeline_ddim.py +1 -0
  71. diffusers/pipelines/ddpm/pipeline_ddpm.py +1 -0
  72. diffusers/pipelines/deepfloyd_if/pipeline_if.py +13 -1
  73. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +13 -1
  74. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +13 -1
  75. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +13 -1
  76. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +13 -1
  77. diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +13 -1
  78. diffusers/pipelines/deprecated/__init__.py +153 -0
  79. diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/__init__.py +3 -3
  80. diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_alt_diffusion.py +177 -34
  81. diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_alt_diffusion_img2img.py +182 -37
  82. diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/pipeline_output.py +1 -1
  83. diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/__init__.py +1 -1
  84. diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/mel.py +2 -2
  85. diffusers/pipelines/{audio_diffusion → deprecated/audio_diffusion}/pipeline_audio_diffusion.py +4 -4
  86. diffusers/pipelines/{latent_diffusion_uncond → deprecated/latent_diffusion_uncond}/__init__.py +1 -1
  87. diffusers/pipelines/{latent_diffusion_uncond → deprecated/latent_diffusion_uncond}/pipeline_latent_diffusion_uncond.py +4 -4
  88. diffusers/pipelines/{pndm → deprecated/pndm}/__init__.py +1 -1
  89. diffusers/pipelines/{pndm → deprecated/pndm}/pipeline_pndm.py +4 -4
  90. diffusers/pipelines/{repaint → deprecated/repaint}/__init__.py +1 -1
  91. diffusers/pipelines/{repaint → deprecated/repaint}/pipeline_repaint.py +5 -5
  92. diffusers/pipelines/{score_sde_ve → deprecated/score_sde_ve}/__init__.py +1 -1
  93. diffusers/pipelines/{score_sde_ve → deprecated/score_sde_ve}/pipeline_score_sde_ve.py +5 -4
  94. diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/__init__.py +6 -6
  95. diffusers/pipelines/{spectrogram_diffusion/continous_encoder.py → deprecated/spectrogram_diffusion/continuous_encoder.py} +2 -2
  96. diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/midi_utils.py +1 -1
  97. diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/notes_encoder.py +2 -2
  98. diffusers/pipelines/{spectrogram_diffusion → deprecated/spectrogram_diffusion}/pipeline_spectrogram_diffusion.py +8 -7
  99. diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py +55 -0
  100. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_cycle_diffusion.py +34 -13
  101. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_onnx_stable_diffusion_inpaint_legacy.py +7 -6
  102. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_inpaint_legacy.py +12 -11
  103. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_model_editing.py +17 -11
  104. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_paradigms.py +11 -10
  105. diffusers/pipelines/{stable_diffusion → deprecated/stable_diffusion_variants}/pipeline_stable_diffusion_pix2pix_zero.py +14 -13
  106. diffusers/pipelines/{stochastic_karras_ve → deprecated/stochastic_karras_ve}/__init__.py +1 -1
  107. diffusers/pipelines/{stochastic_karras_ve → deprecated/stochastic_karras_ve}/pipeline_stochastic_karras_ve.py +4 -4
  108. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/__init__.py +3 -3
  109. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/modeling_text_unet.py +83 -51
  110. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion.py +4 -4
  111. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_dual_guided.py +7 -6
  112. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_image_variation.py +7 -6
  113. diffusers/pipelines/{versatile_diffusion → deprecated/versatile_diffusion}/pipeline_versatile_diffusion_text_to_image.py +7 -6
  114. diffusers/pipelines/{vq_diffusion → deprecated/vq_diffusion}/__init__.py +3 -3
  115. diffusers/pipelines/{vq_diffusion → deprecated/vq_diffusion}/pipeline_vq_diffusion.py +5 -5
  116. diffusers/pipelines/dit/pipeline_dit.py +1 -0
  117. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +1 -1
  118. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +3 -3
  119. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +1 -1
  120. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +1 -1
  121. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +1 -1
  122. diffusers/pipelines/kandinsky3/__init__.py +49 -0
  123. diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py +98 -0
  124. diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +589 -0
  125. diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +654 -0
  126. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +111 -11
  127. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +102 -9
  128. diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +1 -0
  129. diffusers/pipelines/musicldm/pipeline_musicldm.py +1 -1
  130. diffusers/pipelines/onnx_utils.py +8 -5
  131. diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +7 -2
  132. diffusers/pipelines/pipeline_flax_utils.py +11 -8
  133. diffusers/pipelines/pipeline_utils.py +63 -42
  134. diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +247 -38
  135. diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +3 -3
  136. diffusers/pipelines/stable_diffusion/__init__.py +37 -65
  137. diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +75 -78
  138. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +2 -2
  139. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +2 -4
  140. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +1 -0
  141. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +174 -11
  142. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +8 -3
  143. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +1 -0
  144. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +178 -11
  145. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +224 -13
  146. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +74 -20
  147. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +4 -0
  148. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +7 -0
  149. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +5 -0
  150. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +5 -0
  151. diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py +48 -0
  152. diffusers/pipelines/{stable_diffusion → stable_diffusion_attend_and_excite}/pipeline_stable_diffusion_attend_and_excite.py +6 -2
  153. diffusers/pipelines/stable_diffusion_diffedit/__init__.py +48 -0
  154. diffusers/pipelines/{stable_diffusion → stable_diffusion_diffedit}/pipeline_stable_diffusion_diffedit.py +3 -3
  155. diffusers/pipelines/stable_diffusion_gligen/__init__.py +50 -0
  156. diffusers/pipelines/{stable_diffusion → stable_diffusion_gligen}/pipeline_stable_diffusion_gligen.py +3 -2
  157. diffusers/pipelines/{stable_diffusion → stable_diffusion_gligen}/pipeline_stable_diffusion_gligen_text_image.py +4 -3
  158. diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py +60 -0
  159. diffusers/pipelines/{stable_diffusion → stable_diffusion_k_diffusion}/pipeline_stable_diffusion_k_diffusion.py +7 -1
  160. diffusers/pipelines/stable_diffusion_ldm3d/__init__.py +48 -0
  161. diffusers/pipelines/{stable_diffusion → stable_diffusion_ldm3d}/pipeline_stable_diffusion_ldm3d.py +51 -7
  162. diffusers/pipelines/stable_diffusion_panorama/__init__.py +48 -0
  163. diffusers/pipelines/{stable_diffusion → stable_diffusion_panorama}/pipeline_stable_diffusion_panorama.py +57 -8
  164. diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +58 -6
  165. diffusers/pipelines/stable_diffusion_sag/__init__.py +48 -0
  166. diffusers/pipelines/{stable_diffusion → stable_diffusion_sag}/pipeline_stable_diffusion_sag.py +68 -10
  167. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +194 -17
  168. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +205 -16
  169. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +206 -17
  170. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +23 -17
  171. diffusers/pipelines/stable_video_diffusion/__init__.py +58 -0
  172. diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +652 -0
  173. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +108 -12
  174. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +115 -14
  175. diffusers/pipelines/text_to_video_synthesis/__init__.py +2 -0
  176. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +6 -0
  177. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +23 -3
  178. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +334 -10
  179. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +1331 -0
  180. diffusers/pipelines/unclip/pipeline_unclip.py +2 -1
  181. diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +1 -0
  182. diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +1 -1
  183. diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +14 -4
  184. diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +9 -5
  185. diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +1 -1
  186. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +2 -2
  187. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +5 -1
  188. diffusers/schedulers/__init__.py +4 -4
  189. diffusers/schedulers/deprecated/__init__.py +50 -0
  190. diffusers/schedulers/{scheduling_karras_ve.py → deprecated/scheduling_karras_ve.py} +4 -4
  191. diffusers/schedulers/{scheduling_sde_vp.py → deprecated/scheduling_sde_vp.py} +4 -6
  192. diffusers/schedulers/scheduling_amused.py +162 -0
  193. diffusers/schedulers/scheduling_consistency_models.py +2 -0
  194. diffusers/schedulers/scheduling_ddim.py +1 -3
  195. diffusers/schedulers/scheduling_ddim_inverse.py +2 -7
  196. diffusers/schedulers/scheduling_ddim_parallel.py +1 -3
  197. diffusers/schedulers/scheduling_ddpm.py +47 -3
  198. diffusers/schedulers/scheduling_ddpm_parallel.py +47 -3
  199. diffusers/schedulers/scheduling_deis_multistep.py +28 -6
  200. diffusers/schedulers/scheduling_dpmsolver_multistep.py +28 -6
  201. diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +28 -6
  202. diffusers/schedulers/scheduling_dpmsolver_sde.py +3 -3
  203. diffusers/schedulers/scheduling_dpmsolver_singlestep.py +28 -6
  204. diffusers/schedulers/scheduling_euler_ancestral_discrete.py +59 -3
  205. diffusers/schedulers/scheduling_euler_discrete.py +102 -16
  206. diffusers/schedulers/scheduling_heun_discrete.py +17 -5
  207. diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +17 -5
  208. diffusers/schedulers/scheduling_k_dpm_2_discrete.py +17 -5
  209. diffusers/schedulers/scheduling_lcm.py +123 -29
  210. diffusers/schedulers/scheduling_lms_discrete.py +3 -3
  211. diffusers/schedulers/scheduling_pndm.py +1 -3
  212. diffusers/schedulers/scheduling_repaint.py +1 -3
  213. diffusers/schedulers/scheduling_unipc_multistep.py +28 -6
  214. diffusers/schedulers/scheduling_utils.py +3 -1
  215. diffusers/schedulers/scheduling_utils_flax.py +3 -1
  216. diffusers/training_utils.py +1 -1
  217. diffusers/utils/__init__.py +1 -2
  218. diffusers/utils/constants.py +10 -12
  219. diffusers/utils/dummy_pt_objects.py +75 -0
  220. diffusers/utils/dummy_torch_and_transformers_objects.py +105 -0
  221. diffusers/utils/dynamic_modules_utils.py +18 -22
  222. diffusers/utils/export_utils.py +8 -3
  223. diffusers/utils/hub_utils.py +24 -36
  224. diffusers/utils/logging.py +11 -11
  225. diffusers/utils/outputs.py +5 -5
  226. diffusers/utils/peft_utils.py +88 -44
  227. diffusers/utils/state_dict_utils.py +8 -0
  228. diffusers/utils/testing_utils.py +199 -1
  229. diffusers/utils/torch_utils.py +4 -4
  230. {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/METADATA +86 -69
  231. diffusers-0.25.0.dist-info/RECORD +360 -0
  232. {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/WHEEL +1 -1
  233. {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/entry_points.txt +0 -1
  234. diffusers/loaders.py +0 -3336
  235. diffusers-0.23.1.dist-info/RECORD +0 -323
  236. /diffusers/pipelines/{alt_diffusion → deprecated/alt_diffusion}/modeling_roberta_series.py +0 -0
  237. {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/LICENSE +0 -0
  238. {diffusers-0.23.1.dist-info → diffusers-0.25.0.dist-info}/top_level.txt +0 -0
@@ -19,10 +19,10 @@ import numpy as np
19
19
  import PIL.Image
20
20
  import torch
21
21
  import torch.nn.functional as F
22
- from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer
22
+ from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection
23
23
 
24
24
  from ...image_processor import PipelineImageInput, VaeImageProcessor
25
- from ...loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin
25
+ from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin
26
26
  from ...models import AutoencoderKL, ControlNetModel, UNet2DConditionModel
27
27
  from ...models.lora import adjust_lora_scale_text_encoder
28
28
  from ...schedulers import KarrasDiffusionSchedulers
@@ -92,9 +92,13 @@ EXAMPLE_DOC_STRING = """
92
92
 
93
93
 
94
94
  # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents
95
- def retrieve_latents(encoder_output, generator):
96
- if hasattr(encoder_output, "latent_dist"):
95
+ def retrieve_latents(
96
+ encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample"
97
+ ):
98
+ if hasattr(encoder_output, "latent_dist") and sample_mode == "sample":
97
99
  return encoder_output.latent_dist.sample(generator)
100
+ elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax":
101
+ return encoder_output.latent_dist.mode()
98
102
  elif hasattr(encoder_output, "latents"):
99
103
  return encoder_output.latents
100
104
  else:
@@ -126,7 +130,7 @@ def prepare_image(image):
126
130
 
127
131
 
128
132
  class StableDiffusionControlNetImg2ImgPipeline(
129
- DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin
133
+ DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin
130
134
  ):
131
135
  r"""
132
136
  Pipeline for image-to-image generation using Stable Diffusion with ControlNet guidance.
@@ -136,6 +140,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
136
140
 
137
141
  The pipeline also inherits the following loading methods:
138
142
  - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings
143
+ - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights
144
+ - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights
145
+ - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files
146
+ - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters
139
147
 
140
148
  Args:
141
149
  vae ([`AutoencoderKL`]):
@@ -160,9 +168,11 @@ class StableDiffusionControlNetImg2ImgPipeline(
160
168
  feature_extractor ([`~transformers.CLIPImageProcessor`]):
161
169
  A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`.
162
170
  """
171
+
163
172
  model_cpu_offload_seq = "text_encoder->unet->vae"
164
- _optional_components = ["safety_checker", "feature_extractor"]
173
+ _optional_components = ["safety_checker", "feature_extractor", "image_encoder"]
165
174
  _exclude_from_cpu_offload = ["safety_checker"]
175
+ _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"]
166
176
 
167
177
  def __init__(
168
178
  self,
@@ -174,6 +184,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
174
184
  scheduler: KarrasDiffusionSchedulers,
175
185
  safety_checker: StableDiffusionSafetyChecker,
176
186
  feature_extractor: CLIPImageProcessor,
187
+ image_encoder: CLIPVisionModelWithProjection = None,
177
188
  requires_safety_checker: bool = True,
178
189
  ):
179
190
  super().__init__()
@@ -206,6 +217,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
206
217
  scheduler=scheduler,
207
218
  safety_checker=safety_checker,
208
219
  feature_extractor=feature_extractor,
220
+ image_encoder=image_encoder,
209
221
  )
210
222
  self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
211
223
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True)
@@ -462,6 +474,31 @@ class StableDiffusionControlNetImg2ImgPipeline(
462
474
 
463
475
  return prompt_embeds, negative_prompt_embeds
464
476
 
477
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image
478
+ def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None):
479
+ dtype = next(self.image_encoder.parameters()).dtype
480
+
481
+ if not isinstance(image, torch.Tensor):
482
+ image = self.feature_extractor(image, return_tensors="pt").pixel_values
483
+
484
+ image = image.to(device=device, dtype=dtype)
485
+ if output_hidden_states:
486
+ image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2]
487
+ image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0)
488
+ uncond_image_enc_hidden_states = self.image_encoder(
489
+ torch.zeros_like(image), output_hidden_states=True
490
+ ).hidden_states[-2]
491
+ uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave(
492
+ num_images_per_prompt, dim=0
493
+ )
494
+ return image_enc_hidden_states, uncond_image_enc_hidden_states
495
+ else:
496
+ image_embeds = self.image_encoder(image).image_embeds
497
+ image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0)
498
+ uncond_image_embeds = torch.zeros_like(image_embeds)
499
+
500
+ return image_embeds, uncond_image_embeds
501
+
465
502
  # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
466
503
  def run_safety_checker(self, image, device, dtype):
467
504
  if self.safety_checker is None:
@@ -518,15 +555,21 @@ class StableDiffusionControlNetImg2ImgPipeline(
518
555
  controlnet_conditioning_scale=1.0,
519
556
  control_guidance_start=0.0,
520
557
  control_guidance_end=1.0,
558
+ callback_on_step_end_tensor_inputs=None,
521
559
  ):
522
- if (callback_steps is None) or (
523
- callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0)
524
- ):
560
+ if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0):
525
561
  raise ValueError(
526
562
  f"`callback_steps` has to be a positive integer but is {callback_steps} of type"
527
563
  f" {type(callback_steps)}."
528
564
  )
529
565
 
566
+ if callback_on_step_end_tensor_inputs is not None and not all(
567
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
568
+ ):
569
+ raise ValueError(
570
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
571
+ )
572
+
530
573
  if prompt is not None and prompt_embeds is not None:
531
574
  raise ValueError(
532
575
  f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
@@ -807,6 +850,29 @@ class StableDiffusionControlNetImg2ImgPipeline(
807
850
  """Disables the FreeU mechanism if enabled."""
808
851
  self.unet.disable_freeu()
809
852
 
853
+ @property
854
+ def guidance_scale(self):
855
+ return self._guidance_scale
856
+
857
+ @property
858
+ def clip_skip(self):
859
+ return self._clip_skip
860
+
861
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
862
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
863
+ # corresponds to doing no classifier free guidance.
864
+ @property
865
+ def do_classifier_free_guidance(self):
866
+ return self._guidance_scale > 1
867
+
868
+ @property
869
+ def cross_attention_kwargs(self):
870
+ return self._cross_attention_kwargs
871
+
872
+ @property
873
+ def num_timesteps(self):
874
+ return self._num_timesteps
875
+
810
876
  @torch.no_grad()
811
877
  @replace_example_docstring(EXAMPLE_DOC_STRING)
812
878
  def __call__(
@@ -826,16 +892,18 @@ class StableDiffusionControlNetImg2ImgPipeline(
826
892
  latents: Optional[torch.FloatTensor] = None,
827
893
  prompt_embeds: Optional[torch.FloatTensor] = None,
828
894
  negative_prompt_embeds: Optional[torch.FloatTensor] = None,
895
+ ip_adapter_image: Optional[PipelineImageInput] = None,
829
896
  output_type: Optional[str] = "pil",
830
897
  return_dict: bool = True,
831
- callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
832
- callback_steps: int = 1,
833
898
  cross_attention_kwargs: Optional[Dict[str, Any]] = None,
834
899
  controlnet_conditioning_scale: Union[float, List[float]] = 0.8,
835
900
  guess_mode: bool = False,
836
901
  control_guidance_start: Union[float, List[float]] = 0.0,
837
902
  control_guidance_end: Union[float, List[float]] = 1.0,
838
903
  clip_skip: Optional[int] = None,
904
+ callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
905
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
906
+ **kwargs,
839
907
  ):
840
908
  r"""
841
909
  The call function to the pipeline for generation.
@@ -886,17 +954,12 @@ class StableDiffusionControlNetImg2ImgPipeline(
886
954
  negative_prompt_embeds (`torch.FloatTensor`, *optional*):
887
955
  Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
888
956
  not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
957
+ ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
889
958
  output_type (`str`, *optional*, defaults to `"pil"`):
890
959
  The output format of the generated image. Choose between `PIL.Image` or `np.array`.
891
960
  return_dict (`bool`, *optional*, defaults to `True`):
892
961
  Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
893
962
  plain tuple.
894
- callback (`Callable`, *optional*):
895
- A function that calls every `callback_steps` steps during inference. The function is called with the
896
- following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`.
897
- callback_steps (`int`, *optional*, defaults to 1):
898
- The frequency at which the `callback` function is called. If not specified, the callback is called at
899
- every step.
900
963
  cross_attention_kwargs (`dict`, *optional*):
901
964
  A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in
902
965
  [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py).
@@ -914,6 +977,15 @@ class StableDiffusionControlNetImg2ImgPipeline(
914
977
  clip_skip (`int`, *optional*):
915
978
  Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that
916
979
  the output of the pre-final layer will be used for computing the prompt embeddings.
980
+ callback_on_step_end (`Callable`, *optional*):
981
+ A function that calls at the end of each denoising steps during the inference. The function is called
982
+ with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int,
983
+ callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by
984
+ `callback_on_step_end_tensor_inputs`.
985
+ callback_on_step_end_tensor_inputs (`List`, *optional*):
986
+ The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list
987
+ will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the
988
+ `._callback_tensor_inputs` attribute of your pipeine class.
917
989
 
918
990
  Examples:
919
991
 
@@ -924,6 +996,23 @@ class StableDiffusionControlNetImg2ImgPipeline(
924
996
  second element is a list of `bool`s indicating whether the corresponding generated image contains
925
997
  "not-safe-for-work" (nsfw) content.
926
998
  """
999
+
1000
+ callback = kwargs.pop("callback", None)
1001
+ callback_steps = kwargs.pop("callback_steps", None)
1002
+
1003
+ if callback is not None:
1004
+ deprecate(
1005
+ "callback",
1006
+ "1.0.0",
1007
+ "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
1008
+ )
1009
+ if callback_steps is not None:
1010
+ deprecate(
1011
+ "callback_steps",
1012
+ "1.0.0",
1013
+ "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",
1014
+ )
1015
+
927
1016
  controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet
928
1017
 
929
1018
  # align format for control guidance
@@ -933,9 +1022,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
933
1022
  control_guidance_end = len(control_guidance_start) * [control_guidance_end]
934
1023
  elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list):
935
1024
  mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1
936
- control_guidance_start, control_guidance_end = mult * [control_guidance_start], mult * [
937
- control_guidance_end
938
- ]
1025
+ control_guidance_start, control_guidance_end = (
1026
+ mult * [control_guidance_start],
1027
+ mult * [control_guidance_end],
1028
+ )
939
1029
 
940
1030
  # 1. Check inputs. Raise error if not correct
941
1031
  self.check_inputs(
@@ -948,8 +1038,13 @@ class StableDiffusionControlNetImg2ImgPipeline(
948
1038
  controlnet_conditioning_scale,
949
1039
  control_guidance_start,
950
1040
  control_guidance_end,
1041
+ callback_on_step_end_tensor_inputs,
951
1042
  )
952
1043
 
1044
+ self._guidance_scale = guidance_scale
1045
+ self._clip_skip = clip_skip
1046
+ self._cross_attention_kwargs = cross_attention_kwargs
1047
+
953
1048
  # 2. Define call parameters
954
1049
  if prompt is not None and isinstance(prompt, str):
955
1050
  batch_size = 1
@@ -959,10 +1054,6 @@ class StableDiffusionControlNetImg2ImgPipeline(
959
1054
  batch_size = prompt_embeds.shape[0]
960
1055
 
961
1056
  device = self._execution_device
962
- # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
963
- # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
964
- # corresponds to doing no classifier free guidance.
965
- do_classifier_free_guidance = guidance_scale > 1.0
966
1057
 
967
1058
  if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float):
968
1059
  controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets)
@@ -976,27 +1067,32 @@ class StableDiffusionControlNetImg2ImgPipeline(
976
1067
 
977
1068
  # 3. Encode input prompt
978
1069
  text_encoder_lora_scale = (
979
- cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None
1070
+ self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None
980
1071
  )
981
1072
  prompt_embeds, negative_prompt_embeds = self.encode_prompt(
982
1073
  prompt,
983
1074
  device,
984
1075
  num_images_per_prompt,
985
- do_classifier_free_guidance,
1076
+ self.do_classifier_free_guidance,
986
1077
  negative_prompt,
987
1078
  prompt_embeds=prompt_embeds,
988
1079
  negative_prompt_embeds=negative_prompt_embeds,
989
1080
  lora_scale=text_encoder_lora_scale,
990
- clip_skip=clip_skip,
1081
+ clip_skip=self.clip_skip,
991
1082
  )
992
1083
  # For classifier free guidance, we need to do two forward passes.
993
1084
  # Here we concatenate the unconditional and text embeddings into a single batch
994
1085
  # to avoid doing two forward passes
995
- if do_classifier_free_guidance:
1086
+ if self.do_classifier_free_guidance:
996
1087
  prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
997
1088
 
1089
+ if ip_adapter_image is not None:
1090
+ image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt)
1091
+ if self.do_classifier_free_guidance:
1092
+ image_embeds = torch.cat([negative_image_embeds, image_embeds])
1093
+
998
1094
  # 4. Prepare image
999
- image = self.image_processor.preprocess(image).to(dtype=torch.float32)
1095
+ image = self.image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32)
1000
1096
 
1001
1097
  # 5. Prepare controlnet_conditioning_image
1002
1098
  if isinstance(controlnet, ControlNetModel):
@@ -1008,7 +1104,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
1008
1104
  num_images_per_prompt=num_images_per_prompt,
1009
1105
  device=device,
1010
1106
  dtype=controlnet.dtype,
1011
- do_classifier_free_guidance=do_classifier_free_guidance,
1107
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1012
1108
  guess_mode=guess_mode,
1013
1109
  )
1014
1110
  elif isinstance(controlnet, MultiControlNetModel):
@@ -1023,7 +1119,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
1023
1119
  num_images_per_prompt=num_images_per_prompt,
1024
1120
  device=device,
1025
1121
  dtype=controlnet.dtype,
1026
- do_classifier_free_guidance=do_classifier_free_guidance,
1122
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
1027
1123
  guess_mode=guess_mode,
1028
1124
  )
1029
1125
 
@@ -1037,6 +1133,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
1037
1133
  self.scheduler.set_timesteps(num_inference_steps, device=device)
1038
1134
  timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device)
1039
1135
  latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt)
1136
+ self._num_timesteps = len(timesteps)
1040
1137
 
1041
1138
  # 6. Prepare latent variables
1042
1139
  latents = self.prepare_latents(
@@ -1052,7 +1149,10 @@ class StableDiffusionControlNetImg2ImgPipeline(
1052
1149
  # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
1053
1150
  extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
1054
1151
 
1055
- # 7.1 Create tensor stating which controlnets to keep
1152
+ # 7.1 Add image embeds for IP-Adapter
1153
+ added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None
1154
+
1155
+ # 7.2 Create tensor stating which controlnets to keep
1056
1156
  controlnet_keep = []
1057
1157
  for i in range(len(timesteps)):
1058
1158
  keeps = [
@@ -1066,11 +1166,11 @@ class StableDiffusionControlNetImg2ImgPipeline(
1066
1166
  with self.progress_bar(total=num_inference_steps) as progress_bar:
1067
1167
  for i, t in enumerate(timesteps):
1068
1168
  # expand the latents if we are doing classifier free guidance
1069
- latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
1169
+ latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents
1070
1170
  latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
1071
1171
 
1072
1172
  # controlnet(s) inference
1073
- if guess_mode and do_classifier_free_guidance:
1173
+ if guess_mode and self.do_classifier_free_guidance:
1074
1174
  # Infer ControlNet only for the conditional batch.
1075
1175
  control_model_input = latents
1076
1176
  control_model_input = self.scheduler.scale_model_input(control_model_input, t)
@@ -1097,7 +1197,7 @@ class StableDiffusionControlNetImg2ImgPipeline(
1097
1197
  return_dict=False,
1098
1198
  )
1099
1199
 
1100
- if guess_mode and do_classifier_free_guidance:
1200
+ if guess_mode and self.do_classifier_free_guidance:
1101
1201
  # Infered ControlNet only for the conditional batch.
1102
1202
  # To apply the output of ControlNet to both the unconditional and conditional batches,
1103
1203
  # add 0 to the unconditional batch to keep it unchanged.
@@ -1109,20 +1209,31 @@ class StableDiffusionControlNetImg2ImgPipeline(
1109
1209
  latent_model_input,
1110
1210
  t,
1111
1211
  encoder_hidden_states=prompt_embeds,
1112
- cross_attention_kwargs=cross_attention_kwargs,
1212
+ cross_attention_kwargs=self.cross_attention_kwargs,
1113
1213
  down_block_additional_residuals=down_block_res_samples,
1114
1214
  mid_block_additional_residual=mid_block_res_sample,
1215
+ added_cond_kwargs=added_cond_kwargs,
1115
1216
  return_dict=False,
1116
1217
  )[0]
1117
1218
 
1118
1219
  # perform guidance
1119
- if do_classifier_free_guidance:
1220
+ if self.do_classifier_free_guidance:
1120
1221
  noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
1121
1222
  noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
1122
1223
 
1123
1224
  # compute the previous noisy sample x_t -> x_t-1
1124
1225
  latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
1125
1226
 
1227
+ if callback_on_step_end is not None:
1228
+ callback_kwargs = {}
1229
+ for k in callback_on_step_end_tensor_inputs:
1230
+ callback_kwargs[k] = locals()[k]
1231
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
1232
+
1233
+ latents = callback_outputs.pop("latents", latents)
1234
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
1235
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
1236
+
1126
1237
  # call the callback, if provided
1127
1238
  if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1128
1239
  progress_bar.update()