diffusers 0.27.1__py3-none-any.whl → 0.32.2__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (445) hide show
  1. diffusers/__init__.py +233 -6
  2. diffusers/callbacks.py +209 -0
  3. diffusers/commands/env.py +102 -6
  4. diffusers/configuration_utils.py +45 -16
  5. diffusers/dependency_versions_table.py +4 -3
  6. diffusers/image_processor.py +434 -110
  7. diffusers/loaders/__init__.py +42 -9
  8. diffusers/loaders/ip_adapter.py +626 -36
  9. diffusers/loaders/lora_base.py +900 -0
  10. diffusers/loaders/lora_conversion_utils.py +991 -125
  11. diffusers/loaders/lora_pipeline.py +3812 -0
  12. diffusers/loaders/peft.py +571 -7
  13. diffusers/loaders/single_file.py +405 -173
  14. diffusers/loaders/single_file_model.py +385 -0
  15. diffusers/loaders/single_file_utils.py +1783 -713
  16. diffusers/loaders/textual_inversion.py +41 -23
  17. diffusers/loaders/transformer_flux.py +181 -0
  18. diffusers/loaders/transformer_sd3.py +89 -0
  19. diffusers/loaders/unet.py +464 -540
  20. diffusers/loaders/unet_loader_utils.py +163 -0
  21. diffusers/models/__init__.py +76 -7
  22. diffusers/models/activations.py +65 -10
  23. diffusers/models/adapter.py +53 -53
  24. diffusers/models/attention.py +605 -18
  25. diffusers/models/attention_flax.py +1 -1
  26. diffusers/models/attention_processor.py +4304 -687
  27. diffusers/models/autoencoders/__init__.py +8 -0
  28. diffusers/models/autoencoders/autoencoder_asym_kl.py +15 -17
  29. diffusers/models/autoencoders/autoencoder_dc.py +620 -0
  30. diffusers/models/autoencoders/autoencoder_kl.py +110 -28
  31. diffusers/models/autoencoders/autoencoder_kl_allegro.py +1149 -0
  32. diffusers/models/autoencoders/autoencoder_kl_cogvideox.py +1482 -0
  33. diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +1176 -0
  34. diffusers/models/autoencoders/autoencoder_kl_ltx.py +1338 -0
  35. diffusers/models/autoencoders/autoencoder_kl_mochi.py +1166 -0
  36. diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +19 -24
  37. diffusers/models/autoencoders/autoencoder_oobleck.py +464 -0
  38. diffusers/models/autoencoders/autoencoder_tiny.py +21 -18
  39. diffusers/models/autoencoders/consistency_decoder_vae.py +45 -20
  40. diffusers/models/autoencoders/vae.py +41 -29
  41. diffusers/models/autoencoders/vq_model.py +182 -0
  42. diffusers/models/controlnet.py +47 -800
  43. diffusers/models/controlnet_flux.py +70 -0
  44. diffusers/models/controlnet_sd3.py +68 -0
  45. diffusers/models/controlnet_sparsectrl.py +116 -0
  46. diffusers/models/controlnets/__init__.py +23 -0
  47. diffusers/models/controlnets/controlnet.py +872 -0
  48. diffusers/models/{controlnet_flax.py → controlnets/controlnet_flax.py} +9 -9
  49. diffusers/models/controlnets/controlnet_flux.py +536 -0
  50. diffusers/models/controlnets/controlnet_hunyuan.py +401 -0
  51. diffusers/models/controlnets/controlnet_sd3.py +489 -0
  52. diffusers/models/controlnets/controlnet_sparsectrl.py +788 -0
  53. diffusers/models/controlnets/controlnet_union.py +832 -0
  54. diffusers/models/controlnets/controlnet_xs.py +1946 -0
  55. diffusers/models/controlnets/multicontrolnet.py +183 -0
  56. diffusers/models/downsampling.py +85 -18
  57. diffusers/models/embeddings.py +1856 -158
  58. diffusers/models/embeddings_flax.py +23 -9
  59. diffusers/models/model_loading_utils.py +480 -0
  60. diffusers/models/modeling_flax_pytorch_utils.py +2 -1
  61. diffusers/models/modeling_flax_utils.py +2 -7
  62. diffusers/models/modeling_outputs.py +14 -0
  63. diffusers/models/modeling_pytorch_flax_utils.py +1 -1
  64. diffusers/models/modeling_utils.py +611 -146
  65. diffusers/models/normalization.py +361 -20
  66. diffusers/models/resnet.py +18 -23
  67. diffusers/models/transformers/__init__.py +16 -0
  68. diffusers/models/transformers/auraflow_transformer_2d.py +544 -0
  69. diffusers/models/transformers/cogvideox_transformer_3d.py +542 -0
  70. diffusers/models/transformers/dit_transformer_2d.py +240 -0
  71. diffusers/models/transformers/dual_transformer_2d.py +9 -8
  72. diffusers/models/transformers/hunyuan_transformer_2d.py +578 -0
  73. diffusers/models/transformers/latte_transformer_3d.py +327 -0
  74. diffusers/models/transformers/lumina_nextdit2d.py +340 -0
  75. diffusers/models/transformers/pixart_transformer_2d.py +445 -0
  76. diffusers/models/transformers/prior_transformer.py +13 -13
  77. diffusers/models/transformers/sana_transformer.py +488 -0
  78. diffusers/models/transformers/stable_audio_transformer.py +458 -0
  79. diffusers/models/transformers/t5_film_transformer.py +17 -19
  80. diffusers/models/transformers/transformer_2d.py +297 -187
  81. diffusers/models/transformers/transformer_allegro.py +422 -0
  82. diffusers/models/transformers/transformer_cogview3plus.py +386 -0
  83. diffusers/models/transformers/transformer_flux.py +593 -0
  84. diffusers/models/transformers/transformer_hunyuan_video.py +791 -0
  85. diffusers/models/transformers/transformer_ltx.py +469 -0
  86. diffusers/models/transformers/transformer_mochi.py +499 -0
  87. diffusers/models/transformers/transformer_sd3.py +461 -0
  88. diffusers/models/transformers/transformer_temporal.py +21 -19
  89. diffusers/models/unets/unet_1d.py +8 -8
  90. diffusers/models/unets/unet_1d_blocks.py +31 -31
  91. diffusers/models/unets/unet_2d.py +17 -10
  92. diffusers/models/unets/unet_2d_blocks.py +225 -149
  93. diffusers/models/unets/unet_2d_condition.py +41 -40
  94. diffusers/models/unets/unet_2d_condition_flax.py +6 -5
  95. diffusers/models/unets/unet_3d_blocks.py +192 -1057
  96. diffusers/models/unets/unet_3d_condition.py +22 -27
  97. diffusers/models/unets/unet_i2vgen_xl.py +22 -18
  98. diffusers/models/unets/unet_kandinsky3.py +2 -2
  99. diffusers/models/unets/unet_motion_model.py +1413 -89
  100. diffusers/models/unets/unet_spatio_temporal_condition.py +40 -16
  101. diffusers/models/unets/unet_stable_cascade.py +19 -18
  102. diffusers/models/unets/uvit_2d.py +2 -2
  103. diffusers/models/upsampling.py +95 -26
  104. diffusers/models/vq_model.py +12 -164
  105. diffusers/optimization.py +1 -1
  106. diffusers/pipelines/__init__.py +202 -3
  107. diffusers/pipelines/allegro/__init__.py +48 -0
  108. diffusers/pipelines/allegro/pipeline_allegro.py +938 -0
  109. diffusers/pipelines/allegro/pipeline_output.py +23 -0
  110. diffusers/pipelines/amused/pipeline_amused.py +12 -12
  111. diffusers/pipelines/amused/pipeline_amused_img2img.py +14 -12
  112. diffusers/pipelines/amused/pipeline_amused_inpaint.py +13 -11
  113. diffusers/pipelines/animatediff/__init__.py +8 -0
  114. diffusers/pipelines/animatediff/pipeline_animatediff.py +122 -109
  115. diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +1106 -0
  116. diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +1288 -0
  117. diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +1010 -0
  118. diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +236 -180
  119. diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +1341 -0
  120. diffusers/pipelines/animatediff/pipeline_output.py +3 -2
  121. diffusers/pipelines/audioldm/pipeline_audioldm.py +14 -14
  122. diffusers/pipelines/audioldm2/modeling_audioldm2.py +58 -39
  123. diffusers/pipelines/audioldm2/pipeline_audioldm2.py +121 -36
  124. diffusers/pipelines/aura_flow/__init__.py +48 -0
  125. diffusers/pipelines/aura_flow/pipeline_aura_flow.py +584 -0
  126. diffusers/pipelines/auto_pipeline.py +196 -28
  127. diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
  128. diffusers/pipelines/blip_diffusion/modeling_blip2.py +6 -6
  129. diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +1 -1
  130. diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +2 -2
  131. diffusers/pipelines/cogvideo/__init__.py +54 -0
  132. diffusers/pipelines/cogvideo/pipeline_cogvideox.py +772 -0
  133. diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py +825 -0
  134. diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py +885 -0
  135. diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py +851 -0
  136. diffusers/pipelines/cogvideo/pipeline_output.py +20 -0
  137. diffusers/pipelines/cogview3/__init__.py +47 -0
  138. diffusers/pipelines/cogview3/pipeline_cogview3plus.py +674 -0
  139. diffusers/pipelines/cogview3/pipeline_output.py +21 -0
  140. diffusers/pipelines/consistency_models/pipeline_consistency_models.py +6 -6
  141. diffusers/pipelines/controlnet/__init__.py +86 -80
  142. diffusers/pipelines/controlnet/multicontrolnet.py +7 -182
  143. diffusers/pipelines/controlnet/pipeline_controlnet.py +134 -87
  144. diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +2 -2
  145. diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +93 -77
  146. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +88 -197
  147. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +136 -90
  148. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +176 -80
  149. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +125 -89
  150. diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +1790 -0
  151. diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +1501 -0
  152. diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +1627 -0
  153. diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +2 -2
  154. diffusers/pipelines/controlnet_hunyuandit/__init__.py +48 -0
  155. diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +1060 -0
  156. diffusers/pipelines/controlnet_sd3/__init__.py +57 -0
  157. diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +1133 -0
  158. diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +1153 -0
  159. diffusers/pipelines/controlnet_xs/__init__.py +68 -0
  160. diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py +916 -0
  161. diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +1111 -0
  162. diffusers/pipelines/ddpm/pipeline_ddpm.py +2 -2
  163. diffusers/pipelines/deepfloyd_if/pipeline_if.py +16 -30
  164. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +20 -35
  165. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +23 -41
  166. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +22 -38
  167. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +25 -41
  168. diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +19 -34
  169. diffusers/pipelines/deepfloyd_if/pipeline_output.py +6 -5
  170. diffusers/pipelines/deepfloyd_if/watermark.py +1 -1
  171. diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py +11 -11
  172. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +70 -30
  173. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +48 -25
  174. diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +2 -2
  175. diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +7 -7
  176. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +21 -20
  177. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +27 -29
  178. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +33 -27
  179. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +33 -23
  180. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +36 -30
  181. diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +102 -69
  182. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py +13 -13
  183. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +10 -5
  184. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +11 -6
  185. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +10 -5
  186. diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +5 -5
  187. diffusers/pipelines/dit/pipeline_dit.py +7 -4
  188. diffusers/pipelines/flux/__init__.py +69 -0
  189. diffusers/pipelines/flux/modeling_flux.py +47 -0
  190. diffusers/pipelines/flux/pipeline_flux.py +957 -0
  191. diffusers/pipelines/flux/pipeline_flux_control.py +889 -0
  192. diffusers/pipelines/flux/pipeline_flux_control_img2img.py +945 -0
  193. diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +1141 -0
  194. diffusers/pipelines/flux/pipeline_flux_controlnet.py +1006 -0
  195. diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +998 -0
  196. diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +1204 -0
  197. diffusers/pipelines/flux/pipeline_flux_fill.py +969 -0
  198. diffusers/pipelines/flux/pipeline_flux_img2img.py +856 -0
  199. diffusers/pipelines/flux/pipeline_flux_inpaint.py +1022 -0
  200. diffusers/pipelines/flux/pipeline_flux_prior_redux.py +492 -0
  201. diffusers/pipelines/flux/pipeline_output.py +37 -0
  202. diffusers/pipelines/free_init_utils.py +41 -38
  203. diffusers/pipelines/free_noise_utils.py +596 -0
  204. diffusers/pipelines/hunyuan_video/__init__.py +48 -0
  205. diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +687 -0
  206. diffusers/pipelines/hunyuan_video/pipeline_output.py +20 -0
  207. diffusers/pipelines/hunyuandit/__init__.py +48 -0
  208. diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +916 -0
  209. diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +33 -48
  210. diffusers/pipelines/kandinsky/pipeline_kandinsky.py +8 -8
  211. diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +32 -29
  212. diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +11 -11
  213. diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +12 -12
  214. diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +10 -10
  215. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +6 -6
  216. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +34 -31
  217. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +10 -10
  218. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +10 -10
  219. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +6 -6
  220. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +8 -8
  221. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +7 -7
  222. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +6 -6
  223. diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py +3 -3
  224. diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +22 -35
  225. diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +26 -37
  226. diffusers/pipelines/kolors/__init__.py +54 -0
  227. diffusers/pipelines/kolors/pipeline_kolors.py +1070 -0
  228. diffusers/pipelines/kolors/pipeline_kolors_img2img.py +1250 -0
  229. diffusers/pipelines/kolors/pipeline_output.py +21 -0
  230. diffusers/pipelines/kolors/text_encoder.py +889 -0
  231. diffusers/pipelines/kolors/tokenizer.py +338 -0
  232. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +82 -62
  233. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +77 -60
  234. diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +12 -12
  235. diffusers/pipelines/latte/__init__.py +48 -0
  236. diffusers/pipelines/latte/pipeline_latte.py +881 -0
  237. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +80 -74
  238. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +85 -76
  239. diffusers/pipelines/ledits_pp/pipeline_output.py +2 -2
  240. diffusers/pipelines/ltx/__init__.py +50 -0
  241. diffusers/pipelines/ltx/pipeline_ltx.py +789 -0
  242. diffusers/pipelines/ltx/pipeline_ltx_image2video.py +885 -0
  243. diffusers/pipelines/ltx/pipeline_output.py +20 -0
  244. diffusers/pipelines/lumina/__init__.py +48 -0
  245. diffusers/pipelines/lumina/pipeline_lumina.py +890 -0
  246. diffusers/pipelines/marigold/__init__.py +50 -0
  247. diffusers/pipelines/marigold/marigold_image_processing.py +576 -0
  248. diffusers/pipelines/marigold/pipeline_marigold_depth.py +813 -0
  249. diffusers/pipelines/marigold/pipeline_marigold_normals.py +690 -0
  250. diffusers/pipelines/mochi/__init__.py +48 -0
  251. diffusers/pipelines/mochi/pipeline_mochi.py +748 -0
  252. diffusers/pipelines/mochi/pipeline_output.py +20 -0
  253. diffusers/pipelines/musicldm/pipeline_musicldm.py +14 -14
  254. diffusers/pipelines/pag/__init__.py +80 -0
  255. diffusers/pipelines/pag/pag_utils.py +243 -0
  256. diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +1328 -0
  257. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +1543 -0
  258. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +1610 -0
  259. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +1683 -0
  260. diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +969 -0
  261. diffusers/pipelines/pag/pipeline_pag_kolors.py +1136 -0
  262. diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py +865 -0
  263. diffusers/pipelines/pag/pipeline_pag_sana.py +886 -0
  264. diffusers/pipelines/pag/pipeline_pag_sd.py +1062 -0
  265. diffusers/pipelines/pag/pipeline_pag_sd_3.py +994 -0
  266. diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +1058 -0
  267. diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +866 -0
  268. diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +1094 -0
  269. diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +1356 -0
  270. diffusers/pipelines/pag/pipeline_pag_sd_xl.py +1345 -0
  271. diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +1544 -0
  272. diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +1776 -0
  273. diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +17 -12
  274. diffusers/pipelines/pia/pipeline_pia.py +74 -164
  275. diffusers/pipelines/pipeline_flax_utils.py +5 -10
  276. diffusers/pipelines/pipeline_loading_utils.py +515 -53
  277. diffusers/pipelines/pipeline_utils.py +411 -222
  278. diffusers/pipelines/pixart_alpha/__init__.py +8 -1
  279. diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +76 -93
  280. diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py +873 -0
  281. diffusers/pipelines/sana/__init__.py +47 -0
  282. diffusers/pipelines/sana/pipeline_output.py +21 -0
  283. diffusers/pipelines/sana/pipeline_sana.py +884 -0
  284. diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +27 -23
  285. diffusers/pipelines/shap_e/pipeline_shap_e.py +3 -3
  286. diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +14 -14
  287. diffusers/pipelines/shap_e/renderer.py +1 -1
  288. diffusers/pipelines/stable_audio/__init__.py +50 -0
  289. diffusers/pipelines/stable_audio/modeling_stable_audio.py +158 -0
  290. diffusers/pipelines/stable_audio/pipeline_stable_audio.py +756 -0
  291. diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +71 -25
  292. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +23 -19
  293. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +35 -34
  294. diffusers/pipelines/stable_diffusion/__init__.py +0 -1
  295. diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +20 -11
  296. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +1 -1
  297. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +2 -2
  298. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +6 -6
  299. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +145 -79
  300. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +43 -28
  301. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +13 -8
  302. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +100 -68
  303. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +109 -201
  304. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +131 -32
  305. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +247 -87
  306. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +30 -29
  307. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +35 -27
  308. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +49 -42
  309. diffusers/pipelines/stable_diffusion/safety_checker.py +2 -1
  310. diffusers/pipelines/stable_diffusion_3/__init__.py +54 -0
  311. diffusers/pipelines/stable_diffusion_3/pipeline_output.py +21 -0
  312. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +1140 -0
  313. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +1036 -0
  314. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +1250 -0
  315. diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +29 -20
  316. diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +59 -58
  317. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +31 -25
  318. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +38 -22
  319. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +30 -24
  320. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +24 -23
  321. diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +107 -67
  322. diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +316 -69
  323. diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +10 -5
  324. diffusers/pipelines/stable_diffusion_safe/safety_checker.py +1 -1
  325. diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +98 -30
  326. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +121 -83
  327. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +161 -105
  328. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +142 -218
  329. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +45 -29
  330. diffusers/pipelines/stable_diffusion_xl/watermark.py +9 -3
  331. diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +110 -57
  332. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +69 -39
  333. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +105 -74
  334. diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +3 -2
  335. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +29 -49
  336. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +32 -93
  337. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +37 -25
  338. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +54 -40
  339. diffusers/pipelines/unclip/pipeline_unclip.py +6 -6
  340. diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +6 -6
  341. diffusers/pipelines/unidiffuser/modeling_text_decoder.py +1 -1
  342. diffusers/pipelines/unidiffuser/modeling_uvit.py +12 -12
  343. diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +29 -28
  344. diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +5 -5
  345. diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +5 -10
  346. diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +6 -8
  347. diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +4 -4
  348. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +12 -12
  349. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +15 -14
  350. diffusers/{models/dual_transformer_2d.py → quantizers/__init__.py} +2 -6
  351. diffusers/quantizers/auto.py +139 -0
  352. diffusers/quantizers/base.py +233 -0
  353. diffusers/quantizers/bitsandbytes/__init__.py +2 -0
  354. diffusers/quantizers/bitsandbytes/bnb_quantizer.py +561 -0
  355. diffusers/quantizers/bitsandbytes/utils.py +306 -0
  356. diffusers/quantizers/gguf/__init__.py +1 -0
  357. diffusers/quantizers/gguf/gguf_quantizer.py +159 -0
  358. diffusers/quantizers/gguf/utils.py +456 -0
  359. diffusers/quantizers/quantization_config.py +669 -0
  360. diffusers/quantizers/torchao/__init__.py +15 -0
  361. diffusers/quantizers/torchao/torchao_quantizer.py +292 -0
  362. diffusers/schedulers/__init__.py +12 -2
  363. diffusers/schedulers/deprecated/__init__.py +1 -1
  364. diffusers/schedulers/deprecated/scheduling_karras_ve.py +25 -25
  365. diffusers/schedulers/scheduling_amused.py +5 -5
  366. diffusers/schedulers/scheduling_consistency_decoder.py +11 -11
  367. diffusers/schedulers/scheduling_consistency_models.py +23 -25
  368. diffusers/schedulers/scheduling_cosine_dpmsolver_multistep.py +572 -0
  369. diffusers/schedulers/scheduling_ddim.py +27 -26
  370. diffusers/schedulers/scheduling_ddim_cogvideox.py +452 -0
  371. diffusers/schedulers/scheduling_ddim_flax.py +2 -1
  372. diffusers/schedulers/scheduling_ddim_inverse.py +16 -16
  373. diffusers/schedulers/scheduling_ddim_parallel.py +32 -31
  374. diffusers/schedulers/scheduling_ddpm.py +27 -30
  375. diffusers/schedulers/scheduling_ddpm_flax.py +7 -3
  376. diffusers/schedulers/scheduling_ddpm_parallel.py +33 -36
  377. diffusers/schedulers/scheduling_ddpm_wuerstchen.py +14 -14
  378. diffusers/schedulers/scheduling_deis_multistep.py +150 -50
  379. diffusers/schedulers/scheduling_dpm_cogvideox.py +489 -0
  380. diffusers/schedulers/scheduling_dpmsolver_multistep.py +221 -84
  381. diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +2 -2
  382. diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +158 -52
  383. diffusers/schedulers/scheduling_dpmsolver_sde.py +153 -34
  384. diffusers/schedulers/scheduling_dpmsolver_singlestep.py +275 -86
  385. diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +81 -57
  386. diffusers/schedulers/scheduling_edm_euler.py +62 -39
  387. diffusers/schedulers/scheduling_euler_ancestral_discrete.py +30 -29
  388. diffusers/schedulers/scheduling_euler_discrete.py +255 -74
  389. diffusers/schedulers/scheduling_flow_match_euler_discrete.py +458 -0
  390. diffusers/schedulers/scheduling_flow_match_heun_discrete.py +320 -0
  391. diffusers/schedulers/scheduling_heun_discrete.py +174 -46
  392. diffusers/schedulers/scheduling_ipndm.py +9 -9
  393. diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +138 -29
  394. diffusers/schedulers/scheduling_k_dpm_2_discrete.py +132 -26
  395. diffusers/schedulers/scheduling_karras_ve_flax.py +6 -6
  396. diffusers/schedulers/scheduling_lcm.py +23 -29
  397. diffusers/schedulers/scheduling_lms_discrete.py +105 -28
  398. diffusers/schedulers/scheduling_pndm.py +20 -20
  399. diffusers/schedulers/scheduling_repaint.py +21 -21
  400. diffusers/schedulers/scheduling_sasolver.py +157 -60
  401. diffusers/schedulers/scheduling_sde_ve.py +19 -19
  402. diffusers/schedulers/scheduling_tcd.py +41 -36
  403. diffusers/schedulers/scheduling_unclip.py +19 -16
  404. diffusers/schedulers/scheduling_unipc_multistep.py +243 -47
  405. diffusers/schedulers/scheduling_utils.py +12 -5
  406. diffusers/schedulers/scheduling_utils_flax.py +1 -3
  407. diffusers/schedulers/scheduling_vq_diffusion.py +10 -10
  408. diffusers/training_utils.py +214 -30
  409. diffusers/utils/__init__.py +17 -1
  410. diffusers/utils/constants.py +3 -0
  411. diffusers/utils/doc_utils.py +1 -0
  412. diffusers/utils/dummy_pt_objects.py +592 -7
  413. diffusers/utils/dummy_torch_and_torchsde_objects.py +15 -0
  414. diffusers/utils/dummy_torch_and_transformers_and_sentencepiece_objects.py +47 -0
  415. diffusers/utils/dummy_torch_and_transformers_objects.py +1001 -71
  416. diffusers/utils/dynamic_modules_utils.py +34 -29
  417. diffusers/utils/export_utils.py +50 -6
  418. diffusers/utils/hub_utils.py +131 -17
  419. diffusers/utils/import_utils.py +210 -8
  420. diffusers/utils/loading_utils.py +118 -5
  421. diffusers/utils/logging.py +4 -2
  422. diffusers/utils/peft_utils.py +37 -7
  423. diffusers/utils/state_dict_utils.py +13 -2
  424. diffusers/utils/testing_utils.py +193 -11
  425. diffusers/utils/torch_utils.py +4 -0
  426. diffusers/video_processor.py +113 -0
  427. {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/METADATA +82 -91
  428. diffusers-0.32.2.dist-info/RECORD +550 -0
  429. {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/WHEEL +1 -1
  430. diffusers/loaders/autoencoder.py +0 -146
  431. diffusers/loaders/controlnet.py +0 -136
  432. diffusers/loaders/lora.py +0 -1349
  433. diffusers/models/prior_transformer.py +0 -12
  434. diffusers/models/t5_film_transformer.py +0 -70
  435. diffusers/models/transformer_2d.py +0 -25
  436. diffusers/models/transformer_temporal.py +0 -34
  437. diffusers/models/unet_1d.py +0 -26
  438. diffusers/models/unet_1d_blocks.py +0 -203
  439. diffusers/models/unet_2d.py +0 -27
  440. diffusers/models/unet_2d_blocks.py +0 -375
  441. diffusers/models/unet_2d_condition.py +0 -25
  442. diffusers-0.27.1.dist-info/RECORD +0 -399
  443. {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/LICENSE +0 -0
  444. {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/entry_points.txt +0 -0
  445. {diffusers-0.27.1.dist-info → diffusers-0.32.2.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,969 @@
1
+ # Copyright 2024 HunyuanDiT Authors and The HuggingFace Team. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ import inspect
16
+ from typing import Callable, Dict, List, Optional, Tuple, Union
17
+
18
+ import numpy as np
19
+ import torch
20
+ from transformers import BertModel, BertTokenizer, CLIPImageProcessor, MT5Tokenizer, T5EncoderModel
21
+
22
+ from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
23
+
24
+ from ...callbacks import MultiPipelineCallbacks, PipelineCallback
25
+ from ...image_processor import VaeImageProcessor
26
+ from ...models import AutoencoderKL, HunyuanDiT2DModel
27
+ from ...models.attention_processor import PAGCFGHunyuanAttnProcessor2_0, PAGHunyuanAttnProcessor2_0
28
+ from ...models.embeddings import get_2d_rotary_pos_embed
29
+ from ...pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker
30
+ from ...schedulers import DDPMScheduler
31
+ from ...utils import (
32
+ is_torch_xla_available,
33
+ logging,
34
+ replace_example_docstring,
35
+ )
36
+ from ...utils.torch_utils import randn_tensor
37
+ from ..pipeline_utils import DiffusionPipeline
38
+ from .pag_utils import PAGMixin
39
+
40
+
41
+ if is_torch_xla_available():
42
+ import torch_xla.core.xla_model as xm
43
+
44
+ XLA_AVAILABLE = True
45
+ else:
46
+ XLA_AVAILABLE = False
47
+
48
+
49
+ logger = logging.get_logger(__name__) # pylint: disable=invalid-name
50
+
51
+ EXAMPLE_DOC_STRING = """
52
+ Examples:
53
+ ```python
54
+ >>> import torch
55
+ >>> from diffusers import AutoPipelineForText2Image
56
+
57
+ >>> pipe = AutoPipelineForText2Image.from_pretrained(
58
+ ... "Tencent-Hunyuan/HunyuanDiT-v1.2-Diffusers",
59
+ ... torch_dtype=torch.float16,
60
+ ... enable_pag=True,
61
+ ... pag_applied_layers=[14],
62
+ ... ).to("cuda")
63
+
64
+ >>> # prompt = "an astronaut riding a horse"
65
+ >>> prompt = "一个宇航员在骑马"
66
+ >>> image = pipe(prompt, guidance_scale=4, pag_scale=3).images[0]
67
+ ```
68
+ """
69
+
70
+ STANDARD_RATIO = np.array(
71
+ [
72
+ 1.0, # 1:1
73
+ 4.0 / 3.0, # 4:3
74
+ 3.0 / 4.0, # 3:4
75
+ 16.0 / 9.0, # 16:9
76
+ 9.0 / 16.0, # 9:16
77
+ ]
78
+ )
79
+ STANDARD_SHAPE = [
80
+ [(1024, 1024), (1280, 1280)], # 1:1
81
+ [(1024, 768), (1152, 864), (1280, 960)], # 4:3
82
+ [(768, 1024), (864, 1152), (960, 1280)], # 3:4
83
+ [(1280, 768)], # 16:9
84
+ [(768, 1280)], # 9:16
85
+ ]
86
+ STANDARD_AREA = [np.array([w * h for w, h in shapes]) for shapes in STANDARD_SHAPE]
87
+ SUPPORTED_SHAPE = [
88
+ (1024, 1024),
89
+ (1280, 1280), # 1:1
90
+ (1024, 768),
91
+ (1152, 864),
92
+ (1280, 960), # 4:3
93
+ (768, 1024),
94
+ (864, 1152),
95
+ (960, 1280), # 3:4
96
+ (1280, 768), # 16:9
97
+ (768, 1280), # 9:16
98
+ ]
99
+
100
+
101
+ def map_to_standard_shapes(target_width, target_height):
102
+ target_ratio = target_width / target_height
103
+ closest_ratio_idx = np.argmin(np.abs(STANDARD_RATIO - target_ratio))
104
+ closest_area_idx = np.argmin(np.abs(STANDARD_AREA[closest_ratio_idx] - target_width * target_height))
105
+ width, height = STANDARD_SHAPE[closest_ratio_idx][closest_area_idx]
106
+ return width, height
107
+
108
+
109
+ def get_resize_crop_region_for_grid(src, tgt_size):
110
+ th = tw = tgt_size
111
+ h, w = src
112
+
113
+ r = h / w
114
+
115
+ # resize
116
+ if r > 1:
117
+ resize_height = th
118
+ resize_width = int(round(th / h * w))
119
+ else:
120
+ resize_width = tw
121
+ resize_height = int(round(tw / w * h))
122
+
123
+ crop_top = int(round((th - resize_height) / 2.0))
124
+ crop_left = int(round((tw - resize_width) / 2.0))
125
+
126
+ return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width)
127
+
128
+
129
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg
130
+ def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0):
131
+ r"""
132
+ Rescales `noise_cfg` tensor based on `guidance_rescale` to improve image quality and fix overexposure. Based on
133
+ Section 3.4 from [Common Diffusion Noise Schedules and Sample Steps are
134
+ Flawed](https://arxiv.org/pdf/2305.08891.pdf).
135
+
136
+ Args:
137
+ noise_cfg (`torch.Tensor`):
138
+ The predicted noise tensor for the guided diffusion process.
139
+ noise_pred_text (`torch.Tensor`):
140
+ The predicted noise tensor for the text-guided diffusion process.
141
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
142
+ A rescale factor applied to the noise predictions.
143
+
144
+ Returns:
145
+ noise_cfg (`torch.Tensor`): The rescaled noise prediction tensor.
146
+ """
147
+ std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True)
148
+ std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True)
149
+ # rescale the results from guidance (fixes overexposure)
150
+ noise_pred_rescaled = noise_cfg * (std_text / std_cfg)
151
+ # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images
152
+ noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg
153
+ return noise_cfg
154
+
155
+
156
+ class HunyuanDiTPAGPipeline(DiffusionPipeline, PAGMixin):
157
+ r"""
158
+ Pipeline for English/Chinese-to-image generation using HunyuanDiT and [Perturbed Attention
159
+ Guidance](https://huggingface.co/docs/diffusers/en/using-diffusers/pag).
160
+
161
+ This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the
162
+ library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.)
163
+
164
+ HunyuanDiT uses two text encoders: [mT5](https://huggingface.co/google/mt5-base) and [bilingual CLIP](fine-tuned by
165
+ ourselves)
166
+
167
+ Args:
168
+ vae ([`AutoencoderKL`]):
169
+ Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. We use
170
+ `sdxl-vae-fp16-fix`.
171
+ text_encoder (Optional[`~transformers.BertModel`, `~transformers.CLIPTextModel`]):
172
+ Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)).
173
+ HunyuanDiT uses a fine-tuned [bilingual CLIP].
174
+ tokenizer (Optional[`~transformers.BertTokenizer`, `~transformers.CLIPTokenizer`]):
175
+ A `BertTokenizer` or `CLIPTokenizer` to tokenize text.
176
+ transformer ([`HunyuanDiT2DModel`]):
177
+ The HunyuanDiT model designed by Tencent Hunyuan.
178
+ text_encoder_2 (`T5EncoderModel`):
179
+ The mT5 embedder. Specifically, it is 't5-v1_1-xxl'.
180
+ tokenizer_2 (`MT5Tokenizer`):
181
+ The tokenizer for the mT5 embedder.
182
+ scheduler ([`DDPMScheduler`]):
183
+ A scheduler to be used in combination with HunyuanDiT to denoise the encoded image latents.
184
+ """
185
+
186
+ model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae"
187
+ _optional_components = [
188
+ "safety_checker",
189
+ "feature_extractor",
190
+ "text_encoder_2",
191
+ "tokenizer_2",
192
+ "text_encoder",
193
+ "tokenizer",
194
+ ]
195
+ _exclude_from_cpu_offload = ["safety_checker"]
196
+ _callback_tensor_inputs = [
197
+ "latents",
198
+ "prompt_embeds",
199
+ "negative_prompt_embeds",
200
+ "prompt_embeds_2",
201
+ "negative_prompt_embeds_2",
202
+ ]
203
+
204
+ def __init__(
205
+ self,
206
+ vae: AutoencoderKL,
207
+ text_encoder: BertModel,
208
+ tokenizer: BertTokenizer,
209
+ transformer: HunyuanDiT2DModel,
210
+ scheduler: DDPMScheduler,
211
+ safety_checker: Optional[StableDiffusionSafetyChecker] = None,
212
+ feature_extractor: Optional[CLIPImageProcessor] = None,
213
+ requires_safety_checker: bool = True,
214
+ text_encoder_2: Optional[T5EncoderModel] = None,
215
+ tokenizer_2: Optional[MT5Tokenizer] = None,
216
+ pag_applied_layers: Union[str, List[str]] = "blocks.1", # "blocks.16.attn1", "blocks.16", "16", 16
217
+ ):
218
+ super().__init__()
219
+
220
+ self.register_modules(
221
+ vae=vae,
222
+ text_encoder=text_encoder,
223
+ tokenizer=tokenizer,
224
+ tokenizer_2=tokenizer_2,
225
+ transformer=transformer,
226
+ scheduler=scheduler,
227
+ safety_checker=safety_checker,
228
+ feature_extractor=feature_extractor,
229
+ text_encoder_2=text_encoder_2,
230
+ )
231
+
232
+ if safety_checker is None and requires_safety_checker:
233
+ logger.warning(
234
+ f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure"
235
+ " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered"
236
+ " results in services or applications open to the public. Both the diffusers team and Hugging Face"
237
+ " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling"
238
+ " it only for use-cases that involve analyzing network behavior or auditing its results. For more"
239
+ " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ."
240
+ )
241
+
242
+ if safety_checker is not None and feature_extractor is None:
243
+ raise ValueError(
244
+ "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
245
+ " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
246
+ )
247
+
248
+ self.vae_scale_factor = (
249
+ 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
250
+ )
251
+ self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
252
+ self.register_to_config(requires_safety_checker=requires_safety_checker)
253
+ self.default_sample_size = (
254
+ self.transformer.config.sample_size
255
+ if hasattr(self, "transformer") and self.transformer is not None
256
+ else 128
257
+ )
258
+
259
+ self.set_pag_applied_layers(
260
+ pag_applied_layers, pag_attn_processors=(PAGCFGHunyuanAttnProcessor2_0(), PAGHunyuanAttnProcessor2_0())
261
+ )
262
+
263
+ # Copied from diffusers.pipelines.hunyuandit.pipeline_hunyuandit.HunyuanDiTPipeline.encode_prompt
264
+ def encode_prompt(
265
+ self,
266
+ prompt: str,
267
+ device: torch.device = None,
268
+ dtype: torch.dtype = None,
269
+ num_images_per_prompt: int = 1,
270
+ do_classifier_free_guidance: bool = True,
271
+ negative_prompt: Optional[str] = None,
272
+ prompt_embeds: Optional[torch.Tensor] = None,
273
+ negative_prompt_embeds: Optional[torch.Tensor] = None,
274
+ prompt_attention_mask: Optional[torch.Tensor] = None,
275
+ negative_prompt_attention_mask: Optional[torch.Tensor] = None,
276
+ max_sequence_length: Optional[int] = None,
277
+ text_encoder_index: int = 0,
278
+ ):
279
+ r"""
280
+ Encodes the prompt into text encoder hidden states.
281
+
282
+ Args:
283
+ prompt (`str` or `List[str]`, *optional*):
284
+ prompt to be encoded
285
+ device: (`torch.device`):
286
+ torch device
287
+ dtype (`torch.dtype`):
288
+ torch dtype
289
+ num_images_per_prompt (`int`):
290
+ number of images that should be generated per prompt
291
+ do_classifier_free_guidance (`bool`):
292
+ whether to use classifier free guidance or not
293
+ negative_prompt (`str` or `List[str]`, *optional*):
294
+ The prompt or prompts not to guide the image generation. If not defined, one has to pass
295
+ `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is
296
+ less than `1`).
297
+ prompt_embeds (`torch.Tensor`, *optional*):
298
+ Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not
299
+ provided, text embeddings will be generated from `prompt` input argument.
300
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
301
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt
302
+ weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input
303
+ argument.
304
+ prompt_attention_mask (`torch.Tensor`, *optional*):
305
+ Attention mask for the prompt. Required when `prompt_embeds` is passed directly.
306
+ negative_prompt_attention_mask (`torch.Tensor`, *optional*):
307
+ Attention mask for the negative prompt. Required when `negative_prompt_embeds` is passed directly.
308
+ max_sequence_length (`int`, *optional*): maximum sequence length to use for the prompt.
309
+ text_encoder_index (`int`, *optional*):
310
+ Index of the text encoder to use. `0` for clip and `1` for T5.
311
+ """
312
+ if dtype is None:
313
+ if self.text_encoder_2 is not None:
314
+ dtype = self.text_encoder_2.dtype
315
+ elif self.transformer is not None:
316
+ dtype = self.transformer.dtype
317
+ else:
318
+ dtype = None
319
+
320
+ if device is None:
321
+ device = self._execution_device
322
+
323
+ tokenizers = [self.tokenizer, self.tokenizer_2]
324
+ text_encoders = [self.text_encoder, self.text_encoder_2]
325
+
326
+ tokenizer = tokenizers[text_encoder_index]
327
+ text_encoder = text_encoders[text_encoder_index]
328
+
329
+ if max_sequence_length is None:
330
+ if text_encoder_index == 0:
331
+ max_length = 77
332
+ if text_encoder_index == 1:
333
+ max_length = 256
334
+ else:
335
+ max_length = max_sequence_length
336
+
337
+ if prompt is not None and isinstance(prompt, str):
338
+ batch_size = 1
339
+ elif prompt is not None and isinstance(prompt, list):
340
+ batch_size = len(prompt)
341
+ else:
342
+ batch_size = prompt_embeds.shape[0]
343
+
344
+ if prompt_embeds is None:
345
+ text_inputs = tokenizer(
346
+ prompt,
347
+ padding="max_length",
348
+ max_length=max_length,
349
+ truncation=True,
350
+ return_attention_mask=True,
351
+ return_tensors="pt",
352
+ )
353
+ text_input_ids = text_inputs.input_ids
354
+ untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids
355
+
356
+ if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(
357
+ text_input_ids, untruncated_ids
358
+ ):
359
+ removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1])
360
+ logger.warning(
361
+ "The following part of your input was truncated because CLIP can only handle sequences up to"
362
+ f" {tokenizer.model_max_length} tokens: {removed_text}"
363
+ )
364
+
365
+ prompt_attention_mask = text_inputs.attention_mask.to(device)
366
+ prompt_embeds = text_encoder(
367
+ text_input_ids.to(device),
368
+ attention_mask=prompt_attention_mask,
369
+ )
370
+ prompt_embeds = prompt_embeds[0]
371
+ prompt_attention_mask = prompt_attention_mask.repeat(num_images_per_prompt, 1)
372
+
373
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
374
+
375
+ bs_embed, seq_len, _ = prompt_embeds.shape
376
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
377
+ prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1)
378
+ prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1)
379
+
380
+ # get unconditional embeddings for classifier free guidance
381
+ if do_classifier_free_guidance and negative_prompt_embeds is None:
382
+ uncond_tokens: List[str]
383
+ if negative_prompt is None:
384
+ uncond_tokens = [""] * batch_size
385
+ elif prompt is not None and type(prompt) is not type(negative_prompt):
386
+ raise TypeError(
387
+ f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !="
388
+ f" {type(prompt)}."
389
+ )
390
+ elif isinstance(negative_prompt, str):
391
+ uncond_tokens = [negative_prompt]
392
+ elif batch_size != len(negative_prompt):
393
+ raise ValueError(
394
+ f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:"
395
+ f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches"
396
+ " the batch size of `prompt`."
397
+ )
398
+ else:
399
+ uncond_tokens = negative_prompt
400
+
401
+ max_length = prompt_embeds.shape[1]
402
+ uncond_input = tokenizer(
403
+ uncond_tokens,
404
+ padding="max_length",
405
+ max_length=max_length,
406
+ truncation=True,
407
+ return_tensors="pt",
408
+ )
409
+
410
+ negative_prompt_attention_mask = uncond_input.attention_mask.to(device)
411
+ negative_prompt_embeds = text_encoder(
412
+ uncond_input.input_ids.to(device),
413
+ attention_mask=negative_prompt_attention_mask,
414
+ )
415
+ negative_prompt_embeds = negative_prompt_embeds[0]
416
+ negative_prompt_attention_mask = negative_prompt_attention_mask.repeat(num_images_per_prompt, 1)
417
+
418
+ if do_classifier_free_guidance:
419
+ # duplicate unconditional embeddings for each generation per prompt, using mps friendly method
420
+ seq_len = negative_prompt_embeds.shape[1]
421
+
422
+ negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device)
423
+
424
+ negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1)
425
+ negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1)
426
+
427
+ return prompt_embeds, negative_prompt_embeds, prompt_attention_mask, negative_prompt_attention_mask
428
+
429
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker
430
+ def run_safety_checker(self, image, device, dtype):
431
+ if self.safety_checker is None:
432
+ has_nsfw_concept = None
433
+ else:
434
+ if torch.is_tensor(image):
435
+ feature_extractor_input = self.image_processor.postprocess(image, output_type="pil")
436
+ else:
437
+ feature_extractor_input = self.image_processor.numpy_to_pil(image)
438
+ safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device)
439
+ image, has_nsfw_concept = self.safety_checker(
440
+ images=image, clip_input=safety_checker_input.pixel_values.to(dtype)
441
+ )
442
+ return image, has_nsfw_concept
443
+
444
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs
445
+ def prepare_extra_step_kwargs(self, generator, eta):
446
+ # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature
447
+ # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
448
+ # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
449
+ # and should be between [0, 1]
450
+
451
+ accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
452
+ extra_step_kwargs = {}
453
+ if accepts_eta:
454
+ extra_step_kwargs["eta"] = eta
455
+
456
+ # check if the scheduler accepts generator
457
+ accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
458
+ if accepts_generator:
459
+ extra_step_kwargs["generator"] = generator
460
+ return extra_step_kwargs
461
+
462
+ # Copied from diffusers.pipelines.hunyuandit.pipeline_hunyuandit.HunyuanDiTPipeline.check_inputs
463
+ def check_inputs(
464
+ self,
465
+ prompt,
466
+ height,
467
+ width,
468
+ negative_prompt=None,
469
+ prompt_embeds=None,
470
+ negative_prompt_embeds=None,
471
+ prompt_attention_mask=None,
472
+ negative_prompt_attention_mask=None,
473
+ prompt_embeds_2=None,
474
+ negative_prompt_embeds_2=None,
475
+ prompt_attention_mask_2=None,
476
+ negative_prompt_attention_mask_2=None,
477
+ callback_on_step_end_tensor_inputs=None,
478
+ ):
479
+ if height % 8 != 0 or width % 8 != 0:
480
+ raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
481
+
482
+ if callback_on_step_end_tensor_inputs is not None and not all(
483
+ k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs
484
+ ):
485
+ raise ValueError(
486
+ f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}"
487
+ )
488
+
489
+ if prompt is not None and prompt_embeds is not None:
490
+ raise ValueError(
491
+ f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to"
492
+ " only forward one of the two."
493
+ )
494
+ elif prompt is None and prompt_embeds is None:
495
+ raise ValueError(
496
+ "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined."
497
+ )
498
+ elif prompt is None and prompt_embeds_2 is None:
499
+ raise ValueError(
500
+ "Provide either `prompt` or `prompt_embeds_2`. Cannot leave both `prompt` and `prompt_embeds_2` undefined."
501
+ )
502
+ elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)):
503
+ raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}")
504
+
505
+ if prompt_embeds is not None and prompt_attention_mask is None:
506
+ raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.")
507
+
508
+ if prompt_embeds_2 is not None and prompt_attention_mask_2 is None:
509
+ raise ValueError("Must provide `prompt_attention_mask_2` when specifying `prompt_embeds_2`.")
510
+
511
+ if negative_prompt is not None and negative_prompt_embeds is not None:
512
+ raise ValueError(
513
+ f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:"
514
+ f" {negative_prompt_embeds}. Please make sure to only forward one of the two."
515
+ )
516
+
517
+ if negative_prompt_embeds is not None and negative_prompt_attention_mask is None:
518
+ raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.")
519
+
520
+ if negative_prompt_embeds_2 is not None and negative_prompt_attention_mask_2 is None:
521
+ raise ValueError(
522
+ "Must provide `negative_prompt_attention_mask_2` when specifying `negative_prompt_embeds_2`."
523
+ )
524
+ if prompt_embeds is not None and negative_prompt_embeds is not None:
525
+ if prompt_embeds.shape != negative_prompt_embeds.shape:
526
+ raise ValueError(
527
+ "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but"
528
+ f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`"
529
+ f" {negative_prompt_embeds.shape}."
530
+ )
531
+ if prompt_embeds_2 is not None and negative_prompt_embeds_2 is not None:
532
+ if prompt_embeds_2.shape != negative_prompt_embeds_2.shape:
533
+ raise ValueError(
534
+ "`prompt_embeds_2` and `negative_prompt_embeds_2` must have the same shape when passed directly, but"
535
+ f" got: `prompt_embeds_2` {prompt_embeds_2.shape} != `negative_prompt_embeds_2`"
536
+ f" {negative_prompt_embeds_2.shape}."
537
+ )
538
+
539
+ # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
540
+ def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
541
+ shape = (
542
+ batch_size,
543
+ num_channels_latents,
544
+ int(height) // self.vae_scale_factor,
545
+ int(width) // self.vae_scale_factor,
546
+ )
547
+ if isinstance(generator, list) and len(generator) != batch_size:
548
+ raise ValueError(
549
+ f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
550
+ f" size of {batch_size}. Make sure the batch size matches the length of the generators."
551
+ )
552
+
553
+ if latents is None:
554
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
555
+ else:
556
+ latents = latents.to(device)
557
+
558
+ # scale the initial noise by the standard deviation required by the scheduler
559
+ latents = latents * self.scheduler.init_noise_sigma
560
+ return latents
561
+
562
+ @property
563
+ def guidance_scale(self):
564
+ return self._guidance_scale
565
+
566
+ @property
567
+ def guidance_rescale(self):
568
+ return self._guidance_rescale
569
+
570
+ # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
571
+ # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
572
+ # corresponds to doing no classifier free guidance.
573
+ @property
574
+ def do_classifier_free_guidance(self):
575
+ return self._guidance_scale > 1
576
+
577
+ @property
578
+ def num_timesteps(self):
579
+ return self._num_timesteps
580
+
581
+ @property
582
+ def interrupt(self):
583
+ return self._interrupt
584
+
585
+ @torch.no_grad()
586
+ @replace_example_docstring(EXAMPLE_DOC_STRING)
587
+ def __call__(
588
+ self,
589
+ prompt: Union[str, List[str]] = None,
590
+ height: Optional[int] = None,
591
+ width: Optional[int] = None,
592
+ num_inference_steps: Optional[int] = 50,
593
+ guidance_scale: Optional[float] = 5.0,
594
+ negative_prompt: Optional[Union[str, List[str]]] = None,
595
+ num_images_per_prompt: Optional[int] = 1,
596
+ eta: Optional[float] = 0.0,
597
+ generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
598
+ latents: Optional[torch.Tensor] = None,
599
+ prompt_embeds: Optional[torch.Tensor] = None,
600
+ prompt_embeds_2: Optional[torch.Tensor] = None,
601
+ negative_prompt_embeds: Optional[torch.Tensor] = None,
602
+ negative_prompt_embeds_2: Optional[torch.Tensor] = None,
603
+ prompt_attention_mask: Optional[torch.Tensor] = None,
604
+ prompt_attention_mask_2: Optional[torch.Tensor] = None,
605
+ negative_prompt_attention_mask: Optional[torch.Tensor] = None,
606
+ negative_prompt_attention_mask_2: Optional[torch.Tensor] = None,
607
+ output_type: Optional[str] = "pil",
608
+ return_dict: bool = True,
609
+ callback_on_step_end: Optional[
610
+ Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks]
611
+ ] = None,
612
+ callback_on_step_end_tensor_inputs: List[str] = ["latents"],
613
+ guidance_rescale: float = 0.0,
614
+ original_size: Optional[Tuple[int, int]] = (1024, 1024),
615
+ target_size: Optional[Tuple[int, int]] = None,
616
+ crops_coords_top_left: Tuple[int, int] = (0, 0),
617
+ use_resolution_binning: bool = True,
618
+ pag_scale: float = 3.0,
619
+ pag_adaptive_scale: float = 0.0,
620
+ ):
621
+ r"""
622
+ The call function to the pipeline for generation with HunyuanDiT.
623
+
624
+ Args:
625
+ prompt (`str` or `List[str]`, *optional*):
626
+ The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`.
627
+ height (`int`):
628
+ The height in pixels of the generated image.
629
+ width (`int`):
630
+ The width in pixels of the generated image.
631
+ num_inference_steps (`int`, *optional*, defaults to 50):
632
+ The number of denoising steps. More denoising steps usually lead to a higher quality image at the
633
+ expense of slower inference. This parameter is modulated by `strength`.
634
+ guidance_scale (`float`, *optional*, defaults to 7.5):
635
+ A higher guidance scale value encourages the model to generate images closely linked to the text
636
+ `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`.
637
+ negative_prompt (`str` or `List[str]`, *optional*):
638
+ The prompt or prompts to guide what to not include in image generation. If not defined, you need to
639
+ pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`).
640
+ num_images_per_prompt (`int`, *optional*, defaults to 1):
641
+ The number of images to generate per prompt.
642
+ eta (`float`, *optional*, defaults to 0.0):
643
+ Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies
644
+ to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers.
645
+ generator (`torch.Generator` or `List[torch.Generator]`, *optional*):
646
+ A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make
647
+ generation deterministic.
648
+ prompt_embeds (`torch.Tensor`, *optional*):
649
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
650
+ provided, text embeddings are generated from the `prompt` input argument.
651
+ prompt_embeds_2 (`torch.Tensor`, *optional*):
652
+ Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not
653
+ provided, text embeddings are generated from the `prompt` input argument.
654
+ negative_prompt_embeds (`torch.Tensor`, *optional*):
655
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
656
+ not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
657
+ negative_prompt_embeds_2 (`torch.Tensor`, *optional*):
658
+ Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If
659
+ not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument.
660
+ prompt_attention_mask (`torch.Tensor`, *optional*):
661
+ Attention mask for the prompt. Required when `prompt_embeds` is passed directly.
662
+ prompt_attention_mask_2 (`torch.Tensor`, *optional*):
663
+ Attention mask for the prompt. Required when `prompt_embeds_2` is passed directly.
664
+ negative_prompt_attention_mask (`torch.Tensor`, *optional*):
665
+ Attention mask for the negative prompt. Required when `negative_prompt_embeds` is passed directly.
666
+ negative_prompt_attention_mask_2 (`torch.Tensor`, *optional*):
667
+ Attention mask for the negative prompt. Required when `negative_prompt_embeds_2` is passed directly.
668
+ output_type (`str`, *optional*, defaults to `"pil"`):
669
+ The output format of the generated image. Choose between `PIL.Image` or `np.array`.
670
+ return_dict (`bool`, *optional*, defaults to `True`):
671
+ Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a
672
+ plain tuple.
673
+ callback_on_step_end (`Callable[[int, int, Dict], None]`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*):
674
+ A callback function or a list of callback functions to be called at the end of each denoising step.
675
+ callback_on_step_end_tensor_inputs (`List[str]`, *optional*):
676
+ A list of tensor inputs that should be passed to the callback function. If not defined, all tensor
677
+ inputs will be passed.
678
+ guidance_rescale (`float`, *optional*, defaults to 0.0):
679
+ Rescale the noise_cfg according to `guidance_rescale`. Based on findings of [Common Diffusion Noise
680
+ Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4
681
+ original_size (`Tuple[int, int]`, *optional*, defaults to `(1024, 1024)`):
682
+ The original size of the image. Used to calculate the time ids.
683
+ target_size (`Tuple[int, int]`, *optional*):
684
+ The target size of the image. Used to calculate the time ids.
685
+ crops_coords_top_left (`Tuple[int, int]`, *optional*, defaults to `(0, 0)`):
686
+ The top left coordinates of the crop. Used to calculate the time ids.
687
+ use_resolution_binning (`bool`, *optional*, defaults to `True`):
688
+ Whether to use resolution binning or not. If `True`, the input resolution will be mapped to the closest
689
+ standard resolution. Supported resolutions are 1024x1024, 1280x1280, 1024x768, 1152x864, 1280x960,
690
+ 768x1024, 864x1152, 960x1280, 1280x768, and 768x1280. It is recommended to set this to `True`.
691
+ pag_scale (`float`, *optional*, defaults to 3.0):
692
+ The scale factor for the perturbed attention guidance. If it is set to 0.0, the perturbed attention
693
+ guidance will not be used.
694
+ pag_adaptive_scale (`float`, *optional*, defaults to 0.0):
695
+ The adaptive scale factor for the perturbed attention guidance. If it is set to 0.0, `pag_scale` is
696
+ used.
697
+
698
+ Examples:
699
+
700
+ Returns:
701
+ [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`:
702
+ If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned,
703
+ otherwise a `tuple` is returned where the first element is a list with the generated images and the
704
+ second element is a list of `bool`s indicating whether the corresponding generated image contains
705
+ "not-safe-for-work" (nsfw) content.
706
+ """
707
+
708
+ if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):
709
+ callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs
710
+
711
+ # 0. Default height and width
712
+ height = height or self.default_sample_size * self.vae_scale_factor
713
+ width = width or self.default_sample_size * self.vae_scale_factor
714
+ height = int((height // 16) * 16)
715
+ width = int((width // 16) * 16)
716
+
717
+ if use_resolution_binning and (height, width) not in SUPPORTED_SHAPE:
718
+ width, height = map_to_standard_shapes(width, height)
719
+ height = int(height)
720
+ width = int(width)
721
+ logger.warning(f"Reshaped to (height, width)=({height}, {width}), Supported shapes are {SUPPORTED_SHAPE}")
722
+
723
+ # 1. Check inputs. Raise error if not correct
724
+ self.check_inputs(
725
+ prompt,
726
+ height,
727
+ width,
728
+ negative_prompt,
729
+ prompt_embeds,
730
+ negative_prompt_embeds,
731
+ prompt_attention_mask,
732
+ negative_prompt_attention_mask,
733
+ prompt_embeds_2,
734
+ negative_prompt_embeds_2,
735
+ prompt_attention_mask_2,
736
+ negative_prompt_attention_mask_2,
737
+ callback_on_step_end_tensor_inputs,
738
+ )
739
+ self._guidance_scale = guidance_scale
740
+ self._guidance_rescale = guidance_rescale
741
+ self._interrupt = False
742
+ self._pag_scale = pag_scale
743
+ self._pag_adaptive_scale = pag_adaptive_scale
744
+
745
+ # 2. Define call parameters
746
+ if prompt is not None and isinstance(prompt, str):
747
+ batch_size = 1
748
+ elif prompt is not None and isinstance(prompt, list):
749
+ batch_size = len(prompt)
750
+ else:
751
+ batch_size = prompt_embeds.shape[0]
752
+
753
+ device = self._execution_device
754
+
755
+ # 3. Encode input prompt
756
+ (
757
+ prompt_embeds,
758
+ negative_prompt_embeds,
759
+ prompt_attention_mask,
760
+ negative_prompt_attention_mask,
761
+ ) = self.encode_prompt(
762
+ prompt=prompt,
763
+ device=device,
764
+ dtype=self.transformer.dtype,
765
+ num_images_per_prompt=num_images_per_prompt,
766
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
767
+ negative_prompt=negative_prompt,
768
+ prompt_embeds=prompt_embeds,
769
+ negative_prompt_embeds=negative_prompt_embeds,
770
+ prompt_attention_mask=prompt_attention_mask,
771
+ negative_prompt_attention_mask=negative_prompt_attention_mask,
772
+ max_sequence_length=77,
773
+ text_encoder_index=0,
774
+ )
775
+ (
776
+ prompt_embeds_2,
777
+ negative_prompt_embeds_2,
778
+ prompt_attention_mask_2,
779
+ negative_prompt_attention_mask_2,
780
+ ) = self.encode_prompt(
781
+ prompt=prompt,
782
+ device=device,
783
+ dtype=self.transformer.dtype,
784
+ num_images_per_prompt=num_images_per_prompt,
785
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
786
+ negative_prompt=negative_prompt,
787
+ prompt_embeds=prompt_embeds_2,
788
+ negative_prompt_embeds=negative_prompt_embeds_2,
789
+ prompt_attention_mask=prompt_attention_mask_2,
790
+ negative_prompt_attention_mask=negative_prompt_attention_mask_2,
791
+ max_sequence_length=256,
792
+ text_encoder_index=1,
793
+ )
794
+
795
+ # 4. Prepare timesteps
796
+ self.scheduler.set_timesteps(num_inference_steps, device=device)
797
+ timesteps = self.scheduler.timesteps
798
+
799
+ # 5. Prepare latent variables
800
+ num_channels_latents = self.transformer.config.in_channels
801
+ latents = self.prepare_latents(
802
+ batch_size * num_images_per_prompt,
803
+ num_channels_latents,
804
+ height,
805
+ width,
806
+ prompt_embeds.dtype,
807
+ device,
808
+ generator,
809
+ latents,
810
+ )
811
+
812
+ # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline
813
+ extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta)
814
+
815
+ # 7. Create image_rotary_emb, style embedding & time ids
816
+ grid_height = height // 8 // self.transformer.config.patch_size
817
+ grid_width = width // 8 // self.transformer.config.patch_size
818
+ base_size = 512 // 8 // self.transformer.config.patch_size
819
+ grid_crops_coords = get_resize_crop_region_for_grid((grid_height, grid_width), base_size)
820
+ image_rotary_emb = get_2d_rotary_pos_embed(
821
+ self.transformer.inner_dim // self.transformer.num_heads,
822
+ grid_crops_coords,
823
+ (grid_height, grid_width),
824
+ device=device,
825
+ output_type="pt",
826
+ )
827
+
828
+ style = torch.tensor([0], device=device)
829
+
830
+ target_size = target_size or (height, width)
831
+ add_time_ids = list(original_size + target_size + crops_coords_top_left)
832
+ add_time_ids = torch.tensor([add_time_ids], dtype=prompt_embeds.dtype)
833
+
834
+ # For classifier free guidance, we need to do two forward passes.
835
+ # Here we concatenate the unconditional and text embeddings into a single batch
836
+ # to avoid doing two forward passes
837
+ if self.do_perturbed_attention_guidance:
838
+ prompt_embeds = self._prepare_perturbed_attention_guidance(
839
+ prompt_embeds, negative_prompt_embeds, self.do_classifier_free_guidance
840
+ )
841
+ prompt_attention_mask = self._prepare_perturbed_attention_guidance(
842
+ prompt_attention_mask, negative_prompt_attention_mask, self.do_classifier_free_guidance
843
+ )
844
+ prompt_embeds_2 = self._prepare_perturbed_attention_guidance(
845
+ prompt_embeds_2, negative_prompt_embeds_2, self.do_classifier_free_guidance
846
+ )
847
+ prompt_attention_mask_2 = self._prepare_perturbed_attention_guidance(
848
+ prompt_attention_mask_2, negative_prompt_attention_mask_2, self.do_classifier_free_guidance
849
+ )
850
+ add_time_ids = torch.cat([add_time_ids] * 3, dim=0)
851
+ style = torch.cat([style] * 3, dim=0)
852
+ elif self.do_classifier_free_guidance:
853
+ prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds])
854
+ prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask])
855
+ prompt_embeds_2 = torch.cat([negative_prompt_embeds_2, prompt_embeds_2])
856
+ prompt_attention_mask_2 = torch.cat([negative_prompt_attention_mask_2, prompt_attention_mask_2])
857
+ add_time_ids = torch.cat([add_time_ids] * 2, dim=0)
858
+ style = torch.cat([style] * 2, dim=0)
859
+
860
+ prompt_embeds = prompt_embeds.to(device=device)
861
+ prompt_attention_mask = prompt_attention_mask.to(device=device)
862
+ prompt_embeds_2 = prompt_embeds_2.to(device=device)
863
+ prompt_attention_mask_2 = prompt_attention_mask_2.to(device=device)
864
+ add_time_ids = add_time_ids.to(dtype=prompt_embeds.dtype, device=device).repeat(
865
+ batch_size * num_images_per_prompt, 1
866
+ )
867
+ style = style.to(device=device).repeat(batch_size * num_images_per_prompt)
868
+
869
+ # 8. Denoising loop
870
+ num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order
871
+ self._num_timesteps = len(timesteps)
872
+
873
+ if self.do_perturbed_attention_guidance:
874
+ original_attn_proc = self.transformer.attn_processors
875
+ self._set_pag_attn_processor(
876
+ pag_applied_layers=self.pag_applied_layers,
877
+ do_classifier_free_guidance=self.do_classifier_free_guidance,
878
+ )
879
+
880
+ with self.progress_bar(total=num_inference_steps) as progress_bar:
881
+ for i, t in enumerate(timesteps):
882
+ if self.interrupt:
883
+ continue
884
+
885
+ # expand the latents if we are doing classifier free guidance
886
+ latent_model_input = torch.cat([latents] * (prompt_embeds.shape[0] // latents.shape[0]))
887
+ latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
888
+
889
+ # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input
890
+ t_expand = torch.tensor([t] * latent_model_input.shape[0], device=device).to(
891
+ dtype=latent_model_input.dtype
892
+ )
893
+
894
+ # predict the noise residual
895
+ noise_pred = self.transformer(
896
+ latent_model_input,
897
+ t_expand,
898
+ encoder_hidden_states=prompt_embeds,
899
+ text_embedding_mask=prompt_attention_mask,
900
+ encoder_hidden_states_t5=prompt_embeds_2,
901
+ text_embedding_mask_t5=prompt_attention_mask_2,
902
+ image_meta_size=add_time_ids,
903
+ style=style,
904
+ image_rotary_emb=image_rotary_emb,
905
+ return_dict=False,
906
+ )[0]
907
+
908
+ noise_pred, _ = noise_pred.chunk(2, dim=1)
909
+
910
+ # perform guidance
911
+ if self.do_perturbed_attention_guidance:
912
+ noise_pred, noise_pred_text = self._apply_perturbed_attention_guidance(
913
+ noise_pred, self.do_classifier_free_guidance, self.guidance_scale, t, True
914
+ )
915
+ elif self.do_classifier_free_guidance:
916
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
917
+ noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
918
+
919
+ if self.do_classifier_free_guidance and guidance_rescale > 0.0:
920
+ # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf
921
+ noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale)
922
+
923
+ # compute the previous noisy sample x_t -> x_t-1
924
+ latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0]
925
+
926
+ if callback_on_step_end is not None:
927
+ callback_kwargs = {}
928
+ for k in callback_on_step_end_tensor_inputs:
929
+ callback_kwargs[k] = locals()[k]
930
+ callback_outputs = callback_on_step_end(self, i, t, callback_kwargs)
931
+
932
+ latents = callback_outputs.pop("latents", latents)
933
+ prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds)
934
+ negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds)
935
+ prompt_embeds_2 = callback_outputs.pop("prompt_embeds_2", prompt_embeds_2)
936
+ negative_prompt_embeds_2 = callback_outputs.pop(
937
+ "negative_prompt_embeds_2", negative_prompt_embeds_2
938
+ )
939
+
940
+ if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
941
+ progress_bar.update()
942
+
943
+ if XLA_AVAILABLE:
944
+ xm.mark_step()
945
+
946
+ if not output_type == "latent":
947
+ image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0]
948
+ image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype)
949
+ else:
950
+ image = latents
951
+ has_nsfw_concept = None
952
+
953
+ if has_nsfw_concept is None:
954
+ do_denormalize = [True] * image.shape[0]
955
+ else:
956
+ do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept]
957
+
958
+ image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize)
959
+
960
+ # 9. Offload all models
961
+ self.maybe_free_model_hooks()
962
+
963
+ if self.do_perturbed_attention_guidance:
964
+ self.transformer.set_attn_processor(original_attn_proc)
965
+
966
+ if not return_dict:
967
+ return (image, has_nsfw_concept)
968
+
969
+ return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept)