diffusers 0.32.2__py3-none-any.whl → 0.33.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (389) hide show
  1. diffusers/__init__.py +186 -3
  2. diffusers/configuration_utils.py +40 -12
  3. diffusers/dependency_versions_table.py +9 -2
  4. diffusers/hooks/__init__.py +9 -0
  5. diffusers/hooks/faster_cache.py +653 -0
  6. diffusers/hooks/group_offloading.py +793 -0
  7. diffusers/hooks/hooks.py +236 -0
  8. diffusers/hooks/layerwise_casting.py +245 -0
  9. diffusers/hooks/pyramid_attention_broadcast.py +311 -0
  10. diffusers/loaders/__init__.py +6 -0
  11. diffusers/loaders/ip_adapter.py +38 -30
  12. diffusers/loaders/lora_base.py +121 -86
  13. diffusers/loaders/lora_conversion_utils.py +504 -44
  14. diffusers/loaders/lora_pipeline.py +1769 -181
  15. diffusers/loaders/peft.py +167 -57
  16. diffusers/loaders/single_file.py +17 -2
  17. diffusers/loaders/single_file_model.py +53 -5
  18. diffusers/loaders/single_file_utils.py +646 -72
  19. diffusers/loaders/textual_inversion.py +9 -9
  20. diffusers/loaders/transformer_flux.py +8 -9
  21. diffusers/loaders/transformer_sd3.py +120 -39
  22. diffusers/loaders/unet.py +20 -7
  23. diffusers/models/__init__.py +22 -0
  24. diffusers/models/activations.py +9 -9
  25. diffusers/models/attention.py +0 -1
  26. diffusers/models/attention_processor.py +163 -25
  27. diffusers/models/auto_model.py +169 -0
  28. diffusers/models/autoencoders/__init__.py +2 -0
  29. diffusers/models/autoencoders/autoencoder_asym_kl.py +2 -0
  30. diffusers/models/autoencoders/autoencoder_dc.py +106 -4
  31. diffusers/models/autoencoders/autoencoder_kl.py +0 -4
  32. diffusers/models/autoencoders/autoencoder_kl_allegro.py +5 -23
  33. diffusers/models/autoencoders/autoencoder_kl_cogvideox.py +17 -55
  34. diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +17 -97
  35. diffusers/models/autoencoders/autoencoder_kl_ltx.py +326 -107
  36. diffusers/models/autoencoders/autoencoder_kl_magvit.py +1094 -0
  37. diffusers/models/autoencoders/autoencoder_kl_mochi.py +21 -56
  38. diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +11 -42
  39. diffusers/models/autoencoders/autoencoder_kl_wan.py +855 -0
  40. diffusers/models/autoencoders/autoencoder_oobleck.py +1 -0
  41. diffusers/models/autoencoders/autoencoder_tiny.py +0 -4
  42. diffusers/models/autoencoders/consistency_decoder_vae.py +3 -1
  43. diffusers/models/autoencoders/vae.py +31 -141
  44. diffusers/models/autoencoders/vq_model.py +3 -0
  45. diffusers/models/cache_utils.py +108 -0
  46. diffusers/models/controlnets/__init__.py +1 -0
  47. diffusers/models/controlnets/controlnet.py +3 -8
  48. diffusers/models/controlnets/controlnet_flux.py +14 -42
  49. diffusers/models/controlnets/controlnet_sd3.py +58 -34
  50. diffusers/models/controlnets/controlnet_sparsectrl.py +4 -7
  51. diffusers/models/controlnets/controlnet_union.py +27 -18
  52. diffusers/models/controlnets/controlnet_xs.py +7 -46
  53. diffusers/models/controlnets/multicontrolnet_union.py +196 -0
  54. diffusers/models/embeddings.py +18 -7
  55. diffusers/models/model_loading_utils.py +122 -80
  56. diffusers/models/modeling_flax_pytorch_utils.py +1 -1
  57. diffusers/models/modeling_flax_utils.py +1 -1
  58. diffusers/models/modeling_pytorch_flax_utils.py +1 -1
  59. diffusers/models/modeling_utils.py +617 -272
  60. diffusers/models/normalization.py +67 -14
  61. diffusers/models/resnet.py +1 -1
  62. diffusers/models/transformers/__init__.py +6 -0
  63. diffusers/models/transformers/auraflow_transformer_2d.py +9 -35
  64. diffusers/models/transformers/cogvideox_transformer_3d.py +13 -24
  65. diffusers/models/transformers/consisid_transformer_3d.py +789 -0
  66. diffusers/models/transformers/dit_transformer_2d.py +5 -19
  67. diffusers/models/transformers/hunyuan_transformer_2d.py +4 -3
  68. diffusers/models/transformers/latte_transformer_3d.py +20 -15
  69. diffusers/models/transformers/lumina_nextdit2d.py +3 -1
  70. diffusers/models/transformers/pixart_transformer_2d.py +4 -19
  71. diffusers/models/transformers/prior_transformer.py +5 -1
  72. diffusers/models/transformers/sana_transformer.py +144 -40
  73. diffusers/models/transformers/stable_audio_transformer.py +5 -20
  74. diffusers/models/transformers/transformer_2d.py +7 -22
  75. diffusers/models/transformers/transformer_allegro.py +9 -17
  76. diffusers/models/transformers/transformer_cogview3plus.py +6 -17
  77. diffusers/models/transformers/transformer_cogview4.py +462 -0
  78. diffusers/models/transformers/transformer_easyanimate.py +527 -0
  79. diffusers/models/transformers/transformer_flux.py +68 -110
  80. diffusers/models/transformers/transformer_hunyuan_video.py +404 -46
  81. diffusers/models/transformers/transformer_ltx.py +53 -35
  82. diffusers/models/transformers/transformer_lumina2.py +548 -0
  83. diffusers/models/transformers/transformer_mochi.py +6 -17
  84. diffusers/models/transformers/transformer_omnigen.py +469 -0
  85. diffusers/models/transformers/transformer_sd3.py +56 -86
  86. diffusers/models/transformers/transformer_temporal.py +5 -11
  87. diffusers/models/transformers/transformer_wan.py +469 -0
  88. diffusers/models/unets/unet_1d.py +3 -1
  89. diffusers/models/unets/unet_2d.py +21 -20
  90. diffusers/models/unets/unet_2d_blocks.py +19 -243
  91. diffusers/models/unets/unet_2d_condition.py +4 -6
  92. diffusers/models/unets/unet_3d_blocks.py +14 -127
  93. diffusers/models/unets/unet_3d_condition.py +8 -12
  94. diffusers/models/unets/unet_i2vgen_xl.py +5 -13
  95. diffusers/models/unets/unet_kandinsky3.py +0 -4
  96. diffusers/models/unets/unet_motion_model.py +20 -114
  97. diffusers/models/unets/unet_spatio_temporal_condition.py +7 -8
  98. diffusers/models/unets/unet_stable_cascade.py +8 -35
  99. diffusers/models/unets/uvit_2d.py +1 -4
  100. diffusers/optimization.py +2 -2
  101. diffusers/pipelines/__init__.py +57 -8
  102. diffusers/pipelines/allegro/pipeline_allegro.py +22 -2
  103. diffusers/pipelines/amused/pipeline_amused.py +15 -2
  104. diffusers/pipelines/amused/pipeline_amused_img2img.py +15 -2
  105. diffusers/pipelines/amused/pipeline_amused_inpaint.py +15 -2
  106. diffusers/pipelines/animatediff/pipeline_animatediff.py +15 -2
  107. diffusers/pipelines/animatediff/pipeline_animatediff_controlnet.py +15 -3
  108. diffusers/pipelines/animatediff/pipeline_animatediff_sdxl.py +24 -4
  109. diffusers/pipelines/animatediff/pipeline_animatediff_sparsectrl.py +15 -2
  110. diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +16 -4
  111. diffusers/pipelines/animatediff/pipeline_animatediff_video2video_controlnet.py +16 -4
  112. diffusers/pipelines/audioldm/pipeline_audioldm.py +13 -2
  113. diffusers/pipelines/audioldm2/modeling_audioldm2.py +13 -68
  114. diffusers/pipelines/audioldm2/pipeline_audioldm2.py +39 -9
  115. diffusers/pipelines/aura_flow/pipeline_aura_flow.py +63 -7
  116. diffusers/pipelines/auto_pipeline.py +35 -14
  117. diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
  118. diffusers/pipelines/blip_diffusion/modeling_blip2.py +5 -8
  119. diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +12 -0
  120. diffusers/pipelines/cogvideo/pipeline_cogvideox.py +22 -6
  121. diffusers/pipelines/cogvideo/pipeline_cogvideox_fun_control.py +22 -6
  122. diffusers/pipelines/cogvideo/pipeline_cogvideox_image2video.py +22 -5
  123. diffusers/pipelines/cogvideo/pipeline_cogvideox_video2video.py +22 -6
  124. diffusers/pipelines/cogview3/pipeline_cogview3plus.py +12 -4
  125. diffusers/pipelines/cogview4/__init__.py +49 -0
  126. diffusers/pipelines/cogview4/pipeline_cogview4.py +684 -0
  127. diffusers/pipelines/cogview4/pipeline_cogview4_control.py +732 -0
  128. diffusers/pipelines/cogview4/pipeline_output.py +21 -0
  129. diffusers/pipelines/consisid/__init__.py +49 -0
  130. diffusers/pipelines/consisid/consisid_utils.py +357 -0
  131. diffusers/pipelines/consisid/pipeline_consisid.py +974 -0
  132. diffusers/pipelines/consisid/pipeline_output.py +20 -0
  133. diffusers/pipelines/consistency_models/pipeline_consistency_models.py +11 -0
  134. diffusers/pipelines/controlnet/pipeline_controlnet.py +6 -5
  135. diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +13 -0
  136. diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +17 -5
  137. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +31 -12
  138. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +26 -7
  139. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +20 -3
  140. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +22 -3
  141. diffusers/pipelines/controlnet/pipeline_controlnet_union_inpaint_sd_xl.py +26 -25
  142. diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl.py +224 -109
  143. diffusers/pipelines/controlnet/pipeline_controlnet_union_sd_xl_img2img.py +25 -29
  144. diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +7 -4
  145. diffusers/pipelines/controlnet_hunyuandit/pipeline_hunyuandit_controlnet.py +3 -5
  146. diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet.py +121 -10
  147. diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py +122 -11
  148. diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs.py +12 -1
  149. diffusers/pipelines/controlnet_xs/pipeline_controlnet_xs_sd_xl.py +20 -3
  150. diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +14 -2
  151. diffusers/pipelines/ddim/pipeline_ddim.py +14 -1
  152. diffusers/pipelines/ddpm/pipeline_ddpm.py +15 -1
  153. diffusers/pipelines/deepfloyd_if/pipeline_if.py +12 -0
  154. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +12 -0
  155. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +14 -1
  156. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +12 -0
  157. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +14 -1
  158. diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +14 -1
  159. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +11 -7
  160. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +11 -7
  161. diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +1 -1
  162. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +10 -6
  163. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py +2 -2
  164. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +11 -7
  165. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +1 -1
  166. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +1 -1
  167. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +1 -1
  168. diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +10 -105
  169. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py +1 -1
  170. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +1 -1
  171. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +1 -1
  172. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +1 -1
  173. diffusers/pipelines/dit/pipeline_dit.py +15 -2
  174. diffusers/pipelines/easyanimate/__init__.py +52 -0
  175. diffusers/pipelines/easyanimate/pipeline_easyanimate.py +770 -0
  176. diffusers/pipelines/easyanimate/pipeline_easyanimate_control.py +994 -0
  177. diffusers/pipelines/easyanimate/pipeline_easyanimate_inpaint.py +1234 -0
  178. diffusers/pipelines/easyanimate/pipeline_output.py +20 -0
  179. diffusers/pipelines/flux/pipeline_flux.py +53 -21
  180. diffusers/pipelines/flux/pipeline_flux_control.py +9 -12
  181. diffusers/pipelines/flux/pipeline_flux_control_img2img.py +6 -10
  182. diffusers/pipelines/flux/pipeline_flux_control_inpaint.py +8 -10
  183. diffusers/pipelines/flux/pipeline_flux_controlnet.py +185 -13
  184. diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py +8 -10
  185. diffusers/pipelines/flux/pipeline_flux_controlnet_inpainting.py +16 -16
  186. diffusers/pipelines/flux/pipeline_flux_fill.py +107 -39
  187. diffusers/pipelines/flux/pipeline_flux_img2img.py +193 -15
  188. diffusers/pipelines/flux/pipeline_flux_inpaint.py +199 -19
  189. diffusers/pipelines/free_noise_utils.py +3 -3
  190. diffusers/pipelines/hunyuan_video/__init__.py +4 -0
  191. diffusers/pipelines/hunyuan_video/pipeline_hunyuan_skyreels_image2video.py +804 -0
  192. diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +90 -23
  193. diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py +924 -0
  194. diffusers/pipelines/hunyuandit/pipeline_hunyuandit.py +3 -5
  195. diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +13 -1
  196. diffusers/pipelines/kandinsky/pipeline_kandinsky.py +12 -0
  197. diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +1 -1
  198. diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +12 -0
  199. diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +13 -1
  200. diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +12 -0
  201. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +12 -1
  202. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +13 -0
  203. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +12 -0
  204. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +12 -1
  205. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +12 -1
  206. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +12 -0
  207. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py +12 -0
  208. diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py +12 -0
  209. diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py +12 -0
  210. diffusers/pipelines/kolors/pipeline_kolors.py +10 -8
  211. diffusers/pipelines/kolors/pipeline_kolors_img2img.py +6 -4
  212. diffusers/pipelines/kolors/text_encoder.py +7 -34
  213. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +12 -1
  214. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +13 -1
  215. diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +14 -13
  216. diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py +12 -1
  217. diffusers/pipelines/latte/pipeline_latte.py +36 -7
  218. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +67 -13
  219. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +60 -15
  220. diffusers/pipelines/ltx/__init__.py +2 -0
  221. diffusers/pipelines/ltx/pipeline_ltx.py +25 -13
  222. diffusers/pipelines/ltx/pipeline_ltx_condition.py +1194 -0
  223. diffusers/pipelines/ltx/pipeline_ltx_image2video.py +31 -17
  224. diffusers/pipelines/lumina/__init__.py +2 -2
  225. diffusers/pipelines/lumina/pipeline_lumina.py +83 -20
  226. diffusers/pipelines/lumina2/__init__.py +48 -0
  227. diffusers/pipelines/lumina2/pipeline_lumina2.py +790 -0
  228. diffusers/pipelines/marigold/__init__.py +2 -0
  229. diffusers/pipelines/marigold/marigold_image_processing.py +127 -14
  230. diffusers/pipelines/marigold/pipeline_marigold_depth.py +31 -16
  231. diffusers/pipelines/marigold/pipeline_marigold_intrinsics.py +721 -0
  232. diffusers/pipelines/marigold/pipeline_marigold_normals.py +31 -16
  233. diffusers/pipelines/mochi/pipeline_mochi.py +14 -18
  234. diffusers/pipelines/musicldm/pipeline_musicldm.py +16 -1
  235. diffusers/pipelines/omnigen/__init__.py +50 -0
  236. diffusers/pipelines/omnigen/pipeline_omnigen.py +512 -0
  237. diffusers/pipelines/omnigen/processor_omnigen.py +327 -0
  238. diffusers/pipelines/onnx_utils.py +5 -3
  239. diffusers/pipelines/pag/pag_utils.py +1 -1
  240. diffusers/pipelines/pag/pipeline_pag_controlnet_sd.py +12 -1
  241. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_inpaint.py +15 -4
  242. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl.py +20 -3
  243. diffusers/pipelines/pag/pipeline_pag_controlnet_sd_xl_img2img.py +20 -3
  244. diffusers/pipelines/pag/pipeline_pag_hunyuandit.py +1 -3
  245. diffusers/pipelines/pag/pipeline_pag_kolors.py +6 -4
  246. diffusers/pipelines/pag/pipeline_pag_pixart_sigma.py +16 -3
  247. diffusers/pipelines/pag/pipeline_pag_sana.py +65 -8
  248. diffusers/pipelines/pag/pipeline_pag_sd.py +23 -7
  249. diffusers/pipelines/pag/pipeline_pag_sd_3.py +3 -5
  250. diffusers/pipelines/pag/pipeline_pag_sd_3_img2img.py +3 -5
  251. diffusers/pipelines/pag/pipeline_pag_sd_animatediff.py +13 -1
  252. diffusers/pipelines/pag/pipeline_pag_sd_img2img.py +23 -7
  253. diffusers/pipelines/pag/pipeline_pag_sd_inpaint.py +26 -10
  254. diffusers/pipelines/pag/pipeline_pag_sd_xl.py +12 -4
  255. diffusers/pipelines/pag/pipeline_pag_sd_xl_img2img.py +7 -3
  256. diffusers/pipelines/pag/pipeline_pag_sd_xl_inpaint.py +10 -6
  257. diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +13 -3
  258. diffusers/pipelines/pia/pipeline_pia.py +13 -1
  259. diffusers/pipelines/pipeline_flax_utils.py +7 -7
  260. diffusers/pipelines/pipeline_loading_utils.py +193 -83
  261. diffusers/pipelines/pipeline_utils.py +221 -106
  262. diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +17 -5
  263. diffusers/pipelines/pixart_alpha/pipeline_pixart_sigma.py +17 -4
  264. diffusers/pipelines/sana/__init__.py +2 -0
  265. diffusers/pipelines/sana/pipeline_sana.py +183 -58
  266. diffusers/pipelines/sana/pipeline_sana_sprint.py +889 -0
  267. diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +12 -2
  268. diffusers/pipelines/shap_e/pipeline_shap_e.py +12 -0
  269. diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +12 -0
  270. diffusers/pipelines/shap_e/renderer.py +6 -6
  271. diffusers/pipelines/stable_audio/pipeline_stable_audio.py +1 -1
  272. diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +15 -4
  273. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +12 -8
  274. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +12 -1
  275. diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +3 -2
  276. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +14 -10
  277. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py +3 -3
  278. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py +14 -10
  279. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +2 -2
  280. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +4 -3
  281. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +5 -4
  282. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +2 -2
  283. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +18 -13
  284. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +30 -8
  285. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +24 -10
  286. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +28 -12
  287. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +39 -18
  288. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +17 -6
  289. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +13 -3
  290. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +20 -3
  291. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +14 -2
  292. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +13 -1
  293. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +16 -17
  294. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +136 -18
  295. diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +150 -21
  296. diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +15 -3
  297. diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +26 -11
  298. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +15 -3
  299. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +22 -4
  300. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +30 -13
  301. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +12 -4
  302. diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +15 -3
  303. diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +15 -3
  304. diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +26 -12
  305. diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +16 -4
  306. diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py +1 -1
  307. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +12 -4
  308. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +7 -3
  309. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +10 -6
  310. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +11 -4
  311. diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +13 -2
  312. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +18 -4
  313. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +26 -5
  314. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +13 -1
  315. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +13 -1
  316. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +28 -6
  317. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +26 -4
  318. diffusers/pipelines/transformers_loading_utils.py +121 -0
  319. diffusers/pipelines/unclip/pipeline_unclip.py +11 -1
  320. diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +11 -1
  321. diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +19 -2
  322. diffusers/pipelines/wan/__init__.py +51 -0
  323. diffusers/pipelines/wan/pipeline_output.py +20 -0
  324. diffusers/pipelines/wan/pipeline_wan.py +595 -0
  325. diffusers/pipelines/wan/pipeline_wan_i2v.py +724 -0
  326. diffusers/pipelines/wan/pipeline_wan_video2video.py +727 -0
  327. diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +7 -31
  328. diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +12 -1
  329. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +12 -1
  330. diffusers/quantizers/auto.py +5 -1
  331. diffusers/quantizers/base.py +5 -9
  332. diffusers/quantizers/bitsandbytes/bnb_quantizer.py +41 -29
  333. diffusers/quantizers/bitsandbytes/utils.py +30 -20
  334. diffusers/quantizers/gguf/gguf_quantizer.py +1 -0
  335. diffusers/quantizers/gguf/utils.py +4 -2
  336. diffusers/quantizers/quantization_config.py +59 -4
  337. diffusers/quantizers/quanto/__init__.py +1 -0
  338. diffusers/quantizers/quanto/quanto_quantizer.py +177 -0
  339. diffusers/quantizers/quanto/utils.py +60 -0
  340. diffusers/quantizers/torchao/__init__.py +1 -1
  341. diffusers/quantizers/torchao/torchao_quantizer.py +47 -2
  342. diffusers/schedulers/__init__.py +2 -1
  343. diffusers/schedulers/scheduling_consistency_models.py +1 -2
  344. diffusers/schedulers/scheduling_ddim_inverse.py +1 -1
  345. diffusers/schedulers/scheduling_ddpm.py +2 -3
  346. diffusers/schedulers/scheduling_ddpm_parallel.py +1 -2
  347. diffusers/schedulers/scheduling_dpmsolver_multistep.py +12 -4
  348. diffusers/schedulers/scheduling_edm_euler.py +45 -10
  349. diffusers/schedulers/scheduling_flow_match_euler_discrete.py +116 -28
  350. diffusers/schedulers/scheduling_flow_match_heun_discrete.py +7 -6
  351. diffusers/schedulers/scheduling_heun_discrete.py +1 -1
  352. diffusers/schedulers/scheduling_lcm.py +1 -2
  353. diffusers/schedulers/scheduling_lms_discrete.py +1 -1
  354. diffusers/schedulers/scheduling_repaint.py +5 -1
  355. diffusers/schedulers/scheduling_scm.py +265 -0
  356. diffusers/schedulers/scheduling_tcd.py +1 -2
  357. diffusers/schedulers/scheduling_utils.py +2 -1
  358. diffusers/training_utils.py +14 -7
  359. diffusers/utils/__init__.py +9 -1
  360. diffusers/utils/constants.py +13 -1
  361. diffusers/utils/deprecation_utils.py +1 -1
  362. diffusers/utils/dummy_bitsandbytes_objects.py +17 -0
  363. diffusers/utils/dummy_gguf_objects.py +17 -0
  364. diffusers/utils/dummy_optimum_quanto_objects.py +17 -0
  365. diffusers/utils/dummy_pt_objects.py +233 -0
  366. diffusers/utils/dummy_torch_and_transformers_and_opencv_objects.py +17 -0
  367. diffusers/utils/dummy_torch_and_transformers_objects.py +270 -0
  368. diffusers/utils/dummy_torchao_objects.py +17 -0
  369. diffusers/utils/dynamic_modules_utils.py +1 -1
  370. diffusers/utils/export_utils.py +28 -3
  371. diffusers/utils/hub_utils.py +52 -102
  372. diffusers/utils/import_utils.py +121 -221
  373. diffusers/utils/loading_utils.py +2 -1
  374. diffusers/utils/logging.py +1 -2
  375. diffusers/utils/peft_utils.py +6 -14
  376. diffusers/utils/remote_utils.py +425 -0
  377. diffusers/utils/source_code_parsing_utils.py +52 -0
  378. diffusers/utils/state_dict_utils.py +15 -1
  379. diffusers/utils/testing_utils.py +243 -13
  380. diffusers/utils/torch_utils.py +10 -0
  381. diffusers/utils/typing_utils.py +91 -0
  382. diffusers/video_processor.py +1 -1
  383. {diffusers-0.32.2.dist-info → diffusers-0.33.1.dist-info}/METADATA +21 -4
  384. diffusers-0.33.1.dist-info/RECORD +608 -0
  385. {diffusers-0.32.2.dist-info → diffusers-0.33.1.dist-info}/WHEEL +1 -1
  386. diffusers-0.32.2.dist-info/RECORD +0 -550
  387. {diffusers-0.32.2.dist-info → diffusers-0.33.1.dist-info}/LICENSE +0 -0
  388. {diffusers-0.32.2.dist-info → diffusers-0.33.1.dist-info}/entry_points.txt +0 -0
  389. {diffusers-0.32.2.dist-info → diffusers-0.33.1.dist-info}/top_level.txt +0 -0
@@ -54,6 +54,16 @@ if is_invisible_watermark_available():
54
54
  from ..stable_diffusion_xl.watermark import StableDiffusionXLWatermarker
55
55
 
56
56
 
57
+ from ...utils import is_torch_xla_available
58
+
59
+
60
+ if is_torch_xla_available():
61
+ import torch_xla.core.xla_model as xm
62
+
63
+ XLA_AVAILABLE = True
64
+ else:
65
+ XLA_AVAILABLE = False
66
+
57
67
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
58
68
 
59
69
 
@@ -196,7 +206,7 @@ class StableDiffusionXLControlNetXSPipeline(
196
206
  scheduler=scheduler,
197
207
  feature_extractor=feature_extractor,
198
208
  )
199
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
209
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
200
210
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True)
201
211
  self.control_image_processor = VaeImageProcessor(
202
212
  vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False
@@ -336,7 +346,9 @@ class StableDiffusionXLControlNetXSPipeline(
336
346
  prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True)
337
347
 
338
348
  # We are only ALWAYS interested in the pooled output of the final text encoder
339
- pooled_prompt_embeds = prompt_embeds[0]
349
+ if pooled_prompt_embeds is None and prompt_embeds[0].ndim == 2:
350
+ pooled_prompt_embeds = prompt_embeds[0]
351
+
340
352
  if clip_skip is None:
341
353
  prompt_embeds = prompt_embeds.hidden_states[-2]
342
354
  else:
@@ -395,8 +407,10 @@ class StableDiffusionXLControlNetXSPipeline(
395
407
  uncond_input.input_ids.to(device),
396
408
  output_hidden_states=True,
397
409
  )
410
+
398
411
  # We are only ALWAYS interested in the pooled output of the final text encoder
399
- negative_pooled_prompt_embeds = negative_prompt_embeds[0]
412
+ if negative_pooled_prompt_embeds is None and negative_prompt_embeds[0].ndim == 2:
413
+ negative_pooled_prompt_embeds = negative_prompt_embeds[0]
400
414
  negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2]
401
415
 
402
416
  negative_prompt_embeds_list.append(negative_prompt_embeds)
@@ -1074,6 +1088,9 @@ class StableDiffusionXLControlNetXSPipeline(
1074
1088
  if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0):
1075
1089
  progress_bar.update()
1076
1090
 
1091
+ if XLA_AVAILABLE:
1092
+ xm.mark_step()
1093
+
1077
1094
  # manually for max memory savings
1078
1095
  if self.vae.dtype == torch.float16 and self.vae.config.force_upcast:
1079
1096
  self.upcast_vae()
@@ -17,11 +17,20 @@ from typing import List, Optional, Tuple, Union
17
17
 
18
18
  import torch
19
19
 
20
- from ...utils import logging
20
+ from ...models import UNet1DModel
21
+ from ...schedulers import SchedulerMixin
22
+ from ...utils import is_torch_xla_available, logging
21
23
  from ...utils.torch_utils import randn_tensor
22
24
  from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline
23
25
 
24
26
 
27
+ if is_torch_xla_available():
28
+ import torch_xla.core.xla_model as xm
29
+
30
+ XLA_AVAILABLE = True
31
+ else:
32
+ XLA_AVAILABLE = False
33
+
25
34
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
26
35
 
27
36
 
@@ -42,7 +51,7 @@ class DanceDiffusionPipeline(DiffusionPipeline):
42
51
 
43
52
  model_cpu_offload_seq = "unet"
44
53
 
45
- def __init__(self, unet, scheduler):
54
+ def __init__(self, unet: UNet1DModel, scheduler: SchedulerMixin):
46
55
  super().__init__()
47
56
  self.register_modules(unet=unet, scheduler=scheduler)
48
57
 
@@ -146,6 +155,9 @@ class DanceDiffusionPipeline(DiffusionPipeline):
146
155
  # 2. compute previous audio sample: x_t -> t_t-1
147
156
  audio = self.scheduler.step(model_output, t, audio).prev_sample
148
157
 
158
+ if XLA_AVAILABLE:
159
+ xm.mark_step()
160
+
149
161
  audio = audio.clamp(-1, 1).float().cpu().numpy()
150
162
 
151
163
  audio = audio[:, :, :original_sample_size]
@@ -16,11 +16,21 @@ from typing import List, Optional, Tuple, Union
16
16
 
17
17
  import torch
18
18
 
19
+ from ...models import UNet2DModel
19
20
  from ...schedulers import DDIMScheduler
21
+ from ...utils import is_torch_xla_available
20
22
  from ...utils.torch_utils import randn_tensor
21
23
  from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput
22
24
 
23
25
 
26
+ if is_torch_xla_available():
27
+ import torch_xla.core.xla_model as xm
28
+
29
+ XLA_AVAILABLE = True
30
+ else:
31
+ XLA_AVAILABLE = False
32
+
33
+
24
34
  class DDIMPipeline(DiffusionPipeline):
25
35
  r"""
26
36
  Pipeline for image generation.
@@ -38,7 +48,7 @@ class DDIMPipeline(DiffusionPipeline):
38
48
 
39
49
  model_cpu_offload_seq = "unet"
40
50
 
41
- def __init__(self, unet, scheduler):
51
+ def __init__(self, unet: UNet2DModel, scheduler: DDIMScheduler):
42
52
  super().__init__()
43
53
 
44
54
  # make sure scheduler can always be converted to DDIM
@@ -143,6 +153,9 @@ class DDIMPipeline(DiffusionPipeline):
143
153
  model_output, t, image, eta=eta, use_clipped_model_output=use_clipped_model_output, generator=generator
144
154
  ).prev_sample
145
155
 
156
+ if XLA_AVAILABLE:
157
+ xm.mark_step()
158
+
146
159
  image = (image / 2 + 0.5).clamp(0, 1)
147
160
  image = image.cpu().permute(0, 2, 3, 1).numpy()
148
161
  if output_type == "pil":
@@ -17,10 +17,21 @@ from typing import List, Optional, Tuple, Union
17
17
 
18
18
  import torch
19
19
 
20
+ from ...models import UNet2DModel
21
+ from ...schedulers import DDPMScheduler
22
+ from ...utils import is_torch_xla_available
20
23
  from ...utils.torch_utils import randn_tensor
21
24
  from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput
22
25
 
23
26
 
27
+ if is_torch_xla_available():
28
+ import torch_xla.core.xla_model as xm
29
+
30
+ XLA_AVAILABLE = True
31
+ else:
32
+ XLA_AVAILABLE = False
33
+
34
+
24
35
  class DDPMPipeline(DiffusionPipeline):
25
36
  r"""
26
37
  Pipeline for image generation.
@@ -38,7 +49,7 @@ class DDPMPipeline(DiffusionPipeline):
38
49
 
39
50
  model_cpu_offload_seq = "unet"
40
51
 
41
- def __init__(self, unet, scheduler):
52
+ def __init__(self, unet: UNet2DModel, scheduler: DDPMScheduler):
42
53
  super().__init__()
43
54
  self.register_modules(unet=unet, scheduler=scheduler)
44
55
 
@@ -116,6 +127,9 @@ class DDPMPipeline(DiffusionPipeline):
116
127
  # 2. compute previous image: x_t -> x_t-1
117
128
  image = self.scheduler.step(model_output, t, image, generator=generator).prev_sample
118
129
 
130
+ if XLA_AVAILABLE:
131
+ xm.mark_step()
132
+
119
133
  image = (image / 2 + 0.5).clamp(0, 1)
120
134
  image = image.cpu().permute(0, 2, 3, 1).numpy()
121
135
  if output_type == "pil":
@@ -14,6 +14,7 @@ from ...utils import (
14
14
  BACKENDS_MAPPING,
15
15
  is_bs4_available,
16
16
  is_ftfy_available,
17
+ is_torch_xla_available,
17
18
  logging,
18
19
  replace_example_docstring,
19
20
  )
@@ -24,8 +25,16 @@ from .safety_checker import IFSafetyChecker
24
25
  from .watermark import IFWatermarker
25
26
 
26
27
 
28
+ if is_torch_xla_available():
29
+ import torch_xla.core.xla_model as xm
30
+
31
+ XLA_AVAILABLE = True
32
+ else:
33
+ XLA_AVAILABLE = False
34
+
27
35
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
28
36
 
37
+
29
38
  if is_bs4_available():
30
39
  from bs4 import BeautifulSoup
31
40
 
@@ -735,6 +744,9 @@ class IFPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixin):
735
744
  if callback is not None and i % callback_steps == 0:
736
745
  callback(i, t, intermediate_images)
737
746
 
747
+ if XLA_AVAILABLE:
748
+ xm.mark_step()
749
+
738
750
  image = intermediate_images
739
751
 
740
752
  if output_type == "pil":
@@ -17,6 +17,7 @@ from ...utils import (
17
17
  PIL_INTERPOLATION,
18
18
  is_bs4_available,
19
19
  is_ftfy_available,
20
+ is_torch_xla_available,
20
21
  logging,
21
22
  replace_example_docstring,
22
23
  )
@@ -27,8 +28,16 @@ from .safety_checker import IFSafetyChecker
27
28
  from .watermark import IFWatermarker
28
29
 
29
30
 
31
+ if is_torch_xla_available():
32
+ import torch_xla.core.xla_model as xm
33
+
34
+ XLA_AVAILABLE = True
35
+ else:
36
+ XLA_AVAILABLE = False
37
+
30
38
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
31
39
 
40
+
32
41
  if is_bs4_available():
33
42
  from bs4 import BeautifulSoup
34
43
 
@@ -856,6 +865,9 @@ class IFImg2ImgPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixin):
856
865
  if callback is not None and i % callback_steps == 0:
857
866
  callback(i, t, intermediate_images)
858
867
 
868
+ if XLA_AVAILABLE:
869
+ xm.mark_step()
870
+
859
871
  image = intermediate_images
860
872
 
861
873
  if output_type == "pil":
@@ -35,6 +35,16 @@ if is_ftfy_available():
35
35
  import ftfy
36
36
 
37
37
 
38
+ from ...utils import is_torch_xla_available
39
+
40
+
41
+ if is_torch_xla_available():
42
+ import torch_xla.core.xla_model as xm
43
+
44
+ XLA_AVAILABLE = True
45
+ else:
46
+ XLA_AVAILABLE = False
47
+
38
48
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
39
49
 
40
50
 
@@ -174,7 +184,7 @@ class IFImg2ImgSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLoraLoa
174
184
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
175
185
  )
176
186
 
177
- if unet.config.in_channels != 6:
187
+ if unet is not None and unet.config.in_channels != 6:
178
188
  logger.warning(
179
189
  "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`."
180
190
  )
@@ -974,6 +984,9 @@ class IFImg2ImgSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLoraLoa
974
984
  if callback is not None and i % callback_steps == 0:
975
985
  callback(i, t, intermediate_images)
976
986
 
987
+ if XLA_AVAILABLE:
988
+ xm.mark_step()
989
+
977
990
  image = intermediate_images
978
991
 
979
992
  if output_type == "pil":
@@ -17,6 +17,7 @@ from ...utils import (
17
17
  PIL_INTERPOLATION,
18
18
  is_bs4_available,
19
19
  is_ftfy_available,
20
+ is_torch_xla_available,
20
21
  logging,
21
22
  replace_example_docstring,
22
23
  )
@@ -27,8 +28,16 @@ from .safety_checker import IFSafetyChecker
27
28
  from .watermark import IFWatermarker
28
29
 
29
30
 
31
+ if is_torch_xla_available():
32
+ import torch_xla.core.xla_model as xm
33
+
34
+ XLA_AVAILABLE = True
35
+ else:
36
+ XLA_AVAILABLE = False
37
+
30
38
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
31
39
 
40
+
32
41
  if is_bs4_available():
33
42
  from bs4 import BeautifulSoup
34
43
 
@@ -975,6 +984,9 @@ class IFInpaintingPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixin):
975
984
  if callback is not None and i % callback_steps == 0:
976
985
  callback(i, t, intermediate_images)
977
986
 
987
+ if XLA_AVAILABLE:
988
+ xm.mark_step()
989
+
978
990
  image = intermediate_images
979
991
 
980
992
  if output_type == "pil":
@@ -35,6 +35,16 @@ if is_ftfy_available():
35
35
  import ftfy
36
36
 
37
37
 
38
+ from ...utils import is_torch_xla_available
39
+
40
+
41
+ if is_torch_xla_available():
42
+ import torch_xla.core.xla_model as xm
43
+
44
+ XLA_AVAILABLE = True
45
+ else:
46
+ XLA_AVAILABLE = False
47
+
38
48
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
39
49
 
40
50
 
@@ -176,7 +186,7 @@ class IFInpaintingSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLora
176
186
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
177
187
  )
178
188
 
179
- if unet.config.in_channels != 6:
189
+ if unet is not None and unet.config.in_channels != 6:
180
190
  logger.warning(
181
191
  "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`."
182
192
  )
@@ -1085,6 +1095,9 @@ class IFInpaintingSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLora
1085
1095
  if callback is not None and i % callback_steps == 0:
1086
1096
  callback(i, t, intermediate_images)
1087
1097
 
1098
+ if XLA_AVAILABLE:
1099
+ xm.mark_step()
1100
+
1088
1101
  image = intermediate_images
1089
1102
 
1090
1103
  if output_type == "pil":
@@ -34,6 +34,16 @@ if is_ftfy_available():
34
34
  import ftfy
35
35
 
36
36
 
37
+ from ...utils import is_torch_xla_available
38
+
39
+
40
+ if is_torch_xla_available():
41
+ import torch_xla.core.xla_model as xm
42
+
43
+ XLA_AVAILABLE = True
44
+ else:
45
+ XLA_AVAILABLE = False
46
+
37
47
  logger = logging.get_logger(__name__) # pylint: disable=invalid-name
38
48
 
39
49
 
@@ -132,7 +142,7 @@ class IFSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixi
132
142
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
133
143
  )
134
144
 
135
- if unet.config.in_channels != 6:
145
+ if unet is not None and unet.config.in_channels != 6:
136
146
  logger.warning(
137
147
  "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`."
138
148
  )
@@ -831,6 +841,9 @@ class IFSuperResolutionPipeline(DiffusionPipeline, StableDiffusionLoraLoaderMixi
831
841
  if callback is not None and i % callback_steps == 0:
832
842
  callback(i, t, intermediate_images)
833
843
 
844
+ if XLA_AVAILABLE:
845
+ xm.mark_step()
846
+
834
847
  image = intermediate_images
835
848
 
836
849
  if output_type == "pil":
@@ -210,7 +210,7 @@ class AltDiffusionPipeline(
210
210
  ):
211
211
  super().__init__()
212
212
 
213
- if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
213
+ if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:
214
214
  deprecation_message = (
215
215
  f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
216
216
  f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
@@ -224,7 +224,7 @@ class AltDiffusionPipeline(
224
224
  new_config["steps_offset"] = 1
225
225
  scheduler._internal_dict = FrozenDict(new_config)
226
226
 
227
- if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
227
+ if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:
228
228
  deprecation_message = (
229
229
  f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
230
230
  " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
@@ -253,10 +253,14 @@ class AltDiffusionPipeline(
253
253
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
254
254
  )
255
255
 
256
- is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
257
- version.parse(unet.config._diffusers_version).base_version
258
- ) < version.parse("0.9.0.dev0")
259
- is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
256
+ is_unet_version_less_0_9_0 = (
257
+ unet is not None
258
+ and hasattr(unet.config, "_diffusers_version")
259
+ and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")
260
+ )
261
+ is_unet_sample_size_less_64 = (
262
+ unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
263
+ )
260
264
  if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
261
265
  deprecation_message = (
262
266
  "The configuration file of the unet has set the default `sample_size` to smaller than"
@@ -284,7 +288,7 @@ class AltDiffusionPipeline(
284
288
  feature_extractor=feature_extractor,
285
289
  image_encoder=image_encoder,
286
290
  )
287
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
291
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
288
292
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
289
293
  self.register_to_config(requires_safety_checker=requires_safety_checker)
290
294
 
@@ -238,7 +238,7 @@ class AltDiffusionImg2ImgPipeline(
238
238
  ):
239
239
  super().__init__()
240
240
 
241
- if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
241
+ if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:
242
242
  deprecation_message = (
243
243
  f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
244
244
  f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
@@ -252,7 +252,7 @@ class AltDiffusionImg2ImgPipeline(
252
252
  new_config["steps_offset"] = 1
253
253
  scheduler._internal_dict = FrozenDict(new_config)
254
254
 
255
- if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
255
+ if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:
256
256
  deprecation_message = (
257
257
  f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
258
258
  " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
@@ -281,10 +281,14 @@ class AltDiffusionImg2ImgPipeline(
281
281
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
282
282
  )
283
283
 
284
- is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
285
- version.parse(unet.config._diffusers_version).base_version
286
- ) < version.parse("0.9.0.dev0")
287
- is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
284
+ is_unet_version_less_0_9_0 = (
285
+ unet is not None
286
+ and hasattr(unet.config, "_diffusers_version")
287
+ and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")
288
+ )
289
+ is_unet_sample_size_less_64 = (
290
+ unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
291
+ )
288
292
  if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
289
293
  deprecation_message = (
290
294
  "The configuration file of the unet has set the default `sample_size` to smaller than"
@@ -312,7 +316,7 @@ class AltDiffusionImg2ImgPipeline(
312
316
  feature_extractor=feature_extractor,
313
317
  image_encoder=image_encoder,
314
318
  )
315
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
319
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
316
320
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
317
321
  self.register_to_config(requires_safety_checker=requires_safety_checker)
318
322
 
@@ -91,7 +91,7 @@ class RePaintPipeline(DiffusionPipeline):
91
91
  scheduler: RePaintScheduler
92
92
  model_cpu_offload_seq = "unet"
93
93
 
94
- def __init__(self, unet, scheduler):
94
+ def __init__(self, unet: UNet2DModel, scheduler: RePaintScheduler):
95
95
  super().__init__()
96
96
  self.register_modules(unet=unet, scheduler=scheduler)
97
97
 
@@ -184,7 +184,7 @@ class CycleDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Sta
184
184
  ):
185
185
  super().__init__()
186
186
 
187
- if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
187
+ if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:
188
188
  deprecation_message = (
189
189
  f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
190
190
  f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
@@ -213,10 +213,14 @@ class CycleDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Sta
213
213
  "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety"
214
214
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
215
215
  )
216
- is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
217
- version.parse(unet.config._diffusers_version).base_version
218
- ) < version.parse("0.9.0.dev0")
219
- is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
216
+ is_unet_version_less_0_9_0 = (
217
+ unet is not None
218
+ and hasattr(unet.config, "_diffusers_version")
219
+ and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")
220
+ )
221
+ is_unet_sample_size_less_64 = (
222
+ unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
223
+ )
220
224
  if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
221
225
  deprecation_message = (
222
226
  "The configuration file of the unet has set the default `sample_size` to smaller than"
@@ -243,7 +247,7 @@ class CycleDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Sta
243
247
  safety_checker=safety_checker,
244
248
  feature_extractor=feature_extractor,
245
249
  )
246
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
250
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
247
251
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
248
252
  self.register_to_config(requires_safety_checker=requires_safety_checker)
249
253
 
@@ -93,7 +93,7 @@ class OnnxStableDiffusionInpaintPipelineLegacy(DiffusionPipeline):
93
93
  ):
94
94
  super().__init__()
95
95
 
96
- if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
96
+ if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:
97
97
  deprecation_message = (
98
98
  f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
99
99
  f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
@@ -107,7 +107,7 @@ class OnnxStableDiffusionInpaintPipelineLegacy(DiffusionPipeline):
107
107
  new_config["steps_offset"] = 1
108
108
  scheduler._internal_dict = FrozenDict(new_config)
109
109
 
110
- if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
110
+ if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:
111
111
  deprecation_message = (
112
112
  f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
113
113
  " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
@@ -140,7 +140,7 @@ class StableDiffusionInpaintPipelineLegacy(
140
140
  )
141
141
  deprecate("legacy is outdated", "1.0.0", deprecation_message, standard_warn=False)
142
142
 
143
- if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1:
143
+ if scheduler is not None and getattr(scheduler.config, "steps_offset", 1) != 1:
144
144
  deprecation_message = (
145
145
  f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`"
146
146
  f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure "
@@ -154,7 +154,7 @@ class StableDiffusionInpaintPipelineLegacy(
154
154
  new_config["steps_offset"] = 1
155
155
  scheduler._internal_dict = FrozenDict(new_config)
156
156
 
157
- if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True:
157
+ if scheduler is not None and getattr(scheduler.config, "clip_sample", False) is True:
158
158
  deprecation_message = (
159
159
  f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`."
160
160
  " `clip_sample` should be set to False in the configuration file. Please make sure to update the"
@@ -183,10 +183,14 @@ class StableDiffusionInpaintPipelineLegacy(
183
183
  " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead."
184
184
  )
185
185
 
186
- is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse(
187
- version.parse(unet.config._diffusers_version).base_version
188
- ) < version.parse("0.9.0.dev0")
189
- is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
186
+ is_unet_version_less_0_9_0 = (
187
+ unet is not None
188
+ and hasattr(unet.config, "_diffusers_version")
189
+ and version.parse(version.parse(unet.config._diffusers_version).base_version) < version.parse("0.9.0.dev0")
190
+ )
191
+ is_unet_sample_size_less_64 = (
192
+ unet is not None and hasattr(unet.config, "sample_size") and unet.config.sample_size < 64
193
+ )
190
194
  if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64:
191
195
  deprecation_message = (
192
196
  "The configuration file of the unet has set the default `sample_size` to smaller than"
@@ -213,7 +217,7 @@ class StableDiffusionInpaintPipelineLegacy(
213
217
  safety_checker=safety_checker,
214
218
  feature_extractor=feature_extractor,
215
219
  )
216
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
220
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
217
221
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
218
222
  self.register_to_config(requires_safety_checker=requires_safety_checker)
219
223
 
@@ -121,7 +121,7 @@ class StableDiffusionModelEditingPipeline(
121
121
  safety_checker=safety_checker,
122
122
  feature_extractor=feature_extractor,
123
123
  )
124
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
124
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
125
125
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
126
126
  self.register_to_config(requires_safety_checker=requires_safety_checker)
127
127
 
@@ -143,7 +143,7 @@ class StableDiffusionParadigmsPipeline(
143
143
  safety_checker=safety_checker,
144
144
  feature_extractor=feature_extractor,
145
145
  )
146
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
146
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
147
147
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
148
148
  self.register_to_config(requires_safety_checker=requires_safety_checker)
149
149
 
@@ -365,7 +365,7 @@ class StableDiffusionPix2PixZeroPipeline(DiffusionPipeline, StableDiffusionMixin
365
365
  caption_generator=caption_generator,
366
366
  inverse_scheduler=inverse_scheduler,
367
367
  )
368
- self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
368
+ self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8
369
369
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
370
370
  self.register_to_config(requires_safety_checker=requires_safety_checker)
371
371