diffusers 0.26.2__py3-none-any.whl → 0.27.0__py3-none-any.whl

Sign up to get free protection for your applications and to get access to all the features.
Files changed (299) hide show
  1. diffusers/__init__.py +20 -1
  2. diffusers/commands/__init__.py +1 -1
  3. diffusers/commands/diffusers_cli.py +1 -1
  4. diffusers/commands/env.py +1 -1
  5. diffusers/commands/fp16_safetensors.py +1 -1
  6. diffusers/configuration_utils.py +7 -3
  7. diffusers/dependency_versions_check.py +1 -1
  8. diffusers/dependency_versions_table.py +2 -2
  9. diffusers/experimental/rl/value_guided_sampling.py +1 -1
  10. diffusers/image_processor.py +110 -4
  11. diffusers/loaders/autoencoder.py +28 -8
  12. diffusers/loaders/controlnet.py +17 -8
  13. diffusers/loaders/ip_adapter.py +86 -23
  14. diffusers/loaders/lora.py +105 -310
  15. diffusers/loaders/lora_conversion_utils.py +1 -1
  16. diffusers/loaders/peft.py +1 -1
  17. diffusers/loaders/single_file.py +51 -12
  18. diffusers/loaders/single_file_utils.py +278 -49
  19. diffusers/loaders/textual_inversion.py +23 -4
  20. diffusers/loaders/unet.py +195 -41
  21. diffusers/loaders/utils.py +1 -1
  22. diffusers/models/__init__.py +3 -1
  23. diffusers/models/activations.py +9 -9
  24. diffusers/models/attention.py +26 -36
  25. diffusers/models/attention_flax.py +1 -1
  26. diffusers/models/attention_processor.py +171 -114
  27. diffusers/models/autoencoders/autoencoder_asym_kl.py +1 -1
  28. diffusers/models/autoencoders/autoencoder_kl.py +3 -1
  29. diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py +1 -1
  30. diffusers/models/autoencoders/autoencoder_tiny.py +4 -2
  31. diffusers/models/autoencoders/consistency_decoder_vae.py +1 -1
  32. diffusers/models/autoencoders/vae.py +1 -1
  33. diffusers/models/controlnet.py +1 -1
  34. diffusers/models/controlnet_flax.py +1 -1
  35. diffusers/models/downsampling.py +8 -12
  36. diffusers/models/dual_transformer_2d.py +1 -1
  37. diffusers/models/embeddings.py +3 -4
  38. diffusers/models/embeddings_flax.py +1 -1
  39. diffusers/models/lora.py +33 -10
  40. diffusers/models/modeling_flax_pytorch_utils.py +1 -1
  41. diffusers/models/modeling_flax_utils.py +1 -1
  42. diffusers/models/modeling_pytorch_flax_utils.py +1 -1
  43. diffusers/models/modeling_utils.py +4 -6
  44. diffusers/models/normalization.py +1 -1
  45. diffusers/models/resnet.py +31 -58
  46. diffusers/models/resnet_flax.py +1 -1
  47. diffusers/models/t5_film_transformer.py +1 -1
  48. diffusers/models/transformer_2d.py +1 -1
  49. diffusers/models/transformer_temporal.py +1 -1
  50. diffusers/models/transformers/dual_transformer_2d.py +1 -1
  51. diffusers/models/transformers/t5_film_transformer.py +1 -1
  52. diffusers/models/transformers/transformer_2d.py +29 -31
  53. diffusers/models/transformers/transformer_temporal.py +1 -1
  54. diffusers/models/unet_1d.py +1 -1
  55. diffusers/models/unet_1d_blocks.py +1 -1
  56. diffusers/models/unet_2d.py +1 -1
  57. diffusers/models/unet_2d_blocks.py +1 -1
  58. diffusers/models/unet_2d_condition.py +1 -1
  59. diffusers/models/unets/__init__.py +1 -0
  60. diffusers/models/unets/unet_1d.py +1 -1
  61. diffusers/models/unets/unet_1d_blocks.py +1 -1
  62. diffusers/models/unets/unet_2d.py +4 -4
  63. diffusers/models/unets/unet_2d_blocks.py +238 -98
  64. diffusers/models/unets/unet_2d_blocks_flax.py +1 -1
  65. diffusers/models/unets/unet_2d_condition.py +420 -323
  66. diffusers/models/unets/unet_2d_condition_flax.py +21 -12
  67. diffusers/models/unets/unet_3d_blocks.py +50 -40
  68. diffusers/models/unets/unet_3d_condition.py +47 -8
  69. diffusers/models/unets/unet_i2vgen_xl.py +75 -30
  70. diffusers/models/unets/unet_kandinsky3.py +1 -1
  71. diffusers/models/unets/unet_motion_model.py +48 -8
  72. diffusers/models/unets/unet_spatio_temporal_condition.py +1 -1
  73. diffusers/models/unets/unet_stable_cascade.py +610 -0
  74. diffusers/models/unets/uvit_2d.py +1 -1
  75. diffusers/models/upsampling.py +10 -16
  76. diffusers/models/vae_flax.py +1 -1
  77. diffusers/models/vq_model.py +1 -1
  78. diffusers/optimization.py +1 -1
  79. diffusers/pipelines/__init__.py +26 -0
  80. diffusers/pipelines/amused/pipeline_amused.py +1 -1
  81. diffusers/pipelines/amused/pipeline_amused_img2img.py +1 -1
  82. diffusers/pipelines/amused/pipeline_amused_inpaint.py +1 -1
  83. diffusers/pipelines/animatediff/pipeline_animatediff.py +162 -417
  84. diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py +165 -137
  85. diffusers/pipelines/animatediff/pipeline_output.py +7 -6
  86. diffusers/pipelines/audioldm/pipeline_audioldm.py +3 -19
  87. diffusers/pipelines/audioldm2/modeling_audioldm2.py +1 -1
  88. diffusers/pipelines/audioldm2/pipeline_audioldm2.py +3 -3
  89. diffusers/pipelines/auto_pipeline.py +7 -16
  90. diffusers/pipelines/blip_diffusion/blip_image_processing.py +1 -1
  91. diffusers/pipelines/blip_diffusion/modeling_blip2.py +1 -1
  92. diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py +2 -2
  93. diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py +2 -2
  94. diffusers/pipelines/consistency_models/pipeline_consistency_models.py +1 -1
  95. diffusers/pipelines/controlnet/pipeline_controlnet.py +90 -90
  96. diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py +2 -2
  97. diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +98 -90
  98. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py +92 -90
  99. diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py +145 -70
  100. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +126 -89
  101. diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +108 -96
  102. diffusers/pipelines/controlnet/pipeline_flax_controlnet.py +2 -2
  103. diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py +1 -1
  104. diffusers/pipelines/ddim/pipeline_ddim.py +1 -1
  105. diffusers/pipelines/ddpm/pipeline_ddpm.py +1 -1
  106. diffusers/pipelines/deepfloyd_if/pipeline_if.py +4 -4
  107. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py +4 -4
  108. diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py +5 -5
  109. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py +4 -4
  110. diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py +5 -5
  111. diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py +5 -5
  112. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py +10 -120
  113. diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py +10 -91
  114. diffusers/pipelines/deprecated/audio_diffusion/mel.py +1 -1
  115. diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py +1 -1
  116. diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py +1 -1
  117. diffusers/pipelines/deprecated/pndm/pipeline_pndm.py +1 -1
  118. diffusers/pipelines/deprecated/repaint/pipeline_repaint.py +1 -1
  119. diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py +1 -1
  120. diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py +1 -1
  121. diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py +1 -1
  122. diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py +1 -1
  123. diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py +1 -1
  124. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py +5 -4
  125. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py +5 -4
  126. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py +7 -22
  127. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py +5 -39
  128. diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py +5 -5
  129. diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py +1 -1
  130. diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py +31 -22
  131. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py +1 -1
  132. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py +1 -1
  133. diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py +1 -2
  134. diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py +1 -1
  135. diffusers/pipelines/dit/pipeline_dit.py +1 -1
  136. diffusers/pipelines/free_init_utils.py +184 -0
  137. diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py +22 -104
  138. diffusers/pipelines/kandinsky/pipeline_kandinsky.py +1 -1
  139. diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py +1 -1
  140. diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py +1 -1
  141. diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py +2 -2
  142. diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py +1 -1
  143. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +1 -1
  144. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +1 -1
  145. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py +1 -1
  146. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py +1 -1
  147. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +1 -1
  148. diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py +2 -2
  149. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py +104 -93
  150. diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py +112 -74
  151. diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py +1 -1
  152. diffusers/pipelines/ledits_pp/__init__.py +55 -0
  153. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py +1505 -0
  154. diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py +1797 -0
  155. diffusers/pipelines/ledits_pp/pipeline_output.py +43 -0
  156. diffusers/pipelines/musicldm/pipeline_musicldm.py +3 -19
  157. diffusers/pipelines/onnx_utils.py +1 -1
  158. diffusers/pipelines/paint_by_example/image_encoder.py +1 -1
  159. diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +3 -3
  160. diffusers/pipelines/pia/pipeline_pia.py +168 -327
  161. diffusers/pipelines/pipeline_flax_utils.py +1 -1
  162. diffusers/pipelines/pipeline_loading_utils.py +508 -0
  163. diffusers/pipelines/pipeline_utils.py +188 -534
  164. diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py +56 -10
  165. diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py +3 -3
  166. diffusers/pipelines/shap_e/camera.py +1 -1
  167. diffusers/pipelines/shap_e/pipeline_shap_e.py +1 -1
  168. diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py +1 -1
  169. diffusers/pipelines/shap_e/renderer.py +1 -1
  170. diffusers/pipelines/stable_cascade/__init__.py +50 -0
  171. diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py +482 -0
  172. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py +311 -0
  173. diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py +638 -0
  174. diffusers/pipelines/stable_diffusion/clip_image_project_model.py +1 -1
  175. diffusers/pipelines/stable_diffusion/convert_from_ckpt.py +4 -1
  176. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py +1 -1
  177. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py +2 -2
  178. diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py +1 -1
  179. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py +1 -1
  180. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py +1 -1
  181. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py +1 -1
  182. diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py +1 -1
  183. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +90 -146
  184. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py +5 -4
  185. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +4 -32
  186. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +92 -119
  187. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +92 -119
  188. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py +13 -59
  189. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py +3 -31
  190. diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py +5 -33
  191. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py +5 -21
  192. diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py +7 -21
  193. diffusers/pipelines/stable_diffusion/safety_checker.py +1 -1
  194. diffusers/pipelines/stable_diffusion/safety_checker_flax.py +1 -1
  195. diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py +1 -1
  196. diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py +5 -21
  197. diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py +9 -38
  198. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py +5 -34
  199. diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py +6 -35
  200. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py +7 -6
  201. diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py +4 -124
  202. diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py +282 -80
  203. diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py +94 -46
  204. diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py +3 -3
  205. diffusers/pipelines/stable_diffusion_safe/safety_checker.py +1 -1
  206. diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py +6 -22
  207. diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py +1 -1
  208. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +96 -148
  209. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +98 -154
  210. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +98 -153
  211. diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py +25 -87
  212. diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py +89 -80
  213. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py +5 -49
  214. diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py +80 -88
  215. diffusers/pipelines/text_to_video_synthesis/pipeline_output.py +8 -6
  216. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py +15 -86
  217. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py +20 -93
  218. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py +5 -5
  219. diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py +3 -19
  220. diffusers/pipelines/unclip/pipeline_unclip.py +1 -1
  221. diffusers/pipelines/unclip/pipeline_unclip_image_variation.py +1 -1
  222. diffusers/pipelines/unclip/text_proj.py +1 -1
  223. diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py +35 -35
  224. diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py +1 -1
  225. diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py +4 -21
  226. diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py +2 -2
  227. diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py +4 -5
  228. diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py +8 -8
  229. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py +1 -1
  230. diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py +2 -2
  231. diffusers/schedulers/__init__.py +7 -1
  232. diffusers/schedulers/deprecated/scheduling_karras_ve.py +1 -1
  233. diffusers/schedulers/deprecated/scheduling_sde_vp.py +1 -1
  234. diffusers/schedulers/scheduling_consistency_models.py +42 -19
  235. diffusers/schedulers/scheduling_ddim.py +2 -4
  236. diffusers/schedulers/scheduling_ddim_flax.py +13 -5
  237. diffusers/schedulers/scheduling_ddim_inverse.py +2 -4
  238. diffusers/schedulers/scheduling_ddim_parallel.py +2 -4
  239. diffusers/schedulers/scheduling_ddpm.py +2 -4
  240. diffusers/schedulers/scheduling_ddpm_flax.py +1 -1
  241. diffusers/schedulers/scheduling_ddpm_parallel.py +2 -4
  242. diffusers/schedulers/scheduling_ddpm_wuerstchen.py +1 -1
  243. diffusers/schedulers/scheduling_deis_multistep.py +46 -19
  244. diffusers/schedulers/scheduling_dpmsolver_multistep.py +107 -21
  245. diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py +1 -1
  246. diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py +9 -7
  247. diffusers/schedulers/scheduling_dpmsolver_sde.py +35 -35
  248. diffusers/schedulers/scheduling_dpmsolver_singlestep.py +52 -21
  249. diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py +683 -0
  250. diffusers/schedulers/scheduling_edm_euler.py +381 -0
  251. diffusers/schedulers/scheduling_euler_ancestral_discrete.py +43 -15
  252. diffusers/schedulers/scheduling_euler_discrete.py +42 -17
  253. diffusers/schedulers/scheduling_euler_discrete_flax.py +1 -1
  254. diffusers/schedulers/scheduling_heun_discrete.py +35 -35
  255. diffusers/schedulers/scheduling_ipndm.py +37 -11
  256. diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py +44 -44
  257. diffusers/schedulers/scheduling_k_dpm_2_discrete.py +44 -44
  258. diffusers/schedulers/scheduling_karras_ve_flax.py +1 -1
  259. diffusers/schedulers/scheduling_lcm.py +38 -14
  260. diffusers/schedulers/scheduling_lms_discrete.py +43 -15
  261. diffusers/schedulers/scheduling_lms_discrete_flax.py +1 -1
  262. diffusers/schedulers/scheduling_pndm.py +2 -4
  263. diffusers/schedulers/scheduling_pndm_flax.py +2 -4
  264. diffusers/schedulers/scheduling_repaint.py +1 -1
  265. diffusers/schedulers/scheduling_sasolver.py +41 -9
  266. diffusers/schedulers/scheduling_sde_ve.py +1 -1
  267. diffusers/schedulers/scheduling_sde_ve_flax.py +1 -1
  268. diffusers/schedulers/scheduling_tcd.py +686 -0
  269. diffusers/schedulers/scheduling_unclip.py +1 -1
  270. diffusers/schedulers/scheduling_unipc_multistep.py +46 -19
  271. diffusers/schedulers/scheduling_utils.py +2 -1
  272. diffusers/schedulers/scheduling_utils_flax.py +1 -1
  273. diffusers/schedulers/scheduling_vq_diffusion.py +1 -1
  274. diffusers/training_utils.py +9 -2
  275. diffusers/utils/__init__.py +2 -1
  276. diffusers/utils/accelerate_utils.py +1 -1
  277. diffusers/utils/constants.py +1 -1
  278. diffusers/utils/doc_utils.py +1 -1
  279. diffusers/utils/dummy_pt_objects.py +60 -0
  280. diffusers/utils/dummy_torch_and_transformers_objects.py +75 -0
  281. diffusers/utils/dynamic_modules_utils.py +1 -1
  282. diffusers/utils/export_utils.py +3 -3
  283. diffusers/utils/hub_utils.py +60 -16
  284. diffusers/utils/import_utils.py +15 -1
  285. diffusers/utils/loading_utils.py +2 -0
  286. diffusers/utils/logging.py +1 -1
  287. diffusers/utils/model_card_template.md +24 -0
  288. diffusers/utils/outputs.py +14 -7
  289. diffusers/utils/peft_utils.py +1 -1
  290. diffusers/utils/state_dict_utils.py +1 -1
  291. diffusers/utils/testing_utils.py +2 -0
  292. diffusers/utils/torch_utils.py +1 -1
  293. {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/METADATA +5 -5
  294. diffusers-0.27.0.dist-info/RECORD +399 -0
  295. diffusers-0.26.2.dist-info/RECORD +0 -384
  296. {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/LICENSE +0 -0
  297. {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/WHEEL +0 -0
  298. {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/entry_points.txt +0 -0
  299. {diffusers-0.26.2.dist-info → diffusers-0.27.0.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- # Copyright 2023 TencentARC and The HuggingFace Team. All rights reserved.
1
+ # Copyright 2024 TencentARC and The HuggingFace Team. All rights reserved.
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -51,7 +51,7 @@ from ...utils import (
51
51
  unscale_lora_layers,
52
52
  )
53
53
  from ...utils.torch_utils import randn_tensor
54
- from ..pipeline_utils import DiffusionPipeline
54
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
55
55
  from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput
56
56
 
57
57
 
@@ -181,6 +181,7 @@ def retrieve_timesteps(
181
181
 
182
182
  class StableDiffusionXLAdapterPipeline(
183
183
  DiffusionPipeline,
184
+ StableDiffusionMixin,
184
185
  TextualInversionLoaderMixin,
185
186
  StableDiffusionXLLoraLoaderMixin,
186
187
  IPAdapterMixin,
@@ -270,39 +271,6 @@ class StableDiffusionXLAdapterPipeline(
270
271
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
271
272
  self.default_sample_size = self.unet.config.sample_size
272
273
 
273
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing
274
- def enable_vae_slicing(self):
275
- r"""
276
- Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
277
- compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
278
- """
279
- self.vae.enable_slicing()
280
-
281
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing
282
- def disable_vae_slicing(self):
283
- r"""
284
- Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
285
- computing decoding in one step.
286
- """
287
- self.vae.disable_slicing()
288
-
289
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling
290
- def enable_vae_tiling(self):
291
- r"""
292
- Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
293
- compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
294
- processing larger images.
295
- """
296
- self.vae.enable_tiling()
297
-
298
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling
299
- def disable_vae_tiling(self):
300
- r"""
301
- Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
302
- computing decoding in one step.
303
- """
304
- self.vae.disable_tiling()
305
-
306
274
  # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt
307
275
  def encode_prompt(
308
276
  self,
@@ -399,7 +367,7 @@ class StableDiffusionXLAdapterPipeline(
399
367
  prompt_2 = prompt_2 or prompt
400
368
  prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2
401
369
 
402
- # textual inversion: procecss multi-vector tokens if necessary
370
+ # textual inversion: process multi-vector tokens if necessary
403
371
  prompt_embeds_list = []
404
372
  prompts = [prompt, prompt_2]
405
373
  for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders):
@@ -564,31 +532,54 @@ class StableDiffusionXLAdapterPipeline(
564
532
  return image_embeds, uncond_image_embeds
565
533
 
566
534
  # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds
567
- def prepare_ip_adapter_image_embeds(self, ip_adapter_image, device, num_images_per_prompt):
568
- if not isinstance(ip_adapter_image, list):
569
- ip_adapter_image = [ip_adapter_image]
535
+ def prepare_ip_adapter_image_embeds(
536
+ self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance
537
+ ):
538
+ if ip_adapter_image_embeds is None:
539
+ if not isinstance(ip_adapter_image, list):
540
+ ip_adapter_image = [ip_adapter_image]
570
541
 
571
- if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
572
- raise ValueError(
573
- f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
574
- )
542
+ if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers):
543
+ raise ValueError(
544
+ f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters."
545
+ )
575
546
 
576
- image_embeds = []
577
- for single_ip_adapter_image, image_proj_layer in zip(
578
- ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
579
- ):
580
- output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
581
- single_image_embeds, single_negative_image_embeds = self.encode_image(
582
- single_ip_adapter_image, device, 1, output_hidden_state
583
- )
584
- single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0)
585
- single_negative_image_embeds = torch.stack([single_negative_image_embeds] * num_images_per_prompt, dim=0)
547
+ image_embeds = []
548
+ for single_ip_adapter_image, image_proj_layer in zip(
549
+ ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers
550
+ ):
551
+ output_hidden_state = not isinstance(image_proj_layer, ImageProjection)
552
+ single_image_embeds, single_negative_image_embeds = self.encode_image(
553
+ single_ip_adapter_image, device, 1, output_hidden_state
554
+ )
555
+ single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0)
556
+ single_negative_image_embeds = torch.stack(
557
+ [single_negative_image_embeds] * num_images_per_prompt, dim=0
558
+ )
586
559
 
587
- if self.do_classifier_free_guidance:
588
- single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
589
- single_image_embeds = single_image_embeds.to(device)
560
+ if do_classifier_free_guidance:
561
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
562
+ single_image_embeds = single_image_embeds.to(device)
590
563
 
591
- image_embeds.append(single_image_embeds)
564
+ image_embeds.append(single_image_embeds)
565
+ else:
566
+ repeat_dims = [1]
567
+ image_embeds = []
568
+ for single_image_embeds in ip_adapter_image_embeds:
569
+ if do_classifier_free_guidance:
570
+ single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2)
571
+ single_image_embeds = single_image_embeds.repeat(
572
+ num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
573
+ )
574
+ single_negative_image_embeds = single_negative_image_embeds.repeat(
575
+ num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:]))
576
+ )
577
+ single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds])
578
+ else:
579
+ single_image_embeds = single_image_embeds.repeat(
580
+ num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:]))
581
+ )
582
+ image_embeds.append(single_image_embeds)
592
583
 
593
584
  return image_embeds
594
585
 
@@ -624,6 +615,8 @@ class StableDiffusionXLAdapterPipeline(
624
615
  negative_prompt_embeds=None,
625
616
  pooled_prompt_embeds=None,
626
617
  negative_pooled_prompt_embeds=None,
618
+ ip_adapter_image=None,
619
+ ip_adapter_image_embeds=None,
627
620
  callback_on_step_end_tensor_inputs=None,
628
621
  ):
629
622
  if height % 8 != 0 or width % 8 != 0:
@@ -690,6 +683,21 @@ class StableDiffusionXLAdapterPipeline(
690
683
  "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`."
691
684
  )
692
685
 
686
+ if ip_adapter_image is not None and ip_adapter_image_embeds is not None:
687
+ raise ValueError(
688
+ "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined."
689
+ )
690
+
691
+ if ip_adapter_image_embeds is not None:
692
+ if not isinstance(ip_adapter_image_embeds, list):
693
+ raise ValueError(
694
+ f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}"
695
+ )
696
+ elif ip_adapter_image_embeds[0].ndim not in [3, 4]:
697
+ raise ValueError(
698
+ f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D"
699
+ )
700
+
693
701
  # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
694
702
  def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None):
695
703
  shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor)
@@ -775,34 +783,6 @@ class StableDiffusionXLAdapterPipeline(
775
783
 
776
784
  return height, width
777
785
 
778
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu
779
- def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):
780
- r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.
781
-
782
- The suffixes after the scaling factors represent the stages where they are being applied.
783
-
784
- Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values
785
- that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
786
-
787
- Args:
788
- s1 (`float`):
789
- Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
790
- mitigate "oversmoothing effect" in the enhanced denoising process.
791
- s2 (`float`):
792
- Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
793
- mitigate "oversmoothing effect" in the enhanced denoising process.
794
- b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
795
- b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
796
- """
797
- if not hasattr(self, "unet"):
798
- raise ValueError("The pipeline must have `unet` for using FreeU.")
799
- self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)
800
-
801
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu
802
- def disable_freeu(self):
803
- """Disables the FreeU mechanism if enabled."""
804
- self.unet.disable_freeu()
805
-
806
786
  # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding
807
787
  def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32):
808
788
  """
@@ -867,6 +847,7 @@ class StableDiffusionXLAdapterPipeline(
867
847
  pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
868
848
  negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None,
869
849
  ip_adapter_image: Optional[PipelineImageInput] = None,
850
+ ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None,
870
851
  output_type: Optional[str] = "pil",
871
852
  return_dict: bool = True,
872
853
  callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None,
@@ -959,6 +940,11 @@ class StableDiffusionXLAdapterPipeline(
959
940
  weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt`
960
941
  input argument.
961
942
  ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters.
943
+ ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*):
944
+ Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters.
945
+ Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding
946
+ if `do_classifier_free_guidance` is set to `True`.
947
+ If not provided, embeddings are computed from the `ip_adapter_image` input argument.
962
948
  output_type (`str`, *optional*, defaults to `"pil"`):
963
949
  The output format of the generate image. Choose between
964
950
  [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`.
@@ -1060,6 +1046,8 @@ class StableDiffusionXLAdapterPipeline(
1060
1046
  negative_prompt_embeds,
1061
1047
  pooled_prompt_embeds,
1062
1048
  negative_pooled_prompt_embeds,
1049
+ ip_adapter_image,
1050
+ ip_adapter_image_embeds,
1063
1051
  )
1064
1052
 
1065
1053
  self._guidance_scale = guidance_scale
@@ -1096,9 +1084,13 @@ class StableDiffusionXLAdapterPipeline(
1096
1084
  )
1097
1085
 
1098
1086
  # 3.2 Encode ip_adapter_image
1099
- if ip_adapter_image is not None:
1087
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1100
1088
  image_embeds = self.prepare_ip_adapter_image_embeds(
1101
- ip_adapter_image, device, batch_size * num_images_per_prompt
1089
+ ip_adapter_image,
1090
+ ip_adapter_image_embeds,
1091
+ device,
1092
+ batch_size * num_images_per_prompt,
1093
+ self.do_classifier_free_guidance,
1102
1094
  )
1103
1095
 
1104
1096
  # 4. Prepare timesteps
@@ -1199,7 +1191,7 @@ class StableDiffusionXLAdapterPipeline(
1199
1191
 
1200
1192
  added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids}
1201
1193
 
1202
- if ip_adapter_image is not None:
1194
+ if ip_adapter_image is not None or ip_adapter_image_embeds is not None:
1203
1195
  added_cond_kwargs["image_embeds"] = image_embeds
1204
1196
 
1205
1197
  # predict the noise residual
@@ -2,6 +2,7 @@ from dataclasses import dataclass
2
2
  from typing import List, Union
3
3
 
4
4
  import numpy as np
5
+ import PIL
5
6
  import torch
6
7
 
7
8
  from ...utils import (
@@ -12,12 +13,13 @@ from ...utils import (
12
13
  @dataclass
13
14
  class TextToVideoSDPipelineOutput(BaseOutput):
14
15
  """
15
- Output class for text-to-video pipelines.
16
+ Output class for text-to-video pipelines.
16
17
 
17
- Args:
18
- frames (`List[np.ndarray]` or `torch.FloatTensor`)
19
- List of denoised frames (essentially images) as NumPy arrays of shape `(height, width, num_channels)` or as
20
- a `torch` tensor. The length of the list denotes the video length (the number of frames).
18
+ Args:
19
+ frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]):
20
+ List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised
21
+ PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape
22
+ `(batch_size, num_frames, channels, height, width)`
21
23
  """
22
24
 
23
- frames: Union[List[np.ndarray], torch.FloatTensor]
25
+ frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]]
@@ -1,4 +1,4 @@
1
- # Copyright 2023 The HuggingFace Team. All rights reserved.
1
+ # Copyright 2024 The HuggingFace Team. All rights reserved.
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
@@ -33,7 +33,7 @@ from ...utils import (
33
33
  unscale_lora_layers,
34
34
  )
35
35
  from ...utils.torch_utils import randn_tensor
36
- from ..pipeline_utils import DiffusionPipeline
36
+ from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin
37
37
  from . import TextToVideoSDPipelineOutput
38
38
 
39
39
 
@@ -52,7 +52,7 @@ EXAMPLE_DOC_STRING = """
52
52
  >>> pipe.enable_model_cpu_offload()
53
53
 
54
54
  >>> prompt = "Spiderman is surfing"
55
- >>> video_frames = pipe(prompt).frames
55
+ >>> video_frames = pipe(prompt).frames[0]
56
56
  >>> video_path = export_to_video(video_frames)
57
57
  >>> video_path
58
58
  ```
@@ -76,12 +76,12 @@ def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type:
76
76
  outputs = torch.stack(outputs)
77
77
 
78
78
  elif not output_type == "pil":
79
- raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil]")
79
+ raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']")
80
80
 
81
81
  return outputs
82
82
 
83
83
 
84
- class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin):
84
+ class TextToVideoSDPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin):
85
85
  r"""
86
86
  Pipeline for text-to-video generation.
87
87
 
@@ -129,39 +129,6 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
129
129
  self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
130
130
  self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor)
131
131
 
132
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_slicing
133
- def enable_vae_slicing(self):
134
- r"""
135
- Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to
136
- compute decoding in several steps. This is useful to save some memory and allow larger batch sizes.
137
- """
138
- self.vae.enable_slicing()
139
-
140
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_slicing
141
- def disable_vae_slicing(self):
142
- r"""
143
- Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to
144
- computing decoding in one step.
145
- """
146
- self.vae.disable_slicing()
147
-
148
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_vae_tiling
149
- def enable_vae_tiling(self):
150
- r"""
151
- Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to
152
- compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow
153
- processing larger images.
154
- """
155
- self.vae.enable_tiling()
156
-
157
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_vae_tiling
158
- def disable_vae_tiling(self):
159
- r"""
160
- Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to
161
- computing decoding in one step.
162
- """
163
- self.vae.disable_tiling()
164
-
165
132
  # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt
166
133
  def _encode_prompt(
167
134
  self,
@@ -256,7 +223,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
256
223
  batch_size = prompt_embeds.shape[0]
257
224
 
258
225
  if prompt_embeds is None:
259
- # textual inversion: procecss multi-vector tokens if necessary
226
+ # textual inversion: process multi-vector tokens if necessary
260
227
  if isinstance(self, TextualInversionLoaderMixin):
261
228
  prompt = self.maybe_convert_prompt(prompt, self.tokenizer)
262
229
 
@@ -338,7 +305,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
338
305
  else:
339
306
  uncond_tokens = negative_prompt
340
307
 
341
- # textual inversion: procecss multi-vector tokens if necessary
308
+ # textual inversion: process multi-vector tokens if necessary
342
309
  if isinstance(self, TextualInversionLoaderMixin):
343
310
  uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer)
344
311
 
@@ -384,18 +351,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
384
351
  latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width)
385
352
 
386
353
  image = self.vae.decode(latents).sample
387
- video = (
388
- image[None, :]
389
- .reshape(
390
- (
391
- batch_size,
392
- num_frames,
393
- -1,
394
- )
395
- + image.shape[2:]
396
- )
397
- .permute(0, 2, 1, 3, 4)
398
- )
354
+ video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4)
399
355
  # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
400
356
  video = video.float()
401
357
  return video
@@ -418,7 +374,7 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
418
374
  extra_step_kwargs["generator"] = generator
419
375
  return extra_step_kwargs
420
376
 
421
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.check_inputs
377
+ # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs
422
378
  def check_inputs(
423
379
  self,
424
380
  prompt,
@@ -496,34 +452,6 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
496
452
  latents = latents * self.scheduler.init_noise_sigma
497
453
  return latents
498
454
 
499
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.enable_freeu
500
- def enable_freeu(self, s1: float, s2: float, b1: float, b2: float):
501
- r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497.
502
-
503
- The suffixes after the scaling factors represent the stages where they are being applied.
504
-
505
- Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values
506
- that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL.
507
-
508
- Args:
509
- s1 (`float`):
510
- Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to
511
- mitigate "oversmoothing effect" in the enhanced denoising process.
512
- s2 (`float`):
513
- Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to
514
- mitigate "oversmoothing effect" in the enhanced denoising process.
515
- b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features.
516
- b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features.
517
- """
518
- if not hasattr(self, "unet"):
519
- raise ValueError("The pipeline must have `unet` for using FreeU.")
520
- self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2)
521
-
522
- # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.disable_freeu
523
- def disable_freeu(self):
524
- """Disables the FreeU mechanism if enabled."""
525
- self.unet.disable_freeu()
526
-
527
455
  @torch.no_grad()
528
456
  @replace_example_docstring(EXAMPLE_DOC_STRING)
529
457
  def __call__(
@@ -719,13 +647,14 @@ class TextToVideoSDPipeline(DiffusionPipeline, TextualInversionLoaderMixin, Lora
719
647
  step_idx = i // getattr(self.scheduler, "order", 1)
720
648
  callback(step_idx, t, latents)
721
649
 
650
+ # 8. Post processing
722
651
  if output_type == "latent":
723
- return TextToVideoSDPipelineOutput(frames=latents)
724
-
725
- video_tensor = self.decode_latents(latents)
726
- video = tensor2vid(video_tensor, self.image_processor, output_type)
652
+ video = latents
653
+ else:
654
+ video_tensor = self.decode_latents(latents)
655
+ video = tensor2vid(video_tensor, self.image_processor, output_type)
727
656
 
728
- # Offload all models
657
+ # 9. Offload all models
729
658
  self.maybe_free_model_hooks()
730
659
 
731
660
  if not return_dict: