optimum-rbln 0.8.1a0__py3-none-any.whl → 0.8.1a2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. optimum/rbln/__init__.py +2 -0
  2. optimum/rbln/__version__.py +2 -2
  3. optimum/rbln/configuration_utils.py +53 -33
  4. optimum/rbln/diffusers/configurations/models/configuration_autoencoder_kl.py +9 -2
  5. optimum/rbln/diffusers/configurations/models/configuration_controlnet.py +4 -2
  6. optimum/rbln/diffusers/configurations/models/configuration_prior_transformer.py +9 -2
  7. optimum/rbln/diffusers/configurations/models/configuration_transformer_sd3.py +4 -2
  8. optimum/rbln/diffusers/configurations/models/configuration_unet_2d_condition.py +9 -2
  9. optimum/rbln/diffusers/configurations/models/configuration_vq_model.py +9 -2
  10. optimum/rbln/diffusers/configurations/pipelines/configuration_controlnet.py +33 -9
  11. optimum/rbln/diffusers/configurations/pipelines/configuration_kandinsky2_2.py +30 -12
  12. optimum/rbln/diffusers/configurations/pipelines/configuration_stable_diffusion.py +22 -6
  13. optimum/rbln/diffusers/configurations/pipelines/configuration_stable_diffusion_3.py +16 -6
  14. optimum/rbln/diffusers/configurations/pipelines/configuration_stable_diffusion_xl.py +16 -6
  15. optimum/rbln/diffusers/modeling_diffusers.py +16 -26
  16. optimum/rbln/diffusers/models/autoencoders/autoencoder_kl.py +11 -0
  17. optimum/rbln/diffusers/models/autoencoders/vae.py +1 -8
  18. optimum/rbln/diffusers/models/autoencoders/vq_model.py +11 -0
  19. optimum/rbln/diffusers/models/controlnet.py +13 -7
  20. optimum/rbln/diffusers/models/transformers/prior_transformer.py +10 -0
  21. optimum/rbln/diffusers/models/transformers/transformer_sd3.py +2 -0
  22. optimum/rbln/diffusers/models/unets/unet_2d_condition.py +7 -0
  23. optimum/rbln/diffusers/pipelines/controlnet/multicontrolnet.py +1 -4
  24. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet.py +7 -0
  25. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py +7 -0
  26. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py +7 -0
  27. optimum/rbln/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py +7 -0
  28. optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py +7 -0
  29. optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py +48 -27
  30. optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py +7 -0
  31. optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpaint.py +7 -0
  32. optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py +7 -0
  33. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py +7 -0
  34. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +7 -0
  35. optimum/rbln/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +7 -0
  36. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3.py +7 -0
  37. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_img2img.py +7 -0
  38. optimum/rbln/diffusers/pipelines/stable_diffusion_3/pipeline_stable_diffusion_3_inpaint.py +7 -0
  39. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py +7 -0
  40. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py +7 -0
  41. optimum/rbln/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py +7 -0
  42. optimum/rbln/modeling.py +33 -35
  43. optimum/rbln/modeling_base.py +45 -107
  44. optimum/rbln/transformers/__init__.py +39 -47
  45. optimum/rbln/transformers/configuration_generic.py +16 -13
  46. optimum/rbln/transformers/modeling_generic.py +18 -19
  47. optimum/rbln/transformers/modeling_rope_utils.py +5 -2
  48. optimum/rbln/transformers/models/__init__.py +46 -4
  49. optimum/rbln/transformers/models/audio_spectrogram_transformer/__init__.py +17 -0
  50. optimum/rbln/transformers/models/audio_spectrogram_transformer/configuration_audio_spectrogram_transformer.py +21 -0
  51. optimum/rbln/transformers/models/audio_spectrogram_transformer/modeling_audio_spectrogram_transformer.py +28 -0
  52. optimum/rbln/transformers/models/auto/auto_factory.py +35 -12
  53. optimum/rbln/transformers/models/bart/bart_architecture.py +14 -1
  54. optimum/rbln/transformers/models/blip_2/modeling_blip_2.py +35 -4
  55. optimum/rbln/transformers/models/clip/configuration_clip.py +3 -3
  56. optimum/rbln/transformers/models/clip/modeling_clip.py +11 -12
  57. optimum/rbln/transformers/models/decoderonly/configuration_decoderonly.py +111 -14
  58. optimum/rbln/transformers/models/decoderonly/decoderonly_architecture.py +102 -35
  59. optimum/rbln/transformers/models/decoderonly/modeling_decoderonly.py +229 -175
  60. optimum/rbln/transformers/models/distilbert/__init__.py +19 -0
  61. optimum/rbln/transformers/models/distilbert/configuration_distilbert.py +19 -0
  62. optimum/rbln/transformers/models/distilbert/modeling_distilbert.py +19 -0
  63. optimum/rbln/transformers/models/exaone/configuration_exaone.py +24 -1
  64. optimum/rbln/transformers/models/exaone/exaone_architecture.py +5 -1
  65. optimum/rbln/transformers/models/exaone/modeling_exaone.py +66 -5
  66. optimum/rbln/transformers/models/gemma/configuration_gemma.py +24 -1
  67. optimum/rbln/transformers/models/gemma/gemma_architecture.py +5 -1
  68. optimum/rbln/transformers/models/gemma/modeling_gemma.py +49 -0
  69. optimum/rbln/transformers/models/gemma3/configuration_gemma3.py +3 -3
  70. optimum/rbln/transformers/models/gemma3/gemma3_architecture.py +18 -250
  71. optimum/rbln/transformers/models/gemma3/modeling_gemma3.py +106 -236
  72. optimum/rbln/transformers/models/gpt2/configuration_gpt2.py +4 -1
  73. optimum/rbln/transformers/models/gpt2/gpt2_architecture.py +6 -1
  74. optimum/rbln/transformers/models/idefics3/configuration_idefics3.py +12 -2
  75. optimum/rbln/transformers/models/idefics3/modeling_idefics3.py +41 -4
  76. optimum/rbln/transformers/models/llama/configuration_llama.py +24 -1
  77. optimum/rbln/transformers/models/llama/modeling_llama.py +49 -0
  78. optimum/rbln/transformers/models/llava_next/configuration_llava_next.py +2 -2
  79. optimum/rbln/transformers/models/llava_next/modeling_llava_next.py +32 -4
  80. optimum/rbln/transformers/models/midm/configuration_midm.py +24 -1
  81. optimum/rbln/transformers/models/midm/midm_architecture.py +6 -1
  82. optimum/rbln/transformers/models/midm/modeling_midm.py +66 -5
  83. optimum/rbln/transformers/models/mistral/configuration_mistral.py +24 -1
  84. optimum/rbln/transformers/models/mistral/modeling_mistral.py +62 -4
  85. optimum/rbln/transformers/models/opt/configuration_opt.py +4 -1
  86. optimum/rbln/transformers/models/opt/modeling_opt.py +10 -0
  87. optimum/rbln/transformers/models/opt/opt_architecture.py +7 -1
  88. optimum/rbln/transformers/models/phi/configuration_phi.py +24 -1
  89. optimum/rbln/transformers/models/phi/modeling_phi.py +49 -0
  90. optimum/rbln/transformers/models/phi/phi_architecture.py +1 -1
  91. optimum/rbln/transformers/models/qwen2/configuration_qwen2.py +24 -1
  92. optimum/rbln/transformers/models/qwen2/modeling_qwen2.py +67 -4
  93. optimum/rbln/transformers/models/qwen2_5_vl/configuration_qwen2_5_vl.py +15 -3
  94. optimum/rbln/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py +58 -27
  95. optimum/rbln/transformers/models/qwen2_5_vl/qwen2_5_vl_architecture.py +47 -2
  96. optimum/rbln/transformers/models/resnet/__init__.py +23 -0
  97. optimum/rbln/transformers/models/resnet/configuration_resnet.py +20 -0
  98. optimum/rbln/transformers/models/resnet/modeling_resnet.py +22 -0
  99. optimum/rbln/transformers/models/roberta/__init__.py +24 -0
  100. optimum/rbln/transformers/{configuration_alias.py → models/roberta/configuration_roberta.py} +4 -30
  101. optimum/rbln/transformers/{modeling_alias.py → models/roberta/modeling_roberta.py} +2 -32
  102. optimum/rbln/transformers/models/seq2seq/__init__.py +1 -1
  103. optimum/rbln/transformers/models/seq2seq/{configuration_seq2seq2.py → configuration_seq2seq.py} +2 -2
  104. optimum/rbln/transformers/models/seq2seq/modeling_seq2seq.py +1 -1
  105. optimum/rbln/transformers/models/seq2seq/seq2seq_architecture.py +41 -3
  106. optimum/rbln/transformers/models/siglip/configuration_siglip.py +3 -0
  107. optimum/rbln/transformers/models/siglip/modeling_siglip.py +62 -21
  108. optimum/rbln/transformers/models/t5/modeling_t5.py +46 -4
  109. optimum/rbln/transformers/models/t5/t5_architecture.py +5 -1
  110. optimum/rbln/transformers/models/{time_series_transformers → time_series_transformer}/__init__.py +1 -1
  111. optimum/rbln/transformers/models/{time_series_transformers → time_series_transformer}/configuration_time_series_transformer.py +2 -2
  112. optimum/rbln/transformers/models/{time_series_transformers/modeling_time_series_transformers.py → time_series_transformer/modeling_time_series_transformer.py} +14 -9
  113. optimum/rbln/transformers/models/vit/__init__.py +19 -0
  114. optimum/rbln/transformers/models/vit/configuration_vit.py +19 -0
  115. optimum/rbln/transformers/models/vit/modeling_vit.py +19 -0
  116. optimum/rbln/transformers/models/wav2vec2/__init__.py +1 -1
  117. optimum/rbln/transformers/models/wav2vec2/modeling_wav2vec2.py +1 -1
  118. optimum/rbln/transformers/models/whisper/configuration_whisper.py +3 -1
  119. optimum/rbln/transformers/models/whisper/modeling_whisper.py +35 -15
  120. optimum/rbln/transformers/models/xlm_roberta/__init__.py +16 -2
  121. optimum/rbln/transformers/models/xlm_roberta/configuration_xlm_roberta.py +15 -2
  122. optimum/rbln/transformers/models/xlm_roberta/modeling_xlm_roberta.py +12 -3
  123. optimum/rbln/utils/model_utils.py +20 -0
  124. optimum/rbln/utils/submodule.py +6 -8
  125. {optimum_rbln-0.8.1a0.dist-info → optimum_rbln-0.8.1a2.dist-info}/METADATA +2 -2
  126. {optimum_rbln-0.8.1a0.dist-info → optimum_rbln-0.8.1a2.dist-info}/RECORD +130 -117
  127. /optimum/rbln/transformers/models/{time_series_transformers → time_series_transformer}/time_series_transformers_architecture.py +0 -0
  128. /optimum/rbln/transformers/models/wav2vec2/{configuration_wav2vec.py → configuration_wav2vec2.py} +0 -0
  129. {optimum_rbln-0.8.1a0.dist-info → optimum_rbln-0.8.1a2.dist-info}/WHEEL +0 -0
  130. {optimum_rbln-0.8.1a0.dist-info → optimum_rbln-0.8.1a2.dist-info}/licenses/LICENSE +0 -0
@@ -12,14 +12,18 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from typing import Optional, Tuple
15
+ from typing import Any, Dict, Optional, Tuple
16
16
 
17
17
  from ....configuration_utils import RBLNModelConfig
18
18
  from ....transformers import RBLNCLIPTextModelWithProjectionConfig, RBLNT5EncoderModelConfig
19
19
  from ..models import RBLNAutoencoderKLConfig, RBLNSD3Transformer2DModelConfig
20
20
 
21
21
 
22
- class _RBLNStableDiffusion3PipelineBaseConfig(RBLNModelConfig):
22
+ class RBLNStableDiffusion3PipelineBaseConfig(RBLNModelConfig):
23
+ """
24
+ Base configuration for Stable Diffusion 3 pipelines.
25
+ """
26
+
23
27
  submodules = ["transformer", "text_encoder", "text_encoder_2", "text_encoder_3", "vae"]
24
28
  _vae_uses_encoder = False
25
29
 
@@ -40,7 +44,7 @@ class _RBLNStableDiffusion3PipelineBaseConfig(RBLNModelConfig):
40
44
  height: Optional[int] = None,
41
45
  width: Optional[int] = None,
42
46
  guidance_scale: Optional[float] = None,
43
- **kwargs,
47
+ **kwargs: Dict[str, Any],
44
48
  ):
45
49
  """
46
50
  Args:
@@ -153,13 +157,19 @@ class _RBLNStableDiffusion3PipelineBaseConfig(RBLNModelConfig):
153
157
  return self.vae.sample_size
154
158
 
155
159
 
156
- class RBLNStableDiffusion3PipelineConfig(_RBLNStableDiffusion3PipelineBaseConfig):
160
+ class RBLNStableDiffusion3PipelineConfig(RBLNStableDiffusion3PipelineBaseConfig):
161
+ """Config for SD3 Text2Img Pipeline"""
162
+
157
163
  _vae_uses_encoder = False
158
164
 
159
165
 
160
- class RBLNStableDiffusion3Img2ImgPipelineConfig(_RBLNStableDiffusion3PipelineBaseConfig):
166
+ class RBLNStableDiffusion3Img2ImgPipelineConfig(RBLNStableDiffusion3PipelineBaseConfig):
167
+ """Config for SD3 Img2Img Pipeline"""
168
+
161
169
  _vae_uses_encoder = True
162
170
 
163
171
 
164
- class RBLNStableDiffusion3InpaintPipelineConfig(_RBLNStableDiffusion3PipelineBaseConfig):
172
+ class RBLNStableDiffusion3InpaintPipelineConfig(RBLNStableDiffusion3PipelineBaseConfig):
173
+ """Config for SD3 Inpainting Pipeline"""
174
+
165
175
  _vae_uses_encoder = True
@@ -12,14 +12,18 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from typing import Optional, Tuple
15
+ from typing import Any, Dict, Optional, Tuple
16
16
 
17
17
  from ....configuration_utils import RBLNModelConfig
18
18
  from ....transformers import RBLNCLIPTextModelConfig, RBLNCLIPTextModelWithProjectionConfig
19
19
  from ..models import RBLNAutoencoderKLConfig, RBLNUNet2DConditionModelConfig
20
20
 
21
21
 
22
- class _RBLNStableDiffusionXLPipelineBaseConfig(RBLNModelConfig):
22
+ class RBLNStableDiffusionXLPipelineBaseConfig(RBLNModelConfig):
23
+ """
24
+ Base configuration for Stable Diffusion XL pipelines.
25
+ """
26
+
23
27
  submodules = ["text_encoder", "text_encoder_2", "unet", "vae"]
24
28
  _vae_uses_encoder = False
25
29
 
@@ -38,7 +42,7 @@ class _RBLNStableDiffusionXLPipelineBaseConfig(RBLNModelConfig):
38
42
  sample_size: Optional[Tuple[int, int]] = None,
39
43
  image_size: Optional[Tuple[int, int]] = None,
40
44
  guidance_scale: Optional[float] = None,
41
- **kwargs,
45
+ **kwargs: Dict[str, Any],
42
46
  ):
43
47
  """
44
48
  Args:
@@ -134,13 +138,19 @@ class _RBLNStableDiffusionXLPipelineBaseConfig(RBLNModelConfig):
134
138
  return self.vae.sample_size
135
139
 
136
140
 
137
- class RBLNStableDiffusionXLPipelineConfig(_RBLNStableDiffusionXLPipelineBaseConfig):
141
+ class RBLNStableDiffusionXLPipelineConfig(RBLNStableDiffusionXLPipelineBaseConfig):
142
+ """Config for SDXL Text2Img Pipeline"""
143
+
138
144
  _vae_uses_encoder = False
139
145
 
140
146
 
141
- class RBLNStableDiffusionXLImg2ImgPipelineConfig(_RBLNStableDiffusionXLPipelineBaseConfig):
147
+ class RBLNStableDiffusionXLImg2ImgPipelineConfig(RBLNStableDiffusionXLPipelineBaseConfig):
148
+ """Config for SDXL Img2Img Pipeline"""
149
+
142
150
  _vae_uses_encoder = True
143
151
 
144
152
 
145
- class RBLNStableDiffusionXLInpaintPipelineConfig(_RBLNStableDiffusionXLPipelineBaseConfig):
153
+ class RBLNStableDiffusionXLInpaintPipelineConfig(RBLNStableDiffusionXLPipelineBaseConfig):
154
+ """Config for SDXL Inpainting Pipeline"""
155
+
146
156
  _vae_uses_encoder = True
@@ -19,10 +19,11 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Type, Union
19
19
 
20
20
  import torch
21
21
 
22
- from ..configuration_utils import ContextRblnConfig, RBLNModelConfig
22
+ from ..configuration_utils import ContextRblnConfig, RBLNModelConfig, get_rbln_config_class
23
23
  from ..modeling import RBLNModel
24
24
  from ..utils.decorator_utils import remove_compile_time_kwargs
25
25
  from ..utils.logging import get_logger
26
+ from ..utils.model_utils import get_rbln_model_cls
26
27
 
27
28
 
28
29
  logger = get_logger(__name__)
@@ -110,18 +111,10 @@ class RBLNDiffusionMixin:
110
111
 
111
112
  @classmethod
112
113
  def get_rbln_config_class(cls) -> Type[RBLNModelConfig]:
113
- """
114
- Lazily loads and caches the corresponding RBLN model config class.
115
- """
114
+ # Lazily loads and caches the corresponding RBLN model config class.
116
115
  if cls._rbln_config_class is None:
117
116
  rbln_config_class_name = cls.__name__ + "Config"
118
- library = importlib.import_module("optimum.rbln")
119
- cls._rbln_config_class = getattr(library, rbln_config_class_name, None)
120
- if cls._rbln_config_class is None:
121
- raise ValueError(
122
- f"RBLN config class {rbln_config_class_name} not found. This is an internal error. "
123
- "Please report it to the developers."
124
- )
117
+ cls._rbln_config_class = get_rbln_config_class(rbln_config_class_name)
125
118
  return cls._rbln_config_class
126
119
 
127
120
  @classmethod
@@ -143,7 +136,7 @@ class RBLNDiffusionMixin:
143
136
  lora_ids: Optional[Union[str, List[str]]] = None,
144
137
  lora_weights_names: Optional[Union[str, List[str]]] = None,
145
138
  lora_scales: Optional[Union[float, List[float]]] = None,
146
- **kwargs,
139
+ **kwargs: Dict[str, Any],
147
140
  ) -> "RBLNDiffusionMixin":
148
141
  """
149
142
  Load a pretrained diffusion pipeline from a model checkpoint, with optional compilation for RBLN NPUs.
@@ -157,24 +150,25 @@ class RBLNDiffusionMixin:
157
150
  Args:
158
151
  model_id (`str`):
159
152
  The model ID or path to the pretrained model to load. Can be either:
153
+
160
154
  - A model ID from the HuggingFace Hub
161
155
  - A local path to a saved model directory
162
- export (`bool`, *optional*, defaults to `False`):
156
+ export:
163
157
  If True, takes a PyTorch model from `model_id` and compiles it for RBLN NPU execution.
164
158
  If False, loads an already compiled RBLN model from `model_id` without recompilation.
165
- model_save_dir (`os.PathLike`, *optional*):
159
+ model_save_dir:
166
160
  Directory to save the compiled model artifacts. Only used when `export=True`.
167
161
  If not provided and `export=True`, a temporary directory is used.
168
- rbln_config (`Dict[str, Any]`, *optional*, defaults to `{}`):
162
+ rbln_config:
169
163
  Configuration options for RBLN compilation. Can include settings for specific submodules
170
164
  such as `text_encoder`, `unet`, and `vae`. Configuration can be tailored to the specific
171
165
  pipeline being compiled.
172
- lora_ids (`str` or `List[str]`, *optional*):
166
+ lora_ids:
173
167
  LoRA adapter ID(s) to load and apply before compilation. LoRA weights are fused
174
168
  into the model weights during compilation. Only used when `export=True`.
175
- lora_weights_names (`str` or `List[str]`, *optional*):
169
+ lora_weights_names:
176
170
  Names of specific LoRA weight files to load, corresponding to lora_ids. Only used when `export=True`.
177
- lora_scales (`float` or `List[float]`, *optional*):
171
+ lora_scales:
178
172
  Scaling factor(s) to apply to the LoRA adapter(s). Only used when `export=True`.
179
173
  **kwargs:
180
174
  Additional arguments to pass to the underlying diffusion pipeline constructor or the
@@ -182,8 +176,8 @@ class RBLNDiffusionMixin:
182
176
  or the particular diffusion pipeline being used.
183
177
 
184
178
  Returns:
185
- `RBLNDiffusionMixin`: A compiled or loaded diffusion pipeline that can be used for inference on RBLN NPU.
186
- The returned object is an instance of the class that called this method, inheriting from RBLNDiffusionMixin.
179
+ A compiled or loaded diffusion pipeline that can be used for inference on RBLN NPU.
180
+ The returned object is an instance of the class that called this method, inheriting from RBLNDiffusionMixin.
187
181
  """
188
182
  rbln_config, kwargs = cls.get_rbln_config_class().initialize_from_kwargs(rbln_config, **kwargs)
189
183
 
@@ -210,7 +204,7 @@ class RBLNDiffusionMixin:
210
204
  "Expected 'optimum.rbln'. Please check the model_index.json configuration."
211
205
  )
212
206
 
213
- submodule_cls: Type[RBLNModel] = getattr(importlib.import_module("optimum.rbln"), class_name)
207
+ submodule_cls = get_rbln_model_cls(class_name)
214
208
  submodule_config = getattr(rbln_config, submodule_name)
215
209
  submodule = submodule_cls.from_pretrained(
216
210
  model_id, export=False, subfolder=submodule_name, rbln_config=submodule_config
@@ -293,7 +287,6 @@ class RBLNDiffusionMixin:
293
287
  elif isinstance(submodule, RBLNModel):
294
288
  pass
295
289
  elif submodule_name == "controlnet" and hasattr(submodule, "nets"):
296
- # In case of multicontrolnet
297
290
  submodule = cls._compile_multicontrolnet(
298
291
  controlnets=submodule,
299
292
  model_save_dir=model_save_dir,
@@ -301,11 +294,8 @@ class RBLNDiffusionMixin:
301
294
  prefix=prefix,
302
295
  )
303
296
  elif isinstance(submodule, torch.nn.Module):
304
- submodule_cls: RBLNModel = getattr(
305
- importlib.import_module("optimum.rbln"), f"RBLN{submodule.__class__.__name__}"
306
- )
307
297
  subfolder = prefix + submodule_name
308
- submodule = submodule_cls.from_model(
298
+ submodule = submodule_rbln_cls.from_model(
309
299
  model=submodule,
310
300
  subfolder=subfolder,
311
301
  model_save_dir=model_save_dir,
@@ -38,6 +38,17 @@ logger = get_logger(__name__)
38
38
 
39
39
 
40
40
  class RBLNAutoencoderKL(RBLNModel):
41
+ """
42
+ RBLN implementation of AutoencoderKL (VAE) for diffusion models.
43
+
44
+ This model is used to accelerate AutoencoderKL (VAE) models from diffusers library on RBLN NPUs.
45
+ It can be configured to include both encoder and decoder, or just the decoder part for latent-to-image
46
+ conversion.
47
+
48
+ This class inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods
49
+ the library implements for all its models.
50
+ """
51
+
41
52
  auto_model_class = AutoencoderKL
42
53
  hf_library_name = "diffusers"
43
54
  _rbln_config_class = RBLNAutoencoderKLConfig
@@ -12,22 +12,15 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- from typing import TYPE_CHECKING, List
15
+ from typing import List
16
16
 
17
17
  import torch
18
18
  from diffusers import AutoencoderKL, VQModel
19
19
  from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution
20
20
 
21
- from ....utils.logging import get_logger
22
21
  from ....utils.runtime_utils import RBLNPytorchRuntime
23
22
 
24
23
 
25
- if TYPE_CHECKING:
26
- import torch
27
-
28
- logger = get_logger(__name__)
29
-
30
-
31
24
  class RBLNRuntimeVAEEncoder(RBLNPytorchRuntime):
32
25
  def encode(self, x: torch.FloatTensor, **kwargs) -> torch.FloatTensor:
33
26
  moments = self.forward(x.contiguous())
@@ -35,6 +35,17 @@ logger = get_logger(__name__)
35
35
 
36
36
 
37
37
  class RBLNVQModel(RBLNModel):
38
+ """
39
+ RBLN implementation of VQModel for diffusion models.
40
+
41
+ This model is used to accelerate VQModel models from diffusers library on RBLN NPUs.
42
+ It can be configured to include both encoder and decoder, or just the decoder part for latent-to-image
43
+ conversion.
44
+
45
+ This class inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods
46
+ the library implements for all its models.
47
+ """
48
+
38
49
  auto_model_class = VQModel
39
50
  config_name = "config.json"
40
51
  hf_library_name = "diffusers"
@@ -12,7 +12,6 @@
12
12
  # See the License for the specific language governing permissions and
13
13
  # limitations under the License.
14
14
 
15
- import importlib
16
15
  from typing import TYPE_CHECKING, Dict, Optional, Union
17
16
 
18
17
  import torch
@@ -23,6 +22,7 @@ from transformers import PretrainedConfig
23
22
  from ...configuration_utils import RBLNCompileConfig, RBLNModelConfig
24
23
  from ...modeling import RBLNModel
25
24
  from ...utils.logging import get_logger
25
+ from ...utils.model_utils import get_rbln_model_cls
26
26
  from ..configurations import RBLNControlNetModelConfig
27
27
  from ..modeling_diffusers import RBLNDiffusionMixin, RBLNDiffusionMixinConfig
28
28
 
@@ -98,6 +98,15 @@ class _ControlNetModel_Cross_Attention(torch.nn.Module):
98
98
 
99
99
 
100
100
  class RBLNControlNetModel(RBLNModel):
101
+ """
102
+ RBLN implementation of ControlNetModel for diffusion models.
103
+
104
+ This model is used to accelerate ControlNetModel models from diffusers library on RBLN NPUs.
105
+
106
+ This class inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods
107
+ the library implements for all its models.
108
+ """
109
+
101
110
  hf_library_name = "diffusers"
102
111
  auto_model_class = ControlNetModel
103
112
  output_class = ControlNetOutput
@@ -122,13 +131,10 @@ class RBLNControlNetModel(RBLNModel):
122
131
 
123
132
  @classmethod
124
133
  def update_rbln_config_using_pipe(
125
- cls,
126
- pipe: RBLNDiffusionMixin,
127
- rbln_config: "RBLNDiffusionMixinConfig",
128
- submodule_name: str,
134
+ cls, pipe: RBLNDiffusionMixin, rbln_config: "RBLNDiffusionMixinConfig", submodule_name: str
129
135
  ) -> "RBLNDiffusionMixinConfig":
130
- rbln_vae_cls = getattr(importlib.import_module("optimum.rbln"), f"RBLN{pipe.vae.__class__.__name__}")
131
- rbln_unet_cls = getattr(importlib.import_module("optimum.rbln"), f"RBLN{pipe.unet.__class__.__name__}")
136
+ rbln_vae_cls = get_rbln_model_cls(f"RBLN{pipe.vae.__class__.__name__}")
137
+ rbln_unet_cls = get_rbln_model_cls(f"RBLN{pipe.unet.__class__.__name__}")
132
138
 
133
139
  rbln_config.controlnet.max_seq_len = pipe.text_encoder.config.max_position_embeddings
134
140
  text_model_hidden_size = pipe.text_encoder_2.config.hidden_size if hasattr(pipe, "text_encoder_2") else None
@@ -56,6 +56,16 @@ class _PriorTransformer(torch.nn.Module):
56
56
 
57
57
 
58
58
  class RBLNPriorTransformer(RBLNModel):
59
+ """
60
+ RBLN implementation of PriorTransformer for diffusion models like Kandinsky V2.2.
61
+
62
+ The Prior Transformer takes text and/or image embeddings from encoders (like CLIP) and
63
+ maps them to a shared latent space that guides the diffusion process to generate the desired image.
64
+
65
+ This class inherits from [`RBLNModel`]. Check the superclass documentation for the generic methods
66
+ the library implements for all its models.
67
+ """
68
+
59
69
  hf_library_name = "diffusers"
60
70
  auto_model_class = PriorTransformer
61
71
  _output_class = PriorTransformerOutput
@@ -59,6 +59,8 @@ class SD3Transformer2DModelWrapper(torch.nn.Module):
59
59
 
60
60
 
61
61
  class RBLNSD3Transformer2DModel(RBLNModel):
62
+ """RBLN wrapper for the Stable Diffusion 3 MMDiT Transformer model."""
63
+
62
64
  hf_library_name = "diffusers"
63
65
  auto_model_class = SD3Transformer2DModel
64
66
  _output_class = Transformer2DModelOutput
@@ -140,6 +140,13 @@ class _UNet_Kandinsky(torch.nn.Module):
140
140
 
141
141
 
142
142
  class RBLNUNet2DConditionModel(RBLNModel):
143
+ """
144
+ Configuration class for RBLN UNet2DCondition models.
145
+
146
+ This class inherits from RBLNModelConfig and provides specific configuration options
147
+ for UNet2DCondition models used in diffusion-based image generation.
148
+ """
149
+
143
150
  hf_library_name = "diffusers"
144
151
  auto_model_class = UNet2DConditionModel
145
152
  _rbln_config_class = RBLNUNet2DConditionModelConfig
@@ -14,7 +14,7 @@
14
14
 
15
15
  import os
16
16
  from pathlib import Path
17
- from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
17
+ from typing import Any, Dict, List, Optional, Union
18
18
 
19
19
  import torch
20
20
  from diffusers.pipelines.controlnet.multicontrolnet import MultiControlNetModel
@@ -24,9 +24,6 @@ from ....utils.logging import get_logger
24
24
  from ...models.controlnet import RBLNControlNetModel
25
25
 
26
26
 
27
- if TYPE_CHECKING:
28
- pass
29
-
30
27
  logger = get_logger(__name__)
31
28
 
32
29
 
@@ -49,6 +49,13 @@ logger = get_logger(__name__)
49
49
 
50
50
 
51
51
  class RBLNStableDiffusionControlNetPipeline(RBLNDiffusionMixin, StableDiffusionControlNetPipeline):
52
+ """
53
+ RBLN-accelerated implementation of Stable Diffusion pipeline with ControlNet for guided text-to-image generation.
54
+
55
+ This pipeline compiles Stable Diffusion and ControlNet models to run efficiently on RBLN NPUs, enabling high-performance
56
+ inference for generating images with precise structural control using conditioning inputs like edges, depth, or poses.
57
+ """
58
+
52
59
  original_class = StableDiffusionControlNetPipeline
53
60
  _rbln_config_class = RBLNStableDiffusionControlNetPipelineConfig
54
61
  _submodules = ["text_encoder", "unet", "vae", "controlnet"]
@@ -47,6 +47,13 @@ logger = logging.get_logger(__name__)
47
47
 
48
48
 
49
49
  class RBLNStableDiffusionControlNetImg2ImgPipeline(RBLNDiffusionMixin, StableDiffusionControlNetImg2ImgPipeline):
50
+ """
51
+ RBLN-accelerated implementation of Stable Diffusion pipeline with ControlNet for guided image-to-image generation.
52
+
53
+ This pipeline compiles Stable Diffusion and ControlNet models to run efficiently on RBLN NPUs, enabling high-performance
54
+ inference for transforming input images with precise structural control and conditioning guidance.
55
+ """
56
+
50
57
  original_class = StableDiffusionControlNetImg2ImgPipeline
51
58
  _submodules = ["text_encoder", "unet", "vae", "controlnet"]
52
59
  _rbln_config_class = RBLNStableDiffusionControlNetImg2ImgPipelineConfig
@@ -47,6 +47,13 @@ logger = logging.get_logger(__name__)
47
47
 
48
48
 
49
49
  class RBLNStableDiffusionXLControlNetPipeline(RBLNDiffusionMixin, StableDiffusionXLControlNetPipeline):
50
+ """
51
+ RBLN-accelerated implementation of Stable Diffusion XL pipeline with ControlNet for high-resolution guided text-to-image generation.
52
+
53
+ This pipeline compiles Stable Diffusion XL and ControlNet models to run efficiently on RBLN NPUs, enabling high-performance
54
+ inference for generating high-quality images with precise structural control and enhanced detail preservation.
55
+ """
56
+
50
57
  original_class = StableDiffusionXLControlNetPipeline
51
58
  _rbln_config_class = RBLNStableDiffusionXLControlNetPipelineConfig
52
59
  _submodules = ["text_encoder", "text_encoder_2", "unet", "vae", "controlnet"]
@@ -47,6 +47,13 @@ logger = logging.get_logger(__name__)
47
47
 
48
48
 
49
49
  class RBLNStableDiffusionXLControlNetImg2ImgPipeline(RBLNDiffusionMixin, StableDiffusionXLControlNetImg2ImgPipeline):
50
+ """
51
+ RBLN-accelerated implementation of Stable Diffusion XL pipeline with ControlNet for high-resolution guided image-to-image generation.
52
+
53
+ This pipeline compiles Stable Diffusion XL and ControlNet models to run efficiently on RBLN NPUs, enabling high-performance
54
+ inference for transforming input images with precise structural control and enhanced quality preservation.
55
+ """
56
+
50
57
  original_class = StableDiffusionXLControlNetImg2ImgPipeline
51
58
  _rbln_config_class = RBLNStableDiffusionXLControlNetImg2ImgPipelineConfig
52
59
  _submodules = ["text_encoder", "text_encoder_2", "unet", "vae", "controlnet"]
@@ -19,6 +19,13 @@ from ...modeling_diffusers import RBLNDiffusionMixin
19
19
 
20
20
 
21
21
  class RBLNKandinskyV22Pipeline(RBLNDiffusionMixin, KandinskyV22Pipeline):
22
+ """
23
+ RBLN-accelerated implementation of Kandinsky 2.2 pipeline for text-to-image generation.
24
+
25
+ This pipeline compiles Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling high-performance
26
+ inference for generating images with distinctive artistic style and enhanced visual quality.
27
+ """
28
+
22
29
  original_class = KandinskyV22Pipeline
23
30
  _rbln_config_class = RBLNKandinskyV22PipelineConfig
24
31
  _submodules = ["unet", "movq"]
@@ -38,6 +38,13 @@ from .pipeline_kandinsky2_2_prior import RBLNKandinskyV22PriorPipeline
38
38
 
39
39
 
40
40
  class RBLNKandinskyV22CombinedPipeline(RBLNDiffusionMixin, KandinskyV22CombinedPipeline):
41
+ """
42
+ RBLN-accelerated implementation of Kandinsky 2.2 combined pipeline for end-to-end text-to-image generation.
43
+
44
+ This pipeline compiles both prior and decoder Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling
45
+ high-performance inference for complete text-to-image generation with distinctive artistic style.
46
+ """
47
+
41
48
  original_class = KandinskyV22CombinedPipeline
42
49
  _rbln_config_class = RBLNKandinskyV22CombinedPipelineConfig
43
50
  _connected_classes = {"prior_pipe": RBLNKandinskyV22PriorPipeline, "decoder_pipe": RBLNKandinskyV22Pipeline}
@@ -46,15 +53,15 @@ class RBLNKandinskyV22CombinedPipeline(RBLNDiffusionMixin, KandinskyV22CombinedP
46
53
 
47
54
  def __init__(
48
55
  self,
49
- unet: "UNet2DConditionModel",
50
- scheduler: "DDPMScheduler",
51
- movq: "VQModel",
52
- prior_prior: "PriorTransformer",
53
- prior_image_encoder: "CLIPVisionModelWithProjection",
54
- prior_text_encoder: "CLIPTextModelWithProjection",
55
- prior_tokenizer: "CLIPTokenizer",
56
- prior_scheduler: "UnCLIPScheduler",
57
- prior_image_processor: "CLIPImageProcessor",
56
+ unet: UNet2DConditionModel,
57
+ scheduler: DDPMScheduler,
58
+ movq: VQModel,
59
+ prior_prior: PriorTransformer,
60
+ prior_image_encoder: CLIPVisionModelWithProjection,
61
+ prior_text_encoder: CLIPTextModelWithProjection,
62
+ prior_tokenizer: CLIPTokenizer,
63
+ prior_scheduler: UnCLIPScheduler,
64
+ prior_image_processor: CLIPImageProcessor,
58
65
  ):
59
66
  RBLNDiffusionMixin.__init__(self)
60
67
  super(KandinskyV22CombinedPipeline, self).__init__()
@@ -90,6 +97,13 @@ class RBLNKandinskyV22CombinedPipeline(RBLNDiffusionMixin, KandinskyV22CombinedP
90
97
 
91
98
 
92
99
  class RBLNKandinskyV22Img2ImgCombinedPipeline(RBLNDiffusionMixin, KandinskyV22Img2ImgCombinedPipeline):
100
+ """
101
+ RBLN-accelerated implementation of Kandinsky 2.2 combined pipeline for end-to-end image-to-image generation.
102
+
103
+ This pipeline compiles both prior and decoder Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling
104
+ high-performance inference for complete image-to-image transformation with distinctive artistic style.
105
+ """
106
+
93
107
  original_class = KandinskyV22Img2ImgCombinedPipeline
94
108
  _connected_classes = {"prior_pipe": RBLNKandinskyV22PriorPipeline, "decoder_pipe": RBLNKandinskyV22Img2ImgPipeline}
95
109
  _submodules = ["prior_image_encoder", "prior_text_encoder", "prior_prior", "unet", "movq"]
@@ -97,15 +111,15 @@ class RBLNKandinskyV22Img2ImgCombinedPipeline(RBLNDiffusionMixin, KandinskyV22Im
97
111
 
98
112
  def __init__(
99
113
  self,
100
- unet: "UNet2DConditionModel",
101
- scheduler: "DDPMScheduler",
102
- movq: "VQModel",
103
- prior_prior: "PriorTransformer",
104
- prior_image_encoder: "CLIPVisionModelWithProjection",
105
- prior_text_encoder: "CLIPTextModelWithProjection",
106
- prior_tokenizer: "CLIPTokenizer",
107
- prior_scheduler: "UnCLIPScheduler",
108
- prior_image_processor: "CLIPImageProcessor",
114
+ unet: UNet2DConditionModel,
115
+ scheduler: DDPMScheduler,
116
+ movq: VQModel,
117
+ prior_prior: PriorTransformer,
118
+ prior_image_encoder: CLIPVisionModelWithProjection,
119
+ prior_text_encoder: CLIPTextModelWithProjection,
120
+ prior_tokenizer: CLIPTokenizer,
121
+ prior_scheduler: UnCLIPScheduler,
122
+ prior_image_processor: CLIPImageProcessor,
109
123
  ):
110
124
  RBLNDiffusionMixin.__init__(self)
111
125
  super(KandinskyV22Img2ImgCombinedPipeline, self).__init__()
@@ -141,6 +155,13 @@ class RBLNKandinskyV22Img2ImgCombinedPipeline(RBLNDiffusionMixin, KandinskyV22Im
141
155
 
142
156
 
143
157
  class RBLNKandinskyV22InpaintCombinedPipeline(RBLNDiffusionMixin, KandinskyV22InpaintCombinedPipeline):
158
+ """
159
+ RBLN-accelerated implementation of Kandinsky 2.2 combined pipeline for end-to-end image inpainting.
160
+
161
+ This pipeline compiles both prior and decoder Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling
162
+ high-performance inference for complete image inpainting with distinctive artistic style and seamless integration.
163
+ """
164
+
144
165
  original_class = KandinskyV22InpaintCombinedPipeline
145
166
  _connected_classes = {"prior_pipe": RBLNKandinskyV22PriorPipeline, "decoder_pipe": RBLNKandinskyV22InpaintPipeline}
146
167
  _submodules = ["prior_image_encoder", "prior_text_encoder", "prior_prior", "unet", "movq"]
@@ -148,15 +169,15 @@ class RBLNKandinskyV22InpaintCombinedPipeline(RBLNDiffusionMixin, KandinskyV22In
148
169
 
149
170
  def __init__(
150
171
  self,
151
- unet: "UNet2DConditionModel",
152
- scheduler: "DDPMScheduler",
153
- movq: "VQModel",
154
- prior_prior: "PriorTransformer",
155
- prior_image_encoder: "CLIPVisionModelWithProjection",
156
- prior_text_encoder: "CLIPTextModelWithProjection",
157
- prior_tokenizer: "CLIPTokenizer",
158
- prior_scheduler: "UnCLIPScheduler",
159
- prior_image_processor: "CLIPImageProcessor",
172
+ unet: UNet2DConditionModel,
173
+ scheduler: DDPMScheduler,
174
+ movq: VQModel,
175
+ prior_prior: PriorTransformer,
176
+ prior_image_encoder: CLIPVisionModelWithProjection,
177
+ prior_text_encoder: CLIPTextModelWithProjection,
178
+ prior_tokenizer: CLIPTokenizer,
179
+ prior_scheduler: UnCLIPScheduler,
180
+ prior_image_processor: CLIPImageProcessor,
160
181
  ):
161
182
  RBLNDiffusionMixin.__init__(self)
162
183
  super(KandinskyV22InpaintCombinedPipeline, self).__init__()
@@ -19,6 +19,13 @@ from ...modeling_diffusers import RBLNDiffusionMixin
19
19
 
20
20
 
21
21
  class RBLNKandinskyV22Img2ImgPipeline(RBLNDiffusionMixin, KandinskyV22Img2ImgPipeline):
22
+ """
23
+ RBLN-accelerated implementation of Kandinsky 2.2 pipeline for image-to-image generation.
24
+
25
+ This pipeline compiles Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling high-performance
26
+ inference for transforming input images with distinctive artistic style and enhanced visual fidelity.
27
+ """
28
+
22
29
  original_class = KandinskyV22Img2ImgPipeline
23
30
  _rbln_config_class = RBLNKandinskyV22Img2ImgPipelineConfig
24
31
  _submodules = ["unet", "movq"]
@@ -19,6 +19,13 @@ from ...modeling_diffusers import RBLNDiffusionMixin
19
19
 
20
20
 
21
21
  class RBLNKandinskyV22InpaintPipeline(RBLNDiffusionMixin, KandinskyV22InpaintPipeline):
22
+ """
23
+ RBLN-accelerated implementation of Kandinsky 2.2 pipeline for image inpainting.
24
+
25
+ This pipeline compiles Kandinsky 2.2 models to run efficiently on RBLN NPUs, enabling high-performance
26
+ inference for filling masked regions with distinctive artistic style and seamless content integration.
27
+ """
28
+
22
29
  original_class = KandinskyV22InpaintPipeline
23
30
  _rbln_config_class = RBLNKandinskyV22InpaintPipelineConfig
24
31
  _submodules = ["unet", "movq"]
@@ -19,6 +19,13 @@ from ...modeling_diffusers import RBLNDiffusionMixin
19
19
 
20
20
 
21
21
  class RBLNKandinskyV22PriorPipeline(RBLNDiffusionMixin, KandinskyV22PriorPipeline):
22
+ """
23
+ RBLN-accelerated implementation of Kandinsky 2.2 prior pipeline for text and image embedding generation.
24
+
25
+ This pipeline compiles Kandinsky 2.2 prior models to run efficiently on RBLN NPUs, enabling high-performance
26
+ inference for generating image embeddings from text prompts and image inputs for downstream generation tasks.
27
+ """
28
+
22
29
  original_class = KandinskyV22PriorPipeline
23
30
  _rbln_config_class = RBLNKandinskyV22PriorPipelineConfig
24
31
  _submodules = ["text_encoder", "image_encoder", "prior"]
@@ -20,6 +20,13 @@ from ...modeling_diffusers import RBLNDiffusionMixin
20
20
 
21
21
 
22
22
  class RBLNStableDiffusionPipeline(RBLNDiffusionMixin, StableDiffusionPipeline):
23
+ """
24
+ RBLN-accelerated implementation of Stable Diffusion pipeline for text-to-image generation.
25
+
26
+ This pipeline compiles Stable Diffusion models to run efficiently on RBLN NPUs, enabling high-performance
27
+ inference for generating images from text prompts with optimized memory usage and throughput.
28
+ """
29
+
23
30
  original_class = StableDiffusionPipeline
24
31
  _rbln_config_class = RBLNStableDiffusionPipelineConfig
25
32
  _submodules = ["vae", "text_encoder", "unet"]