optimum-rbln 0.7.3a5__tar.gz → 0.7.3a6__tar.gz

optimum_rbln-0.7.3a6/.github/version.yaml

@@ -0,0 +1 @@
+rebel_compiler_version: 0.7.3.dev166+g05e9b26d

optimum_rbln-0.7.3a6/.github/workflows/deploy-on-tag.yaml

@@ -0,0 +1,34 @@
+name: Deploy package on tag released
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+jobs:
+  deploy-on-tag:
+    uses: ./.github/workflows/deploy.yaml
+    with:
+      version: ${{ github.ref_name }}
+    secrets: inherit
+
+  trigger-vllm-model-compilation:
+    name: trigger vllm model compile and generation ci
+    needs: [deploy-on-tag]
+    runs-on: rebel-k8s-runner
+    steps:
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GIT_PAT }}
+          script: |
+            const result = await github.rest.actions.createWorkflowDispatch({
+              owner: 'rebellions-sw',
+              repo: 'rebel_compiler',
+              workflow_id: 'rebel_dispatch_model_generation_for_vllm.yaml',
+              ref: 'dev',
+              inputs: {
+                optimum_rbln_version: ${{ github.ref_name }},
+              }
+            })
+            console.log(result)

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optimum-rbln
-Version: 0.7.3a5
+Version: 0.7.3a6
 Summary: Optimum RBLN is the interface between the Hugging Face Transformers and Diffusers libraries and RBLN accelerators. It provides a set of tools enabling easy model loading and inference on single and multiple rbln device settings for different downstream tasks.
 Project-URL: Homepage, https://rebellions.ai
 Project-URL: Documentation, https://docs.rbln.ai

optimum_rbln-0.7.3a6/examples/kandinsky2_2/run_kandinsky2_2.py

@@ -0,0 +1,57 @@
+import os
+
+import fire
+import torch
+
+from optimum.rbln import RBLNKandinskyV22Pipeline, RBLNKandinskyV22PriorPipeline
+
+
+def main(
+    prior_model_id: str = "kandinsky-community/kandinsky-2-2-prior",
+    inpaint_model_id: str = "kandinsky-community/kandinsky-2-2-decoder",
+    from_diffusers: bool = False,
+    prompt: str = "red cat, 4k photo",
+):
+    if from_diffusers:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=prior_model_id,
+            export=True,
+        )
+        prior_pipe.save_pretrained(os.path.basename(prior_model_id))
+
+        pipe = RBLNKandinskyV22Pipeline.from_pretrained(
+            model_id=inpaint_model_id,
+            export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
+        )
+        pipe.save_pretrained(os.path.basename(inpaint_model_id))
+    else:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=os.path.basename(prior_model_id),
+            export=False,
+        )
+        pipe = RBLNKandinskyV22Pipeline.from_pretrained(
+            model_id=os.path.basename(inpaint_model_id),
+            export=False,
+        )
+
+    generator = torch.manual_seed(42)
+    out = prior_pipe(prompt, generator=generator)
+    image_emb = out.image_embeds
+    zero_image_emb = out.negative_image_embeds
+
+    out = pipe(
+        image_embeds=image_emb,
+        negative_image_embeds=zero_image_emb,
+        height=768,
+        width=768,
+        num_inference_steps=50,
+        generator=generator,
+    )
+    image = out.images[0]
+    image.save(f"{prompt}.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

optimum_rbln-0.7.3a6/examples/kandinsky2_2/run_kandinsky2_2_combined.py

@@ -0,0 +1,31 @@
+import os
+
+import fire
+import torch
+
+from optimum.rbln import RBLNKandinskyV22CombinedPipeline
+
+
+def main(
+    model_id: str = "kandinsky-community/kandinsky-2-2-decoder",
+    from_diffusers: bool = False,
+    prompt: str = "red cat, 4k photo",
+):
+    if from_diffusers:
+        pipe = RBLNKandinskyV22CombinedPipeline.from_pretrained(
+            model_id=model_id,
+            export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
+        )
+        pipe.save_pretrained(os.path.basename(model_id))
+    else:
+        pipe = RBLNKandinskyV22CombinedPipeline.from_pretrained(model_id=os.path.basename(model_id), export=False)
+
+    generator = torch.manual_seed(42)
+    image = pipe(prompt, height=768, width=768, num_inference_steps=50, generator=generator).images[0]
+    image.save(f"{prompt}.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

optimum_rbln-0.7.3a6/examples/kandinsky2_2/run_kandinsky2_2_img2img.py

@@ -0,0 +1,61 @@
+import os
+
+import fire
+import torch
+from diffusers.utils import load_image
+
+from optimum.rbln import RBLNKandinskyV22Img2ImgPipeline, RBLNKandinskyV22PriorPipeline
+
+
+def main(
+    prior_model_id: str = "kandinsky-community/kandinsky-2-2-prior",
+    inpaint_model_id: str = "kandinsky-community/kandinsky-2-2-decoder",
+    from_diffusers: bool = False,
+    prompt: str = "A red cartoon frog, 4k",
+):
+    img_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/frog.png"
+    init_image = load_image(img_url)
+
+    if from_diffusers:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=prior_model_id,
+            export=True,
+        )
+        prior_pipe.save_pretrained(os.path.basename(prior_model_id))
+
+        pipe = RBLNKandinskyV22Img2ImgPipeline.from_pretrained(
+            model_id=inpaint_model_id,
+            export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
+        )
+        pipe.save_pretrained(os.path.basename(inpaint_model_id))
+    else:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=os.path.basename(prior_model_id),
+            export=False,
+        )
+        pipe = RBLNKandinskyV22Img2ImgPipeline.from_pretrained(
+            model_id=os.path.basename(inpaint_model_id),
+            export=False,
+        )
+
+    generator = torch.manual_seed(42)
+    image_emb, zero_image_emb = prior_pipe(prompt, generator=generator, return_dict=False)
+
+    out = pipe(
+        image=init_image,
+        image_embeds=image_emb,
+        negative_image_embeds=zero_image_emb,
+        height=768,
+        width=768,
+        num_inference_steps=100,
+        strength=0.2,
+        generator=generator,
+    )
+    image = out.images[0]
+    image.save(f"{prompt}.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

optimum_rbln-0.7.3a6/examples/kandinsky2_2/run_kandinsky2_2_img2img_combined.py

@@ -0,0 +1,46 @@
+import os
+
+import fire
+import torch
+from diffusers.utils import load_image
+
+from optimum.rbln import RBLNKandinskyV22Img2ImgCombinedPipeline
+
+
+def main(
+    model_id: str = "kandinsky-community/kandinsky-2-2-decoder",
+    from_diffusers: bool = False,
+    prompt: str = "A red cartoon frog, 4k",
+):
+    img_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/frog.png"
+    init_image = load_image(img_url)
+
+    if from_diffusers:
+        pipe = RBLNKandinskyV22Img2ImgCombinedPipeline.from_pretrained(
+            model_id=model_id,
+            export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
+        )
+        pipe.save_pretrained(os.path.basename(model_id))
+    else:
+        pipe = RBLNKandinskyV22Img2ImgCombinedPipeline.from_pretrained(
+            model_id=os.path.basename(model_id), export=False
+        )
+
+    generator = torch.manual_seed(42)
+
+    image = pipe(
+        prompt=prompt,
+        image=init_image,
+        height=768,
+        width=768,
+        num_inference_steps=100,
+        strength=0.2,
+        generator=generator,
+    ).images[0]
+    image.save(f"{prompt}.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/examples/kandinsky2_2/run_kandinsky2_2_inpaint.py

@@ -27,6 +27,8 @@ def main(
         pipe = RBLNKandinskyV22InpaintPipeline.from_pretrained(
             model_id=inpaint_model_id,
             export=True,
+            rbln_img_width=768,
+            rbln_img_height=768,
         )
         pipe.save_pretrained(os.path.basename(inpaint_model_id))
     else:
@@ -42,8 +44,10 @@ def main(
     generator = torch.manual_seed(42)
     image_emb, zero_image_emb = prior_pipe(prompt, generator=generator, return_dict=False)
 
-    mask = np.zeros((512, 512), dtype=np.float32)
-    mask[:170, 170:-170] = 1
+    # Mask out the desired area to inpaint
+    # In this example, we will draw a hat on the cat's head
+    mask = np.zeros((768, 768), dtype=np.float32)
+    mask[:250, 250:-250] = 1
 
     out = pipe(
         image=init_image,

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/examples/kandinsky2_2/run_kandinsky2_2_inpaint_combined.py

@@ -1,6 +1,7 @@
 import os
 
 import fire
+import numpy as np
 import torch
 from diffusers.utils import load_image
 
@@ -10,19 +11,17 @@ from optimum.rbln import RBLNKandinskyV22InpaintCombinedPipeline
 def main(
     model_id: str = "kandinsky-community/kandinsky-2-2-decoder-inpaint",
     from_diffusers: bool = False,
-    prompt: str = "concept art digital painting of an elven castle, inspired by lord of the rings, highly detailed, 8k",
+    prompt: str = "a hat",
 ):
-    img_url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint.png"
-    mask_url = (
-        "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/inpaint_mask.png"
-    )
-    source = load_image(img_url)
-    mask = load_image(mask_url)
+    img_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png"
+    init_image = load_image(img_url)
 
     if from_diffusers:
         pipe = RBLNKandinskyV22InpaintCombinedPipeline.from_pretrained(
             model_id=model_id,
             export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
         )
         pipe.save_pretrained(os.path.basename(model_id))
     else:
@@ -30,7 +29,13 @@ def main(
             model_id=os.path.basename(model_id), export=False
         )
 
-    image = pipe(prompt, image=source, mask_image=mask, generator=torch.manual_seed(42)).images[0]
+    generator = torch.manual_seed(42)
+    # Mask out the desired area to inpaint
+    # In this example, we will draw a hat on the cat's head
+    mask = np.zeros((768, 768), dtype=np.float32)
+    mask[:250, 250:-250] = 1
+
+    image = pipe(prompt, image=init_image, mask_image=mask, generator=generator).images[0]
     image.save(f"{prompt}.png")
 
 

optimum_rbln-0.7.3a6/examples/kandinsky2_2/run_kandinsky2_2_prior_interpolate.py

@@ -0,0 +1,64 @@
+import os
+
+import fire
+import torch
+from diffusers.utils import load_image
+
+from optimum.rbln import RBLNKandinskyV22Pipeline, RBLNKandinskyV22PriorPipeline
+
+
+def main(
+    prior_model_id: str = "kandinsky-community/kandinsky-2-2-prior",
+    inpaint_model_id: str = "kandinsky-community/kandinsky-2-2-decoder",
+    from_diffusers: bool = False,
+):
+    img1 = load_image(
+        "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/cat.png"
+    )
+    img2 = load_image(
+        "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky/starry_night.jpeg"
+    )
+
+    if from_diffusers:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=prior_model_id,
+            export=True,
+        )
+        prior_pipe.save_pretrained(os.path.basename(prior_model_id))
+
+        pipe = RBLNKandinskyV22Pipeline.from_pretrained(
+            model_id=inpaint_model_id,
+            export=True,
+            rbln_img_height=768,
+            rbln_img_width=768,
+        )
+        pipe.save_pretrained(os.path.basename(inpaint_model_id))
+    else:
+        prior_pipe = RBLNKandinskyV22PriorPipeline.from_pretrained(
+            model_id=os.path.basename(prior_model_id),
+            export=False,
+        )
+        pipe = RBLNKandinskyV22Pipeline.from_pretrained(
+            model_id=os.path.basename(inpaint_model_id),
+            export=False,
+        )
+
+    images_texts = ["a cat", img1, img2]
+    weights = [0.3, 0.3, 0.4]
+    generator = torch.manual_seed(42)
+    out = prior_pipe.interpolate(images_texts, weights, generator=generator)
+    image_emb = out.image_embeds
+    zero_image_emb = out.negative_image_embeds
+
+    out = pipe(
+        image_embeds=image_emb,
+        negative_image_embeds=zero_image_emb,
+        num_inference_steps=50,
+        generator=generator,
+    )
+    image = out.images[0]
+    image.save("starry_cat.png")
+
+
+if __name__ == "__main__":
+    fire.Fire(main)

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/__init__.py

@@ -78,9 +78,13 @@ _import_structure = {
         "RBLNAutoencoderKL",
         "RBLNControlNetModel",
         "RBLNPriorTransformer",
+        "RBLNKandinskyV22CombinedPipeline",
+        "RBLNKandinskyV22Img2ImgCombinedPipeline",
         "RBLNKandinskyV22InpaintCombinedPipeline",
         "RBLNKandinskyV22InpaintPipeline",
+        "RBLNKandinskyV22Img2ImgPipeline",
         "RBLNKandinskyV22PriorPipeline",
+        "RBLNKandinskyV22Pipeline",
         "RBLNStableDiffusionPipeline",
         "RBLNStableDiffusionXLPipeline",
         "RBLNUNet2DConditionModel",
@@ -107,8 +111,12 @@ if TYPE_CHECKING:
         RBLNAutoencoderKL,
         RBLNControlNetModel,
         RBLNDiffusionMixin,
+        RBLNKandinskyV22CombinedPipeline,
+        RBLNKandinskyV22Img2ImgCombinedPipeline,
+        RBLNKandinskyV22Img2ImgPipeline,
         RBLNKandinskyV22InpaintCombinedPipeline,
         RBLNKandinskyV22InpaintPipeline,
+        RBLNKandinskyV22Pipeline,
         RBLNKandinskyV22PriorPipeline,
         RBLNMultiControlNetModel,
         RBLNPriorTransformer,

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/__version__.py

@@ -17,5 +17,5 @@ __version__: str
 __version_tuple__: VERSION_TUPLE
 version_tuple: VERSION_TUPLE
 
-__version__ = version = '0.7.3a5'
-__version_tuple__ = version_tuple = (0, 7, 3, 'a5')
+__version__ = version = '0.7.3a6'
+__version_tuple__ = version_tuple = (0, 7, 3, 'a6')

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/diffusers/__init__.py

@@ -24,9 +24,13 @@ ALL_IMPORTABLE_CLASSES.update(LOADABLE_CLASSES["optimum.rbln"])
 
 _import_structure = {
     "pipelines": [
+        "RBLNKandinskyV22CombinedPipeline",
+        "RBLNKandinskyV22Img2ImgCombinedPipeline",
         "RBLNKandinskyV22InpaintCombinedPipeline",
         "RBLNKandinskyV22InpaintPipeline",
+        "RBLNKandinskyV22Img2ImgPipeline",
         "RBLNKandinskyV22PriorPipeline",
+        "RBLNKandinskyV22Pipeline",
         "RBLNStableDiffusionPipeline",
         "RBLNStableDiffusionXLPipeline",
         "RBLNStableDiffusionImg2ImgPipeline",
@@ -66,8 +70,12 @@ if TYPE_CHECKING:
         RBLNVQModel,
     )
     from .pipelines import (
+        RBLNKandinskyV22CombinedPipeline,
+        RBLNKandinskyV22Img2ImgCombinedPipeline,
+        RBLNKandinskyV22Img2ImgPipeline,
         RBLNKandinskyV22InpaintCombinedPipeline,
         RBLNKandinskyV22InpaintPipeline,
+        RBLNKandinskyV22Pipeline,
         RBLNKandinskyV22PriorPipeline,
         RBLNMultiControlNetModel,
         RBLNStableDiffusion3Img2ImgPipeline,

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/diffusers/models/autoencoders/vq_model.py

@@ -90,9 +90,17 @@ class RBLNVQModel(RBLNModel):
         model_config: "PretrainedConfig",
         rbln_kwargs: Dict[str, Any] = {},
     ) -> RBLNConfig:
-        batch_size = rbln_kwargs.get("batch_size") or 1
-        height = rbln_kwargs.get("img_height") or 512
-        width = rbln_kwargs.get("img_width") or 512
+        batch_size = rbln_kwargs.get("batch_size")
+        if batch_size is None:
+            batch_size = 1
+
+        height = rbln_kwargs.get("img_height")
+        if height is None:
+            height = 512
+
+        width = rbln_kwargs.get("img_width")
+        if width is None:
+            width = 512
 
         if hasattr(model_config, "block_out_channels"):
             scale_factor = 2 ** (len(model_config.block_out_channels) - 1)

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/diffusers/models/unets/unet_2d_condition.py

@@ -176,15 +176,22 @@ class RBLNUNet2DConditionModel(RBLNModel):
             raise ValueError("Both image height and image width must be given or not given")
         elif image_size[0] is None and image_size[1] is None:
             if rbln_config["img2img_pipeline"]:
-                # In case of img2img, sample size of unet is determined by vae encoder.
-                vae_sample_size = pipe.vae.config.sample_size
-                if isinstance(vae_sample_size, int):
-                    sample_size = vae_sample_size // scale_factor
-                else:
-                    sample_size = (
-                        vae_sample_size[0] // scale_factor,
-                        vae_sample_size[1] // scale_factor,
+                if hasattr(pipe, "vae"):
+                    # In case of img2img, sample size of unet is determined by vae encoder.
+                    vae_sample_size = pipe.vae.config.sample_size
+                    if isinstance(vae_sample_size, int):
+                        sample_size = vae_sample_size // scale_factor
+                    else:
+                        sample_size = (
+                            vae_sample_size[0] // scale_factor,
+                            vae_sample_size[1] // scale_factor,
+                        )
+                elif hasattr(pipe, "movq"):
+                    logger.warning(
+                        "RBLN config 'img_height' and 'img_width' should have been provided for this pipeline. "
+                        "Both variable will be set 512 by default."
                     )
+                    sample_size = (512 // scale_factor, 512 // scale_factor)
             else:
                 sample_size = pipe.unet.config.sample_size
         else:

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/diffusers/pipelines/__init__.py

@@ -26,9 +26,13 @@ _import_structure = {
         "RBLNStableDiffusionXLControlNetPipeline",
     ],
     "kandinsky2_2": [
+        "RBLNKandinskyV22CombinedPipeline",
+        "RBLNKandinskyV22Img2ImgCombinedPipeline",
         "RBLNKandinskyV22InpaintCombinedPipeline",
         "RBLNKandinskyV22InpaintPipeline",
+        "RBLNKandinskyV22Img2ImgPipeline",
         "RBLNKandinskyV22PriorPipeline",
+        "RBLNKandinskyV22Pipeline",
     ],
     "stable_diffusion": [
         "RBLNStableDiffusionImg2ImgPipeline",
@@ -55,8 +59,12 @@ if TYPE_CHECKING:
         RBLNStableDiffusionXLControlNetPipeline,
     )
     from .kandinsky2_2 import (
+        RBLNKandinskyV22CombinedPipeline,
+        RBLNKandinskyV22Img2ImgCombinedPipeline,
+        RBLNKandinskyV22Img2ImgPipeline,
         RBLNKandinskyV22InpaintCombinedPipeline,
         RBLNKandinskyV22InpaintPipeline,
+        RBLNKandinskyV22Pipeline,
         RBLNKandinskyV22PriorPipeline,
     )
     from .stable_diffusion import (

{optimum_rbln-0.7.3a5 → optimum_rbln-0.7.3a6}/src/optimum/rbln/diffusers/pipelines/kandinsky2_2/__init__.py

@@ -12,6 +12,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .pipeline_kandinsky2_2_combined import RBLNKandinskyV22InpaintCombinedPipeline
+from .pipeline_kandinsky2_2 import RBLNKandinskyV22Pipeline
+from .pipeline_kandinsky2_2_combined import (
+    RBLNKandinskyV22CombinedPipeline,
+    RBLNKandinskyV22Img2ImgCombinedPipeline,
+    RBLNKandinskyV22InpaintCombinedPipeline,
+)
+from .pipeline_kandinsky2_2_img2img import RBLNKandinskyV22Img2ImgPipeline
 from .pipeline_kandinsky2_2_inpaint import RBLNKandinskyV22InpaintPipeline
 from .pipeline_kandinsky2_2_prior import RBLNKandinskyV22PriorPipeline

optimum_rbln-0.7.3a6/src/optimum/rbln/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py

@@ -0,0 +1,25 @@
+# Copyright 2024 Rebellions Inc.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from diffusers import KandinskyV22Pipeline
+
+from ...modeling_diffusers import RBLNDiffusionMixin
+
+
+class RBLNKandinskyV22Pipeline(RBLNDiffusionMixin, KandinskyV22Pipeline):
+    original_class = KandinskyV22Pipeline
+    _submodules = ["unet", "movq"]
+
+    def get_compiled_image_size(self):
+        return self.movq.image_size