InvokeAI 5.10.0.dev4__tar.gz → 5.10.1__tar.gz

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/InvokeAI.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: InvokeAI
-Version: 5.10.0.dev4
+Version: 5.10.1
 Summary: An implementation of Stable Diffusion which provides various new features and options to aid the image generation process
 Author-email: The InvokeAI Project <lincoln.stein@gmail.com>
 License:                                  Apache License
@@ -212,7 +212,7 @@ License-File: LICENSE-SDXL.txt
 Requires-Dist: accelerate
 Requires-Dist: bitsandbytes; sys_platform != "darwin"
 Requires-Dist: compel==2.0.2
-Requires-Dist: diffusers[torch]
+Requires-Dist: diffusers[torch]==0.33.0
 Requires-Dist: gguf
 Requires-Dist: invisible-watermark==0.2.0
 Requires-Dist: mediapipe==0.10.14
@@ -221,6 +221,7 @@ Requires-Dist: onnx==1.16.1
 Requires-Dist: onnxruntime==1.19.2
 Requires-Dist: opencv-python==4.9.0.80
 Requires-Dist: safetensors
+Requires-Dist: sentencepiece
 Requires-Dist: spandrel
 Requires-Dist: torch~=2.6.0
 Requires-Dist: torchsde

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/InvokeAI.egg-info/SOURCES.txt

@@ -32,6 +32,11 @@ invokeai/app/invocations/baseinvocation.py
 invokeai/app/invocations/batch.py
 invokeai/app/invocations/blend_latents.py
 invokeai/app/invocations/canny.py
+invokeai/app/invocations/cogview4_denoise.py
+invokeai/app/invocations/cogview4_image_to_latents.py
+invokeai/app/invocations/cogview4_latents_to_image.py
+invokeai/app/invocations/cogview4_model_loader.py
+invokeai/app/invocations/cogview4_text_encoder.py
 invokeai/app/invocations/collections.py
 invokeai/app/invocations/color_map.py
 invokeai/app/invocations/compel.py
@@ -212,6 +217,7 @@ invokeai/app/services/shared/sqlite_migrator/migrations/migration_15.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_16.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_17.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_18.py
+invokeai/app/services/shared/sqlite_migrator/migrations/migration_19.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_2.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_3.py
 invokeai/app/services/shared/sqlite_migrator/migrations/migration_4.py
@@ -254,6 +260,7 @@ invokeai/app/services/workflow_records/__init__.py
 invokeai/app/services/workflow_records/workflow_records_base.py
 invokeai/app/services/workflow_records/workflow_records_common.py
 invokeai/app/services/workflow_records/workflow_records_sqlite.py
+invokeai/app/services/workflow_records/default_workflows/CogView4_TextToImage.json
 invokeai/app/services/workflow_records/default_workflows/ESRGAN Upscaling with Canny ControlNet.json
 invokeai/app/services/workflow_records/default_workflows/FLUX Image to Image.json
 invokeai/app/services/workflow_records/default_workflows/Face Detailer with IP-Adapter & Canny (See Note in Details).json
@@ -288,7 +295,7 @@ invokeai/app/util/ti_utils.py
 invokeai/app/util/torch_cuda_allocator.py
 invokeai/assets/fonts/inter/Inter-Regular.ttf
 invokeai/backend/__init__.py
-invokeai/backend/llava_onevision_model.py
+invokeai/backend/llava_onevision_pipeline.py
 invokeai/backend/model_patcher.py
 invokeai/backend/raw_model.py
 invokeai/backend/spandrel_image_to_image_model.py
@@ -309,7 +316,6 @@ invokeai/backend/flux/controlnet/xlabs_controlnet_flux.py
 invokeai/backend/flux/controlnet/zero_module.py
 invokeai/backend/flux/extensions/__init__.py
 invokeai/backend/flux/extensions/base_controlnet_extension.py
-invokeai/backend/flux/extensions/inpaint_extension.py
 invokeai/backend/flux/extensions/instantx_controlnet_extension.py
 invokeai/backend/flux/extensions/regional_prompting_extension.py
 invokeai/backend/flux/extensions/xlabs_controlnet_extension.py
@@ -442,6 +448,7 @@ invokeai/backend/model_manager/load/model_cache/torch_module_autocast/custom_mod
 invokeai/backend/model_manager/load/model_cache/torch_module_autocast/custom_modules/utils.py
 invokeai/backend/model_manager/load/model_loaders/__init__.py
 invokeai/backend/model_manager/load/model_loaders/clip_vision.py
+invokeai/backend/model_manager/load/model_loaders/cogview4.py
 invokeai/backend/model_manager/load/model_loaders/controlnet.py
 invokeai/backend/model_manager/load/model_loaders/flux.py
 invokeai/backend/model_manager/load/model_loaders/generic_diffusers.py
@@ -449,7 +456,7 @@ invokeai/backend/model_manager/load/model_loaders/ip_adapter.py
 invokeai/backend/model_manager/load/model_loaders/llava_onevision.py
 invokeai/backend/model_manager/load/model_loaders/lora.py
 invokeai/backend/model_manager/load/model_loaders/onnx.py
-invokeai/backend/model_manager/load/model_loaders/sig_lip_pipeline.py
+invokeai/backend/model_manager/load/model_loaders/sig_lip.py
 invokeai/backend/model_manager/load/model_loaders/spandrel_image_to_image.py
 invokeai/backend/model_manager/load/model_loaders/stable_diffusion.py
 invokeai/backend/model_manager/load/model_loaders/textual_inversion.py
@@ -501,9 +508,8 @@ invokeai/backend/quantization/gguf/utils.py
 invokeai/backend/quantization/scripts/load_flux_model_bnb_llm_int8.py
 invokeai/backend/quantization/scripts/load_flux_model_bnb_nf4.py
 invokeai/backend/quantization/scripts/quantize_t5_xxl_bnb_llm_int8.py
-invokeai/backend/sd3/__init__.py
-invokeai/backend/sd3/extensions/__init__.py
-invokeai/backend/sd3/extensions/inpaint_extension.py
+invokeai/backend/rectified_flow/__init__.py
+invokeai/backend/rectified_flow/rectified_flow_inpaint_extension.py
 invokeai/backend/sig_lip/sig_lip_pipeline.py
 invokeai/backend/stable_diffusion/__init__.py
 invokeai/backend/stable_diffusion/denoise_context.py
@@ -573,12 +579,12 @@ invokeai/frontend/install/__init__.py
 invokeai/frontend/install/import_images.py
 invokeai/frontend/web/__init__.py
 invokeai/frontend/web/dist/index.html
-invokeai/frontend/web/dist/assets/App-BxptzLdb.js
-invokeai/frontend/web/dist/assets/ThemeLocaleProvider-BE0YQ6XM.js
+invokeai/frontend/web/dist/assets/App-D8Ndfa3W.js
+invokeai/frontend/web/dist/assets/ThemeLocaleProvider-BOEWUG60.js
 invokeai/frontend/web/dist/assets/ThemeLocaleProvider-DMkJ1hTD.css
 invokeai/frontend/web/dist/assets/commercial-license-bg-C_dHp7on.png
 invokeai/frontend/web/dist/assets/denoising-strength-4mznP2tj.png
-invokeai/frontend/web/dist/assets/index-kn6zcPGp.js
+invokeai/frontend/web/dist/assets/index-em548e7X.js
 invokeai/frontend/web/dist/assets/inter-cyrillic-ext-wght-normal-B2xhLi22.woff2
 invokeai/frontend/web/dist/assets/inter-cyrillic-wght-normal-CMZtQduZ.woff2
 invokeai/frontend/web/dist/assets/inter-greek-ext-wght-normal-CGAr0uHJ.woff2

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/InvokeAI.egg-info/requires.txt

@@ -1,6 +1,6 @@
 accelerate
 compel==2.0.2
-diffusers[torch]
+diffusers[torch]==0.33.0
 gguf
 invisible-watermark==0.2.0
 mediapipe==0.10.14
@@ -9,6 +9,7 @@ onnx==1.16.1
 onnxruntime==1.19.2
 opencv-python==4.9.0.80
 safetensors
+sentencepiece
 spandrel
 torch~=2.6.0
 torchsde

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: InvokeAI
-Version: 5.10.0.dev4
+Version: 5.10.1
 Summary: An implementation of Stable Diffusion which provides various new features and options to aid the image generation process
 Author-email: The InvokeAI Project <lincoln.stein@gmail.com>
 License:                                  Apache License
@@ -212,7 +212,7 @@ License-File: LICENSE-SDXL.txt
 Requires-Dist: accelerate
 Requires-Dist: bitsandbytes; sys_platform != "darwin"
 Requires-Dist: compel==2.0.2
-Requires-Dist: diffusers[torch]
+Requires-Dist: diffusers[torch]==0.33.0
 Requires-Dist: gguf
 Requires-Dist: invisible-watermark==0.2.0
 Requires-Dist: mediapipe==0.10.14
@@ -221,6 +221,7 @@ Requires-Dist: onnx==1.16.1
 Requires-Dist: onnxruntime==1.19.2
 Requires-Dist: opencv-python==4.9.0.80
 Requires-Dist: safetensors
+Requires-Dist: sentencepiece
 Requires-Dist: spandrel
 Requires-Dist: torch~=2.6.0
 Requires-Dist: torchsde

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/invokeai/app/api/dependencies.py

@@ -39,6 +39,7 @@ from invokeai.app.services.workflow_records.workflow_records_sqlite import Sqlit
 from invokeai.app.services.workflow_thumbnails.workflow_thumbnails_disk import WorkflowThumbnailFileStorageDisk
 from invokeai.backend.stable_diffusion.diffusion.conditioning_data import (
     BasicConditioningInfo,
+    CogView4ConditioningInfo,
     ConditioningFieldData,
     FLUXConditioningInfo,
     SD3ConditioningInfo,
@@ -112,7 +113,6 @@ class ApiDependencies:
                 safe_globals=[torch.Tensor],
                 ephemeral=True,
             ),
-            max_cache_size=0,
         )
         conditioning = ObjectSerializerForwardCache(
             ObjectSerializerDisk[ConditioningFieldData](
@@ -123,6 +123,7 @@ class ApiDependencies:
                     SDXLConditioningInfo,
                     FLUXConditioningInfo,
                     SD3ConditioningInfo,
+                    CogView4ConditioningInfo,
                 ],
                 ephemeral=True,
             ),

{invokeai-5.10.0.dev4 → invokeai-5.10.1}/invokeai/app/api/routers/model_manager.py

@@ -85,6 +85,7 @@ example_model_config = {
     "config_path": "string",
     "key": "string",
     "hash": "string",
+    "file_size": 1,
     "description": "string",
     "source": "string",
     "converted_at": 0,

invokeai-5.10.1/invokeai/app/invocations/cogview4_denoise.py

@@ -0,0 +1,363 @@
+from typing import Callable, Optional
+
+import torch
+import torchvision.transforms as tv_transforms
+from diffusers.models.transformers.transformer_cogview4 import CogView4Transformer2DModel
+from torchvision.transforms.functional import resize as tv_resize
+from tqdm import tqdm
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.constants import LATENT_SCALE_FACTOR
+from invokeai.app.invocations.fields import (
+    CogView4ConditioningField,
+    DenoiseMaskField,
+    FieldDescriptions,
+    Input,
+    InputField,
+    LatentsField,
+    WithBoard,
+    WithMetadata,
+)
+from invokeai.app.invocations.model import TransformerField
+from invokeai.app.invocations.primitives import LatentsOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.flux.sampling_utils import clip_timestep_schedule_fractional
+from invokeai.backend.model_manager.config import BaseModelType
+from invokeai.backend.rectified_flow.rectified_flow_inpaint_extension import RectifiedFlowInpaintExtension
+from invokeai.backend.stable_diffusion.diffusers_pipeline import PipelineIntermediateState
+from invokeai.backend.stable_diffusion.diffusion.conditioning_data import CogView4ConditioningInfo
+from invokeai.backend.util.devices import TorchDevice
+
+
+@invocation(
+    "cogview4_denoise",
+    title="Denoise - CogView4",
+    tags=["image", "cogview4"],
+    category="image",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class CogView4DenoiseInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Run the denoising process with a CogView4 model."""
+
+    # If latents is provided, this means we are doing image-to-image.
+    latents: Optional[LatentsField] = InputField(
+        default=None, description=FieldDescriptions.latents, input=Input.Connection
+    )
+    # denoise_mask is used for image-to-image inpainting. Only the masked region is modified.
+    denoise_mask: Optional[DenoiseMaskField] = InputField(
+        default=None, description=FieldDescriptions.denoise_mask, input=Input.Connection
+    )
+    denoising_start: float = InputField(default=0.0, ge=0, le=1, description=FieldDescriptions.denoising_start)
+    denoising_end: float = InputField(default=1.0, ge=0, le=1, description=FieldDescriptions.denoising_end)
+    transformer: TransformerField = InputField(
+        description=FieldDescriptions.cogview4_model, input=Input.Connection, title="Transformer"
+    )
+    positive_conditioning: CogView4ConditioningField = InputField(
+        description=FieldDescriptions.positive_cond, input=Input.Connection
+    )
+    negative_conditioning: CogView4ConditioningField = InputField(
+        description=FieldDescriptions.negative_cond, input=Input.Connection
+    )
+    cfg_scale: float | list[float] = InputField(default=3.5, description=FieldDescriptions.cfg_scale, title="CFG Scale")
+    width: int = InputField(default=1024, multiple_of=32, description="Width of the generated image.")
+    height: int = InputField(default=1024, multiple_of=32, description="Height of the generated image.")
+    steps: int = InputField(default=25, gt=0, description=FieldDescriptions.steps)
+    seed: int = InputField(default=0, description="Randomness seed for reproducibility.")
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> LatentsOutput:
+        latents = self._run_diffusion(context)
+        latents = latents.detach().to("cpu")
+
+        name = context.tensors.save(tensor=latents)
+        return LatentsOutput.build(latents_name=name, latents=latents, seed=None)
+
+    def _prep_inpaint_mask(self, context: InvocationContext, latents: torch.Tensor) -> torch.Tensor | None:
+        """Prepare the inpaint mask.
+        - Loads the mask
+        - Resizes if necessary
+        - Casts to same device/dtype as latents
+
+        Args:
+            context (InvocationContext): The invocation context, for loading the inpaint mask.
+            latents (torch.Tensor): A latent image tensor. Used to determine the target shape, device, and dtype for the
+                inpaint mask.
+
+        Returns:
+            torch.Tensor | None: Inpaint mask. Values of 0.0 represent the regions to be fully denoised, and 1.0
+                represent the regions to be preserved.
+        """
+        if self.denoise_mask is None:
+            return None
+        mask = context.tensors.load(self.denoise_mask.mask_name)
+
+        # The input denoise_mask contains values in [0, 1], where 0.0 represents the regions to be fully denoised, and
+        # 1.0 represents the regions to be preserved.
+        # We invert the mask so that the regions to be preserved are 0.0 and the regions to be denoised are 1.0.
+        mask = 1.0 - mask
+
+        _, _, latent_height, latent_width = latents.shape
+        mask = tv_resize(
+            img=mask,
+            size=[latent_height, latent_width],
+            interpolation=tv_transforms.InterpolationMode.BILINEAR,
+            antialias=False,
+        )
+
+        mask = mask.to(device=latents.device, dtype=latents.dtype)
+        return mask
+
+    def _load_text_conditioning(
+        self,
+        context: InvocationContext,
+        conditioning_name: str,
+        dtype: torch.dtype,
+        device: torch.device,
+    ) -> torch.Tensor:
+        # Load the conditioning data.
+        cond_data = context.conditioning.load(conditioning_name)
+        assert len(cond_data.conditionings) == 1
+        cogview4_conditioning = cond_data.conditionings[0]
+        assert isinstance(cogview4_conditioning, CogView4ConditioningInfo)
+        cogview4_conditioning = cogview4_conditioning.to(dtype=dtype, device=device)
+
+        return cogview4_conditioning.glm_embeds
+
+    def _get_noise(
+        self,
+        batch_size: int,
+        num_channels_latents: int,
+        height: int,
+        width: int,
+        dtype: torch.dtype,
+        device: torch.device,
+        seed: int,
+    ) -> torch.Tensor:
+        # We always generate noise on the same device and dtype then cast to ensure consistency across devices/dtypes.
+        rand_device = "cpu"
+        rand_dtype = torch.float16
+
+        return torch.randn(
+            batch_size,
+            num_channels_latents,
+            int(height) // LATENT_SCALE_FACTOR,
+            int(width) // LATENT_SCALE_FACTOR,
+            device=rand_device,
+            dtype=rand_dtype,
+            generator=torch.Generator(device=rand_device).manual_seed(seed),
+        ).to(device=device, dtype=dtype)
+
+    def _prepare_cfg_scale(self, num_timesteps: int) -> list[float]:
+        """Prepare the CFG scale list.
+
+        Args:
+            num_timesteps (int): The number of timesteps in the scheduler. Could be different from num_steps depending
+            on the scheduler used (e.g. higher order schedulers).
+
+        Returns:
+            list[float]: _description_
+        """
+        if isinstance(self.cfg_scale, float):
+            cfg_scale = [self.cfg_scale] * num_timesteps
+        elif isinstance(self.cfg_scale, list):
+            assert len(self.cfg_scale) == num_timesteps
+            cfg_scale = self.cfg_scale
+        else:
+            raise ValueError(f"Invalid CFG scale type: {type(self.cfg_scale)}")
+
+        return cfg_scale
+
+    def _convert_timesteps_to_sigmas(self, image_seq_len: int, timesteps: torch.Tensor) -> list[float]:
+        # The logic to prepare the timestep / sigma schedule is based on:
+        # https://github.com/huggingface/diffusers/blob/b38450d5d2e5b87d5ff7088ee5798c85587b9635/src/diffusers/pipelines/cogview4/pipeline_cogview4.py#L575-L595
+        # The default FlowMatchEulerDiscreteScheduler configs are based on:
+        # https://huggingface.co/THUDM/CogView4-6B/blob/fb6f57289c73ac6d139e8d81bd5a4602d1877847/scheduler/scheduler_config.json
+        # This implementation differs slightly from the original for the sake of simplicity (differs in terminal value
+        # handling, not quantizing timesteps to integers, etc.).
+
+        def calculate_timestep_shift(
+            image_seq_len: int, base_seq_len: int = 256, base_shift: float = 0.25, max_shift: float = 0.75
+        ) -> float:
+            m = (image_seq_len / base_seq_len) ** 0.5
+            mu = m * max_shift + base_shift
+            return mu
+
+        def time_shift_linear(mu: float, sigma: float, t: torch.Tensor) -> torch.Tensor:
+            return mu / (mu + (1 / t - 1) ** sigma)
+
+        mu = calculate_timestep_shift(image_seq_len)
+        sigmas = time_shift_linear(mu, 1.0, timesteps)
+        return sigmas.tolist()
+
+    def _run_diffusion(
+        self,
+        context: InvocationContext,
+    ):
+        inference_dtype = torch.bfloat16
+        device = TorchDevice.choose_torch_device()
+
+        transformer_info = context.models.load(self.transformer.transformer)
+        assert isinstance(transformer_info.model, CogView4Transformer2DModel)
+
+        # Load/process the conditioning data.
+        # TODO(ryand): Make CFG optional.
+        do_classifier_free_guidance = True
+        pos_prompt_embeds = self._load_text_conditioning(
+            context=context,
+            conditioning_name=self.positive_conditioning.conditioning_name,
+            dtype=inference_dtype,
+            device=device,
+        )
+        neg_prompt_embeds = self._load_text_conditioning(
+            context=context,
+            conditioning_name=self.negative_conditioning.conditioning_name,
+            dtype=inference_dtype,
+            device=device,
+        )
+
+        # Prepare misc. conditioning variables.
+        # TODO(ryand): We could expose these as params (like with SDXL). But, we should experiment to see if they are
+        # useful first.
+        original_size = torch.tensor([(self.height, self.width)], dtype=pos_prompt_embeds.dtype, device=device)
+        target_size = torch.tensor([(self.height, self.width)], dtype=pos_prompt_embeds.dtype, device=device)
+        crops_coords_top_left = torch.tensor([(0, 0)], dtype=pos_prompt_embeds.dtype, device=device)
+
+        # Prepare the timestep / sigma schedule.
+        patch_size = transformer_info.model.config.patch_size  # type: ignore
+        assert isinstance(patch_size, int)
+        image_seq_len = ((self.height // LATENT_SCALE_FACTOR) * (self.width // LATENT_SCALE_FACTOR)) // (patch_size**2)
+        # We add an extra step to the end to account for the final timestep of 0.0.
+        timesteps: list[float] = torch.linspace(1, 0, self.steps + 1).tolist()
+        # Clip the timesteps schedule based on denoising_start and denoising_end.
+        timesteps = clip_timestep_schedule_fractional(timesteps, self.denoising_start, self.denoising_end)
+        sigmas = self._convert_timesteps_to_sigmas(image_seq_len, torch.tensor(timesteps))
+        total_steps = len(timesteps) - 1
+
+        # Prepare the CFG scale list.
+        cfg_scale = self._prepare_cfg_scale(total_steps)
+
+        # Load the input latents, if provided.
+        init_latents = context.tensors.load(self.latents.latents_name) if self.latents else None
+        if init_latents is not None:
+            init_latents = init_latents.to(device=device, dtype=inference_dtype)
+
+        # Generate initial latent noise.
+        num_channels_latents = transformer_info.model.config.in_channels  # type: ignore
+        assert isinstance(num_channels_latents, int)
+        noise = self._get_noise(
+            batch_size=1,
+            num_channels_latents=num_channels_latents,
+            height=self.height,
+            width=self.width,
+            dtype=inference_dtype,
+            device=device,
+            seed=self.seed,
+        )
+
+        # Prepare input latent image.
+        if init_latents is not None:
+            # Noise the init_latents by the appropriate amount for the first timestep.
+            s_0 = sigmas[0]
+            latents = s_0 * noise + (1.0 - s_0) * init_latents
+        else:
+            # init_latents are not provided, so we are not doing image-to-image (i.e. we are starting from pure noise).
+            if self.denoising_start > 1e-5:
+                raise ValueError("denoising_start should be 0 when initial latents are not provided.")
+            latents = noise
+
+        # If len(timesteps) == 1, then short-circuit. We are just noising the input latents, but not taking any
+        # denoising steps.
+        if len(timesteps) <= 1:
+            return latents
+
+        # Prepare inpaint extension.
+        inpaint_mask = self._prep_inpaint_mask(context, latents)
+        inpaint_extension: RectifiedFlowInpaintExtension | None = None
+        if inpaint_mask is not None:
+            assert init_latents is not None
+            inpaint_extension = RectifiedFlowInpaintExtension(
+                init_latents=init_latents,
+                inpaint_mask=inpaint_mask,
+                noise=noise,
+            )
+
+        step_callback = self._build_step_callback(context)
+
+        step_callback(
+            PipelineIntermediateState(
+                step=0,
+                order=1,
+                total_steps=total_steps,
+                timestep=int(timesteps[0]),
+                latents=latents,
+            ),
+        )
+
+        with transformer_info.model_on_device() as (_, transformer):
+            assert isinstance(transformer, CogView4Transformer2DModel)
+
+            # Denoising loop
+            for step_idx in tqdm(range(total_steps)):
+                t_curr = timesteps[step_idx]
+                sigma_curr = sigmas[step_idx]
+                sigma_prev = sigmas[step_idx + 1]
+
+                # Expand the timestep to match the latent model input.
+                # Multiply by 1000 to match the default FlowMatchEulerDiscreteScheduler num_train_timesteps.
+                timestep = torch.tensor([t_curr * 1000], device=device).expand(latents.shape[0])
+
+                # TODO(ryand): Support both sequential and batched CFG inference.
+                noise_pred_cond = transformer(
+                    hidden_states=latents,
+                    encoder_hidden_states=pos_prompt_embeds,
+                    timestep=timestep,
+                    original_size=original_size,
+                    target_size=target_size,
+                    crop_coords=crops_coords_top_left,
+                    return_dict=False,
+                )[0]
+
+                # Apply CFG.
+                if do_classifier_free_guidance:
+                    noise_pred_uncond = transformer(
+                        hidden_states=latents,
+                        encoder_hidden_states=neg_prompt_embeds,
+                        timestep=timestep,
+                        original_size=original_size,
+                        target_size=target_size,
+                        crop_coords=crops_coords_top_left,
+                        return_dict=False,
+                    )[0]
+
+                    noise_pred = noise_pred_uncond + cfg_scale[step_idx] * (noise_pred_cond - noise_pred_uncond)
+                else:
+                    noise_pred = noise_pred_cond
+
+                # Compute the previous noisy sample x_t -> x_t-1.
+                latents_dtype = latents.dtype
+                # TODO(ryand): Is casting to float32 necessary for precision/stability? I copied this from SD3.
+                latents = latents.to(dtype=torch.float32)
+                latents = latents + (sigma_prev - sigma_curr) * noise_pred
+                latents = latents.to(dtype=latents_dtype)
+
+                if inpaint_extension is not None:
+                    latents = inpaint_extension.merge_intermediate_latents_with_init_latents(latents, sigma_prev)
+
+                step_callback(
+                    PipelineIntermediateState(
+                        step=step_idx + 1,
+                        order=1,
+                        total_steps=total_steps,
+                        timestep=int(t_curr),
+                        latents=latents,
+                    ),
+                )
+
+        return latents
+
+    def _build_step_callback(self, context: InvocationContext) -> Callable[[PipelineIntermediateState], None]:
+        def step_callback(state: PipelineIntermediateState) -> None:
+            context.util.sd_step_callback(state, BaseModelType.CogView4)
+
+        return step_callback

invokeai-5.10.1/invokeai/app/invocations/cogview4_image_to_latents.py

@@ -0,0 +1,69 @@
+import einops
+import torch
+from diffusers.models.autoencoders.autoencoder_kl import AutoencoderKL
+
+from invokeai.app.invocations.baseinvocation import BaseInvocation, Classification, invocation
+from invokeai.app.invocations.fields import (
+    FieldDescriptions,
+    ImageField,
+    Input,
+    InputField,
+    WithBoard,
+    WithMetadata,
+)
+from invokeai.app.invocations.model import VAEField
+from invokeai.app.invocations.primitives import LatentsOutput
+from invokeai.app.services.shared.invocation_context import InvocationContext
+from invokeai.backend.model_manager.load.load_base import LoadedModel
+from invokeai.backend.stable_diffusion.diffusers_pipeline import image_resized_to_grid_as_tensor
+from invokeai.backend.util.devices import TorchDevice
+
+# TODO(ryand): This is effectively a copy of SD3ImageToLatentsInvocation and a subset of ImageToLatentsInvocation. We
+# should refactor to avoid this duplication.
+
+
+@invocation(
+    "cogview4_i2l",
+    title="Image to Latents - CogView4",
+    tags=["image", "latents", "vae", "i2l", "cogview4"],
+    category="image",
+    version="1.0.0",
+    classification=Classification.Prototype,
+)
+class CogView4ImageToLatentsInvocation(BaseInvocation, WithMetadata, WithBoard):
+    """Generates latents from an image."""
+
+    image: ImageField = InputField(description="The image to encode.")
+    vae: VAEField = InputField(description=FieldDescriptions.vae, input=Input.Connection)
+
+    @staticmethod
+    def vae_encode(vae_info: LoadedModel, image_tensor: torch.Tensor) -> torch.Tensor:
+        with vae_info as vae:
+            assert isinstance(vae, AutoencoderKL)
+
+            vae.disable_tiling()
+
+            image_tensor = image_tensor.to(device=TorchDevice.choose_torch_device(), dtype=vae.dtype)
+            with torch.inference_mode():
+                image_tensor_dist = vae.encode(image_tensor).latent_dist
+                # TODO: Use seed to make sampling reproducible.
+                latents: torch.Tensor = image_tensor_dist.sample().to(dtype=vae.dtype)
+
+            latents = vae.config.scaling_factor * latents
+
+        return latents
+
+    @torch.no_grad()
+    def invoke(self, context: InvocationContext) -> LatentsOutput:
+        image = context.images.get_pil(self.image.image_name)
+
+        image_tensor = image_resized_to_grid_as_tensor(image.convert("RGB"))
+        if image_tensor.dim() == 3:
+            image_tensor = einops.rearrange(image_tensor, "c h w -> 1 c h w")
+
+        vae_info = context.models.load(self.vae.vae)
+        latents = self.vae_encode(vae_info=vae_info, image_tensor=image_tensor)
+
+        latents = latents.to("cpu")
+        name = context.tensors.save(tensor=latents)
+        return LatentsOutput.build(latents_name=name, latents=latents, seed=None)