diffsynth 1.0.0__py3-none-any.whl

diffsynth/pipelines/svd_video.py

@@ -0,0 +1,297 @@
+from ..models import ModelManager, SVDImageEncoder, SVDUNet, SVDVAEEncoder, SVDVAEDecoder
+from ..schedulers import ContinuousODEScheduler
+from .base import BasePipeline
+import torch
+from tqdm import tqdm
+from PIL import Image
+import numpy as np
+from einops import rearrange, repeat
+
+
+
+class SVDVideoPipeline(BasePipeline):
+
+    def __init__(self, device="cuda", torch_dtype=torch.float16):
+        super().__init__(device=device, torch_dtype=torch_dtype)
+        self.scheduler = ContinuousODEScheduler()
+        # models
+        self.image_encoder: SVDImageEncoder = None
+        self.unet: SVDUNet = None
+        self.vae_encoder: SVDVAEEncoder = None
+        self.vae_decoder: SVDVAEDecoder = None
+    
+
+    def fetch_models(self, model_manager: ModelManager):
+        self.image_encoder = model_manager.fetch_model("svd_image_encoder")
+        self.unet = model_manager.fetch_model("svd_unet")
+        self.vae_encoder = model_manager.fetch_model("svd_vae_encoder")
+        self.vae_decoder = model_manager.fetch_model("svd_vae_decoder")
+
+
+    @staticmethod
+    def from_model_manager(model_manager: ModelManager, **kwargs):
+        pipe = SVDVideoPipeline(
+            device=model_manager.device,
+            torch_dtype=model_manager.torch_dtype
+        )
+        pipe.fetch_models(model_manager)
+        return pipe
+    
+
+    def encode_image_with_clip(self, image):
+        image = self.preprocess_image(image).to(device=self.device, dtype=self.torch_dtype)
+        image = SVDCLIPImageProcessor().resize_with_antialiasing(image, (224, 224))
+        image = (image + 1.0) / 2.0
+        mean = torch.tensor([0.48145466, 0.4578275, 0.40821073]).reshape(1, 3, 1, 1).to(device=self.device, dtype=self.torch_dtype)
+        std = torch.tensor([0.26862954, 0.26130258, 0.27577711]).reshape(1, 3, 1, 1).to(device=self.device, dtype=self.torch_dtype)
+        image = (image - mean) / std
+        image_emb = self.image_encoder(image)
+        return image_emb
+    
+
+    def encode_image_with_vae(self, image, noise_aug_strength):
+        image = self.preprocess_image(image).to(device=self.device, dtype=self.torch_dtype)
+        noise = torch.randn(image.shape, device="cpu", dtype=self.torch_dtype).to(self.device)
+        image = image + noise_aug_strength * noise
+        image_emb = self.vae_encoder(image) / self.vae_encoder.scaling_factor
+        return image_emb
+    
+
+    def encode_video_with_vae(self, video):
+        video = torch.concat([self.preprocess_image(frame) for frame in video], dim=0)
+        video = rearrange(video, "T C H W -> 1 C T H W")
+        video = video.to(device=self.device, dtype=self.torch_dtype)
+        latents = self.vae_encoder.encode_video(video)
+        latents = rearrange(latents[0], "C T H W -> T C H W")
+        return latents
+    
+
+    def tensor2video(self, frames):
+        frames = rearrange(frames, "C T H W -> T H W C")
+        frames = ((frames.float() + 1) * 127.5).clip(0, 255).cpu().numpy().astype(np.uint8)
+        frames = [Image.fromarray(frame) for frame in frames]
+        return frames
+    
+
+    def calculate_noise_pred(
+        self,
+        latents,
+        timestep,
+        add_time_id,
+        cfg_scales,
+        image_emb_vae_posi, image_emb_clip_posi,
+        image_emb_vae_nega, image_emb_clip_nega
+    ):
+        # Positive side
+        noise_pred_posi = self.unet(
+            torch.cat([latents, image_emb_vae_posi], dim=1),
+            timestep, image_emb_clip_posi, add_time_id
+        )
+        # Negative side
+        noise_pred_nega = self.unet(
+            torch.cat([latents, image_emb_vae_nega], dim=1),
+            timestep, image_emb_clip_nega, add_time_id
+        )
+
+        # Classifier-free guidance
+        noise_pred = noise_pred_nega + cfg_scales * (noise_pred_posi - noise_pred_nega)
+
+        return noise_pred
+    
+
+    def post_process_latents(self, latents, post_normalize=True, contrast_enhance_scale=1.0):
+        if post_normalize:
+            mean, std = latents.mean(), latents.std()
+            latents = (latents - latents.mean(dim=[1, 2, 3], keepdim=True)) / latents.std(dim=[1, 2, 3], keepdim=True) * std + mean
+        latents = latents * contrast_enhance_scale
+        return latents
+
+
+    @torch.no_grad()
+    def __call__(
+        self,
+        input_image=None,
+        input_video=None,
+        mask_frames=[],
+        mask_frame_ids=[],
+        min_cfg_scale=1.0,
+        max_cfg_scale=3.0,
+        denoising_strength=1.0,
+        num_frames=25,
+        height=576,
+        width=1024,
+        fps=7,
+        motion_bucket_id=127,
+        noise_aug_strength=0.02,
+        num_inference_steps=20,
+        post_normalize=True,
+        contrast_enhance_scale=1.2,
+        progress_bar_cmd=tqdm,
+        progress_bar_st=None,
+    ):
+        # Prepare scheduler
+        self.scheduler.set_timesteps(num_inference_steps, denoising_strength=denoising_strength)
+
+        # Prepare latent tensors
+        noise = torch.randn((num_frames, 4, height//8, width//8), device="cpu", dtype=self.torch_dtype).to(self.device)
+        if denoising_strength == 1.0:
+            latents = noise.clone()
+        else:
+            latents = self.encode_video_with_vae(input_video)
+            latents = self.scheduler.add_noise(latents, noise, self.scheduler.timesteps[0])
+
+        # Prepare mask frames
+        if len(mask_frames) > 0:
+            mask_latents = self.encode_video_with_vae(mask_frames)
+
+        # Encode image
+        image_emb_clip_posi = self.encode_image_with_clip(input_image)
+        image_emb_clip_nega = torch.zeros_like(image_emb_clip_posi)
+        image_emb_vae_posi = repeat(self.encode_image_with_vae(input_image, noise_aug_strength), "B C H W -> (B T) C H W", T=num_frames)
+        image_emb_vae_nega = torch.zeros_like(image_emb_vae_posi)
+
+        # Prepare classifier-free guidance
+        cfg_scales = torch.linspace(min_cfg_scale, max_cfg_scale, num_frames)
+        cfg_scales = cfg_scales.reshape(num_frames, 1, 1, 1).to(device=self.device, dtype=self.torch_dtype)
+        
+        # Prepare positional id
+        add_time_id = torch.tensor([[fps-1, motion_bucket_id, noise_aug_strength]], device=self.device)
+
+        # Denoise
+        for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps)):
+
+            # Mask frames
+            for frame_id, mask_frame_id in enumerate(mask_frame_ids):
+                latents[mask_frame_id] = self.scheduler.add_noise(mask_latents[frame_id], noise[mask_frame_id], timestep)
+
+            # Fetch model output
+            noise_pred = self.calculate_noise_pred(
+                latents, timestep, add_time_id, cfg_scales,
+                image_emb_vae_posi, image_emb_clip_posi, image_emb_vae_nega, image_emb_clip_nega
+            )
+
+            # Forward Euler
+            latents = self.scheduler.step(noise_pred, timestep, latents)
+            
+            # Update progress bar
+            if progress_bar_st is not None:
+                progress_bar_st.progress(progress_id / len(self.scheduler.timesteps))
+
+        # Decode image
+        latents = self.post_process_latents(latents, post_normalize=post_normalize, contrast_enhance_scale=contrast_enhance_scale)
+        video = self.vae_decoder.decode_video(latents, progress_bar=progress_bar_cmd)
+        video = self.tensor2video(video)
+
+        return video
+
+
+
+class SVDCLIPImageProcessor:
+    def __init__(self):
+        pass
+
+    def resize_with_antialiasing(self, input, size, interpolation="bicubic", align_corners=True):
+        h, w = input.shape[-2:]
+        factors = (h / size[0], w / size[1])
+
+        # First, we have to determine sigma
+        # Taken from skimage: https://github.com/scikit-image/scikit-image/blob/v0.19.2/skimage/transform/_warps.py#L171
+        sigmas = (
+            max((factors[0] - 1.0) / 2.0, 0.001),
+            max((factors[1] - 1.0) / 2.0, 0.001),
+        )
+
+        # Now kernel size. Good results are for 3 sigma, but that is kind of slow. Pillow uses 1 sigma
+        # https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Resample.c#L206
+        # But they do it in the 2 passes, which gives better results. Let's try 2 sigmas for now
+        ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3))
+
+        # Make sure it is odd
+        if (ks[0] % 2) == 0:
+            ks = ks[0] + 1, ks[1]
+
+        if (ks[1] % 2) == 0:
+            ks = ks[0], ks[1] + 1
+
+        input = self._gaussian_blur2d(input, ks, sigmas)
+
+        output = torch.nn.functional.interpolate(input, size=size, mode=interpolation, align_corners=align_corners)
+        return output
+
+
+    def _compute_padding(self, kernel_size):
+        """Compute padding tuple."""
+        # 4 or 6 ints:  (padding_left, padding_right,padding_top,padding_bottom)
+        # https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad
+        if len(kernel_size) < 2:
+            raise AssertionError(kernel_size)
+        computed = [k - 1 for k in kernel_size]
+
+        # for even kernels we need to do asymmetric padding :(
+        out_padding = 2 * len(kernel_size) * [0]
+
+        for i in range(len(kernel_size)):
+            computed_tmp = computed[-(i + 1)]
+
+            pad_front = computed_tmp // 2
+            pad_rear = computed_tmp - pad_front
+
+            out_padding[2 * i + 0] = pad_front
+            out_padding[2 * i + 1] = pad_rear
+
+        return out_padding
+
+
+    def _filter2d(self, input, kernel):
+        # prepare kernel
+        b, c, h, w = input.shape
+        tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype)
+
+        tmp_kernel = tmp_kernel.expand(-1, c, -1, -1)
+
+        height, width = tmp_kernel.shape[-2:]
+
+        padding_shape: list[int] = self._compute_padding([height, width])
+        input = torch.nn.functional.pad(input, padding_shape, mode="reflect")
+
+        # kernel and input tensor reshape to align element-wise or batch-wise params
+        tmp_kernel = tmp_kernel.reshape(-1, 1, height, width)
+        input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1))
+
+        # convolve the tensor with the kernel.
+        output = torch.nn.functional.conv2d(input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1)
+
+        out = output.view(b, c, h, w)
+        return out
+
+
+    def _gaussian(self, window_size: int, sigma):
+        if isinstance(sigma, float):
+            sigma = torch.tensor([[sigma]])
+
+        batch_size = sigma.shape[0]
+
+        x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1)
+
+        if window_size % 2 == 0:
+            x = x + 0.5
+
+        gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0)))
+
+        return gauss / gauss.sum(-1, keepdim=True)
+
+
+    def _gaussian_blur2d(self, input, kernel_size, sigma):
+        if isinstance(sigma, tuple):
+            sigma = torch.tensor([sigma], dtype=input.dtype)
+        else:
+            sigma = sigma.to(dtype=input.dtype)
+
+        ky, kx = int(kernel_size[0]), int(kernel_size[1])
+        bs = sigma.shape[0]
+        kernel_x = self._gaussian(kx, sigma[:, 1].view(bs, 1))
+        kernel_y = self._gaussian(ky, sigma[:, 0].view(bs, 1))
+        out_x = self._filter2d(input, kernel_x[..., None, :])
+        out = self._filter2d(out_x, kernel_y[..., None])
+
+        return out

diffsynth/processors/FastBlend.py

@@ -0,0 +1,142 @@
+from PIL import Image
+import cupy as cp
+import numpy as np
+from tqdm import tqdm
+from ..extensions.FastBlend.patch_match import PyramidPatchMatcher
+from ..extensions.FastBlend.runners.fast import TableManager
+from .base import VideoProcessor
+
+
+class FastBlendSmoother(VideoProcessor):
+    def __init__(
+        self,
+        inference_mode="fast", batch_size=8, window_size=60,
+        minimum_patch_size=5, threads_per_block=8, num_iter=5, gpu_id=0, guide_weight=10.0, initialize="identity", tracking_window_size=0
+    ):
+        self.inference_mode = inference_mode
+        self.batch_size = batch_size
+        self.window_size = window_size
+        self.ebsynth_config = {
+            "minimum_patch_size": minimum_patch_size,
+            "threads_per_block": threads_per_block,
+            "num_iter": num_iter,
+            "gpu_id": gpu_id,
+            "guide_weight": guide_weight,
+            "initialize": initialize,
+            "tracking_window_size": tracking_window_size
+        }
+
+    @staticmethod
+    def from_model_manager(model_manager, **kwargs):
+        # TODO: fetch GPU ID from model_manager
+        return FastBlendSmoother(**kwargs)
+
+    def inference_fast(self, frames_guide, frames_style):
+        table_manager = TableManager()
+        patch_match_engine = PyramidPatchMatcher(
+            image_height=frames_style[0].shape[0],
+            image_width=frames_style[0].shape[1],
+            channel=3,
+            **self.ebsynth_config
+        )
+        # left part
+        table_l = table_manager.build_remapping_table(frames_guide, frames_style, patch_match_engine, self.batch_size, desc="Fast Mode Step 1/4")
+        table_l = table_manager.remapping_table_to_blending_table(table_l)
+        table_l = table_manager.process_window_sum(frames_guide, table_l, patch_match_engine, self.window_size, self.batch_size, desc="Fast Mode Step 2/4")
+        # right part
+        table_r = table_manager.build_remapping_table(frames_guide[::-1], frames_style[::-1], patch_match_engine, self.batch_size, desc="Fast Mode Step 3/4")
+        table_r = table_manager.remapping_table_to_blending_table(table_r)
+        table_r = table_manager.process_window_sum(frames_guide[::-1], table_r, patch_match_engine, self.window_size, self.batch_size, desc="Fast Mode Step 4/4")[::-1]
+        # merge
+        frames = []
+        for (frame_l, weight_l), frame_m, (frame_r, weight_r) in zip(table_l, frames_style, table_r):
+            weight_m = -1
+            weight = weight_l + weight_m + weight_r
+            frame = frame_l * (weight_l / weight) + frame_m * (weight_m / weight) + frame_r * (weight_r / weight)
+            frames.append(frame)
+        frames = [frame.clip(0, 255).astype("uint8") for frame in frames]
+        frames = [Image.fromarray(frame) for frame in frames]
+        return frames
+    
+    def inference_balanced(self, frames_guide, frames_style):
+        patch_match_engine = PyramidPatchMatcher(
+            image_height=frames_style[0].shape[0],
+            image_width=frames_style[0].shape[1],
+            channel=3,
+            **self.ebsynth_config
+        )
+        output_frames = []
+        # tasks
+        n = len(frames_style)
+        tasks = []
+        for target in range(n):
+            for source in range(target - self.window_size, target + self.window_size + 1):
+                if source >= 0 and source < n and source != target:
+                    tasks.append((source, target))
+        # run
+        frames = [(None, 1) for i in range(n)]
+        for batch_id in tqdm(range(0, len(tasks), self.batch_size), desc="Balanced Mode"):
+            tasks_batch = tasks[batch_id: min(batch_id+self.batch_size, len(tasks))]
+            source_guide = np.stack([frames_guide[source] for source, target in tasks_batch])
+            target_guide = np.stack([frames_guide[target] for source, target in tasks_batch])
+            source_style = np.stack([frames_style[source] for source, target in tasks_batch])
+            _, target_style = patch_match_engine.estimate_nnf(source_guide, target_guide, source_style)
+            for (source, target), result in zip(tasks_batch, target_style):
+                frame, weight = frames[target]
+                if frame is None:
+                    frame = frames_style[target]
+                frames[target] = (
+                    frame * (weight / (weight + 1)) + result / (weight + 1),
+                    weight + 1
+                )
+                if weight + 1 == min(n, target + self.window_size + 1) - max(0, target - self.window_size):
+                    frame = frame.clip(0, 255).astype("uint8")
+                    output_frames.append(Image.fromarray(frame))
+                    frames[target] = (None, 1)
+        return output_frames
+    
+    def inference_accurate(self, frames_guide, frames_style):
+        patch_match_engine = PyramidPatchMatcher(
+            image_height=frames_style[0].shape[0],
+            image_width=frames_style[0].shape[1],
+            channel=3,
+            use_mean_target_style=True,
+            **self.ebsynth_config
+        )
+        output_frames = []
+        # run
+        n = len(frames_style)
+        for target in tqdm(range(n), desc="Accurate Mode"):
+            l, r = max(target - self.window_size, 0), min(target + self.window_size + 1, n)
+            remapped_frames = []
+            for i in range(l, r, self.batch_size):
+                j = min(i + self.batch_size, r)
+                source_guide = np.stack([frames_guide[source] for source in range(i, j)])
+                target_guide = np.stack([frames_guide[target]] * (j - i))
+                source_style = np.stack([frames_style[source] for source in range(i, j)])
+                _, target_style = patch_match_engine.estimate_nnf(source_guide, target_guide, source_style)
+                remapped_frames.append(target_style)
+            frame = np.concatenate(remapped_frames, axis=0).mean(axis=0)
+            frame = frame.clip(0, 255).astype("uint8")
+            output_frames.append(Image.fromarray(frame))
+        return output_frames
+    
+    def release_vram(self):
+        mempool = cp.get_default_memory_pool()
+        pinned_mempool = cp.get_default_pinned_memory_pool()
+        mempool.free_all_blocks()
+        pinned_mempool.free_all_blocks()
+    
+    def __call__(self, rendered_frames, original_frames=None, **kwargs):
+        rendered_frames = [np.array(frame) for frame in rendered_frames]
+        original_frames = [np.array(frame) for frame in original_frames]
+        if self.inference_mode == "fast":
+            output_frames = self.inference_fast(original_frames, rendered_frames)
+        elif self.inference_mode == "balanced":
+            output_frames = self.inference_balanced(original_frames, rendered_frames)
+        elif self.inference_mode == "accurate":
+            output_frames = self.inference_accurate(original_frames, rendered_frames)
+        else:
+            raise ValueError("inference_mode must be fast, balanced or accurate")
+        self.release_vram()
+        return output_frames

diffsynth/processors/PILEditor.py

@@ -0,0 +1,28 @@
+from PIL import ImageEnhance
+from .base import VideoProcessor
+
+
+class ContrastEditor(VideoProcessor):
+    def __init__(self, rate=1.5):
+        self.rate = rate
+
+    @staticmethod
+    def from_model_manager(model_manager, **kwargs):
+        return ContrastEditor(**kwargs)
+    
+    def __call__(self, rendered_frames, **kwargs):
+        rendered_frames = [ImageEnhance.Contrast(i).enhance(self.rate) for i in rendered_frames]
+        return rendered_frames
+
+
+class SharpnessEditor(VideoProcessor):
+    def __init__(self, rate=1.5):
+        self.rate = rate
+
+    @staticmethod
+    def from_model_manager(model_manager, **kwargs):
+        return SharpnessEditor(**kwargs)
+    
+    def __call__(self, rendered_frames, **kwargs):
+        rendered_frames = [ImageEnhance.Sharpness(i).enhance(self.rate) for i in rendered_frames]
+        return rendered_frames

diffsynth/processors/RIFE.py

@@ -0,0 +1,77 @@
+import torch
+import numpy as np
+from PIL import Image
+from .base import VideoProcessor
+
+
+class RIFESmoother(VideoProcessor):
+    def __init__(self, model, device="cuda", scale=1.0, batch_size=4, interpolate=True):
+        self.model = model
+        self.device = device
+
+        # IFNet only does not support float16
+        self.torch_dtype = torch.float32
+
+        # Other parameters
+        self.scale = scale
+        self.batch_size = batch_size
+        self.interpolate = interpolate
+
+    @staticmethod
+    def from_model_manager(model_manager, **kwargs):
+        return RIFESmoother(model_manager.RIFE, device=model_manager.device, **kwargs)
+    
+    def process_image(self, image):
+        width, height = image.size
+        if width % 32 != 0 or height % 32 != 0:
+            width = (width + 31) // 32
+            height = (height + 31) // 32
+            image = image.resize((width, height))
+        image = torch.Tensor(np.array(image, dtype=np.float32)[:, :, [2,1,0]] / 255).permute(2, 0, 1)
+        return image
+    
+    def process_images(self, images):
+        images = [self.process_image(image) for image in images]
+        images = torch.stack(images)
+        return images
+    
+    def decode_images(self, images):
+        images = (images[:, [2,1,0]].permute(0, 2, 3, 1) * 255).clip(0, 255).numpy().astype(np.uint8)
+        images = [Image.fromarray(image) for image in images]
+        return images
+    
+    def process_tensors(self, input_tensor, scale=1.0, batch_size=4):
+        output_tensor = []
+        for batch_id in range(0, input_tensor.shape[0], batch_size):
+            batch_id_ = min(batch_id + batch_size, input_tensor.shape[0])
+            batch_input_tensor = input_tensor[batch_id: batch_id_]
+            batch_input_tensor = batch_input_tensor.to(device=self.device, dtype=self.torch_dtype)
+            flow, mask, merged = self.model(batch_input_tensor, [4/scale, 2/scale, 1/scale])
+            output_tensor.append(merged[2].cpu())
+        output_tensor = torch.concat(output_tensor, dim=0)
+        return output_tensor
+
+    @torch.no_grad()
+    def __call__(self, rendered_frames, **kwargs):
+        # Preprocess
+        processed_images = self.process_images(rendered_frames)
+
+        # Input
+        input_tensor = torch.cat((processed_images[:-2], processed_images[2:]), dim=1)
+
+        # Interpolate
+        output_tensor = self.process_tensors(input_tensor, scale=self.scale, batch_size=self.batch_size)
+        
+        if self.interpolate:
+            # Blend
+            input_tensor = torch.cat((processed_images[1:-1], output_tensor), dim=1)
+            output_tensor = self.process_tensors(input_tensor, scale=self.scale, batch_size=self.batch_size)
+            processed_images[1:-1] = output_tensor
+        else:
+            processed_images[1:-1] = (processed_images[1:-1] + output_tensor) / 2
+
+        # To images
+        output_images = self.decode_images(processed_images)
+        if output_images[0].size != rendered_frames[0].size:
+            output_images = [image.resize(rendered_frames[0].size) for image in output_images]
+        return output_images

diffsynth/processors/base.py

@@ -0,0 +1,6 @@
+class VideoProcessor:
+    def __init__(self):
+        pass
+
+    def __call__(self):
+        raise NotImplementedError

diffsynth/processors/sequencial_processor.py

@@ -0,0 +1,41 @@
+from .base import VideoProcessor
+
+
+class AutoVideoProcessor(VideoProcessor):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def from_model_manager(model_manager, processor_type, **kwargs):
+        if processor_type == "FastBlend":
+            from .FastBlend import FastBlendSmoother
+            return FastBlendSmoother.from_model_manager(model_manager, **kwargs)
+        elif processor_type == "Contrast":
+            from .PILEditor import ContrastEditor
+            return ContrastEditor.from_model_manager(model_manager, **kwargs)
+        elif processor_type == "Sharpness":
+            from .PILEditor import SharpnessEditor
+            return SharpnessEditor.from_model_manager(model_manager, **kwargs)
+        elif processor_type == "RIFE":
+            from .RIFE import RIFESmoother
+            return RIFESmoother.from_model_manager(model_manager, **kwargs)
+        else:
+            raise ValueError(f"invalid processor_type: {processor_type}")
+
+
+class SequencialProcessor(VideoProcessor):
+    def __init__(self, processors=[]):
+        self.processors = processors
+
+    @staticmethod
+    def from_model_manager(model_manager, configs):
+        processors = [
+            AutoVideoProcessor.from_model_manager(model_manager, config["processor_type"], **config["config"])
+            for config in configs
+        ]
+        return SequencialProcessor(processors)
+    
+    def __call__(self, rendered_frames, **kwargs):
+        for processor in self.processors:
+            rendered_frames = processor(rendered_frames, **kwargs)
+        return rendered_frames

diffsynth/prompters/__init__.py

@@ -0,0 +1,6 @@
+from .prompt_refiners import Translator, BeautifulPrompt
+from .sd_prompter import SDPrompter
+from .sdxl_prompter import SDXLPrompter
+from .sd3_prompter import SD3Prompter
+from .hunyuan_dit_prompter import HunyuanDiTPrompter
+from .kolors_prompter import KolorsPrompter

diffsynth/prompters/base_prompter.py

@@ -0,0 +1,57 @@
+from ..models.model_manager import ModelManager
+import torch
+
+
+
+def tokenize_long_prompt(tokenizer, prompt, max_length=None):
+    # Get model_max_length from self.tokenizer
+    length = tokenizer.model_max_length if max_length is None else max_length
+
+    # To avoid the warning. set self.tokenizer.model_max_length to +oo.
+    tokenizer.model_max_length = 99999999
+
+    # Tokenize it!
+    input_ids = tokenizer(prompt, return_tensors="pt").input_ids
+
+    # Determine the real length.
+    max_length = (input_ids.shape[1] + length - 1) // length * length
+
+    # Restore tokenizer.model_max_length
+    tokenizer.model_max_length = length
+    
+    # Tokenize it again with fixed length.
+    input_ids = tokenizer(
+        prompt,
+        return_tensors="pt",
+        padding="max_length",
+        max_length=max_length,
+        truncation=True
+    ).input_ids
+
+    # Reshape input_ids to fit the text encoder.
+    num_sentence = input_ids.shape[1] // length
+    input_ids = input_ids.reshape((num_sentence, length))
+    
+    return input_ids
+
+
+
+class BasePrompter:
+    def __init__(self, refiners=[]):
+        self.refiners = refiners
+
+
+    def load_prompt_refiners(self, model_nameger: ModelManager, refiner_classes=[]):
+        for refiner_class in refiner_classes:
+            refiner = refiner_class.from_model_manager(model_nameger)
+            self.refiners.append(refiner)
+
+
+    @torch.no_grad()
+    def process_prompt(self, prompt, positive=True):
+        if isinstance(prompt, list):
+            prompt = [self.process_prompt(prompt_, positive=positive) for prompt_ in prompt]
+        else:
+            for refiner in self.refiners:
+                prompt = refiner(prompt, positive=positive)
+        return prompt