ai-simple-engine-runtime-diffusers 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ai_simple_engine_runtime_diffusers/__init__.py +4 -0
- ai_simple_engine_runtime_diffusers/models/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/executor/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/executor/diffusers_latent_diffusion_model_executor.py +123 -0
- ai_simple_engine_runtime_diffusers/models/executor/noise/__init__.py +4 -0
- ai_simple_engine_runtime_diffusers/models/executor/noise/pytorch_noise_generator.py +25 -0
- ai_simple_engine_runtime_diffusers/models/executor/prompt_embeddings/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/executor/prompt_embeddings/prompt_embeddings.py +18 -0
- ai_simple_engine_runtime_diffusers/models/executor/text_encoder_model_executor_abstract.py +32 -0
- ai_simple_engine_runtime_diffusers/models/executor/unet_model_executor_abstract.py +36 -0
- ai_simple_engine_runtime_diffusers/models/executor/vae_model_executor_abstract.py +44 -0
- ai_simple_engine_runtime_diffusers/models/loader/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/loader/latent_diffusion_model_loader.py +90 -0
- ai_simple_engine_runtime_diffusers/models/runtime/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/runtime/diffusers_latent_diffusion_model.py +19 -0
- ai_simple_engine_runtime_diffusers/models/runtime/utils.py +19 -0
- ai_simple_engine_runtime_diffusers/models/scheduler/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/scheduler/diffusers_scheduler.py +47 -0
- ai_simple_engine_runtime_diffusers/models/scheduler/registry/__init__.py +0 -0
- ai_simple_engine_runtime_diffusers/models/scheduler/registry/diffusers_scheduler_registry.py +22 -0
- ai_simple_engine_runtime_diffusers/models/scheduler/utils.py +44 -0
- ai_simple_engine_runtime_diffusers/plugin.py +69 -0
- ai_simple_engine_runtime_diffusers-0.0.1.dist-info/METADATA +21 -0
- ai_simple_engine_runtime_diffusers-0.0.1.dist-info/RECORD +26 -0
- ai_simple_engine_runtime_diffusers-0.0.1.dist-info/WHEEL +4 -0
- ai_simple_engine_runtime_diffusers-0.0.1.dist-info/licenses/LICENSE +19 -0
|
File without changes
|
|
File without changes
|
ai_simple_engine_runtime_diffusers/models/executor/diffusers_latent_diffusion_model_executor.py
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
from ai_simple_engine_runtime_diffusers.models.scheduler.diffusers_scheduler import DiffusersScheduler
|
|
2
|
+
from ai_simple_engine_diffusion.model.executor.abstract.latent_diffusion_model_executor_abstract import LatentDiffusionModelExecutorAbstract
|
|
3
|
+
from ai_simple_engine.execution.execution_context import ExecutionContext
|
|
4
|
+
from ai_simple_engine_runtime_diffusers.models.executor.prompt_embeddings.prompt_embeddings import PromptEmbeddings
|
|
5
|
+
from ai_simple_engine_runtime_diffusers.models.scheduler.registry.diffusers_scheduler_registry import DiffusersSchedulerRegistry
|
|
6
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class DiffusersLatentDiffusionModelExecutor (
|
|
12
|
+
LatentDiffusionModelExecutorAbstract
|
|
13
|
+
):
|
|
14
|
+
"""
|
|
15
|
+
Latent diffusion executor based on the Hugging Face
|
|
16
|
+
Diffusers library.
|
|
17
|
+
|
|
18
|
+
Implements the model-specific operations required by
|
|
19
|
+
`LatentDiffusionModelExecutor` using Diffusers
|
|
20
|
+
components such as the tokenizer, text encoder,
|
|
21
|
+
UNet/DiT, VAE and schedulers.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
def create_scheduler(
|
|
25
|
+
self,
|
|
26
|
+
context: ExecutionContext,
|
|
27
|
+
model: LoadedModel,
|
|
28
|
+
):
|
|
29
|
+
scheduler_registry = context.services.resolve(DiffusersSchedulerRegistry)
|
|
30
|
+
|
|
31
|
+
scheduler_class = scheduler_registry.resolve(model.info.scheduler.name)
|
|
32
|
+
|
|
33
|
+
scheduler = scheduler_class.from_config(
|
|
34
|
+
model.instance.scheduler_config,
|
|
35
|
+
**model.info.scheduler.kwargs
|
|
36
|
+
)
|
|
37
|
+
|
|
38
|
+
return DiffusersScheduler(scheduler)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
async def predict_noise(
|
|
42
|
+
self,
|
|
43
|
+
*,
|
|
44
|
+
model: LoadedModel,
|
|
45
|
+
latents,
|
|
46
|
+
embeddings: PromptEmbeddings,
|
|
47
|
+
timestep,
|
|
48
|
+
guidance_scale
|
|
49
|
+
):
|
|
50
|
+
runtime_model = model.instance
|
|
51
|
+
unet = runtime_model.unet
|
|
52
|
+
latent_input = torch.cat([latents, latents])
|
|
53
|
+
|
|
54
|
+
encoder_hidden_states = torch.cat([
|
|
55
|
+
embeddings.negative,
|
|
56
|
+
embeddings.positive
|
|
57
|
+
])
|
|
58
|
+
|
|
59
|
+
noise = unet(
|
|
60
|
+
latent_input,
|
|
61
|
+
timestep,
|
|
62
|
+
encoder_hidden_states = encoder_hidden_states
|
|
63
|
+
).sample
|
|
64
|
+
|
|
65
|
+
noise_uncond, noise_text = noise.chunk(2)
|
|
66
|
+
|
|
67
|
+
return noise_uncond + guidance_scale * (
|
|
68
|
+
noise_text - noise_uncond
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
# Specific methods below
|
|
72
|
+
async def encode_prompt(
|
|
73
|
+
self,
|
|
74
|
+
*,
|
|
75
|
+
model: LoadedModel,
|
|
76
|
+
prompt: str,
|
|
77
|
+
negative_prompt: str
|
|
78
|
+
) -> PromptEmbeddings:
|
|
79
|
+
runtime_model = model.instance
|
|
80
|
+
|
|
81
|
+
tokenizer = runtime_model.tokenizer
|
|
82
|
+
text_encoder = runtime_model.text_encoder
|
|
83
|
+
|
|
84
|
+
positive = tokenizer(
|
|
85
|
+
prompt,
|
|
86
|
+
padding = 'max_length',
|
|
87
|
+
truncation = True,
|
|
88
|
+
max_length = tokenizer.model_max_length,
|
|
89
|
+
return_tensors = 'pt'
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
negative = tokenizer(
|
|
93
|
+
negative_prompt,
|
|
94
|
+
padding = 'max_length',
|
|
95
|
+
truncation = True,
|
|
96
|
+
max_length = tokenizer.model_max_length,
|
|
97
|
+
return_tensors = 'pt'
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
device = next(text_encoder.parameters()).device
|
|
101
|
+
positive = text_encoder(positive.input_ids.to(device))[0]
|
|
102
|
+
negative = text_encoder(negative.input_ids.to(device))[0]
|
|
103
|
+
|
|
104
|
+
return PromptEmbeddings(
|
|
105
|
+
positive = positive,
|
|
106
|
+
negative = negative
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
async def decode_latents(
|
|
110
|
+
self,
|
|
111
|
+
*,
|
|
112
|
+
model,
|
|
113
|
+
latents
|
|
114
|
+
):
|
|
115
|
+
runtime_model = model.model
|
|
116
|
+
|
|
117
|
+
vae = runtime_model.vae
|
|
118
|
+
|
|
119
|
+
latents = latents / vae.config.scaling_factor
|
|
120
|
+
|
|
121
|
+
image = vae.decode(latents).sample
|
|
122
|
+
|
|
123
|
+
return image
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
from ai_simple_engine_diffusion.noise.noise_generator_abstract import NoiseGenerator
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class TorchNoiseGenerator(
|
|
7
|
+
NoiseGenerator
|
|
8
|
+
):
|
|
9
|
+
|
|
10
|
+
def generate(
|
|
11
|
+
self,
|
|
12
|
+
*,
|
|
13
|
+
shape,
|
|
14
|
+
device,
|
|
15
|
+
dtype,
|
|
16
|
+
seed: int
|
|
17
|
+
):
|
|
18
|
+
generator = torch.Generator(device = device).manual_seed(seed)
|
|
19
|
+
|
|
20
|
+
return torch.randn(
|
|
21
|
+
shape,
|
|
22
|
+
generator = generator,
|
|
23
|
+
device = device,
|
|
24
|
+
dtype = dtype,
|
|
25
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO: I don't like this module here
|
|
3
|
+
"""
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
@dataclass(frozen = True)
|
|
8
|
+
class PromptEmbeddings:
|
|
9
|
+
"""
|
|
10
|
+
*Dataclass*
|
|
11
|
+
|
|
12
|
+
Class to include the `positive` and the
|
|
13
|
+
`negative` prompt embeddings at the same
|
|
14
|
+
time.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
positive: object
|
|
18
|
+
negative: object
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO: This is unnecessary, but maybe we want
|
|
3
|
+
each ModelExecutor being capable of performing
|
|
4
|
+
single operations. Check and remove if not.
|
|
5
|
+
"""
|
|
6
|
+
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
7
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
8
|
+
from ai_simple_engine_diffusion.types.data_type import Embeddings
|
|
9
|
+
from abc import ABC, abstractmethod
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
# This will be to perform 'EncodePrompt()'
|
|
13
|
+
class TextEncoderModelExecutorAbstract(
|
|
14
|
+
ModelExecutor,
|
|
15
|
+
ABC
|
|
16
|
+
):
|
|
17
|
+
"""
|
|
18
|
+
*Abstract class*
|
|
19
|
+
|
|
20
|
+
To convert text into embeddings
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
@abstractmethod
|
|
24
|
+
async def encode(
|
|
25
|
+
self,
|
|
26
|
+
model: LoadedModel,
|
|
27
|
+
prompt: str
|
|
28
|
+
) -> Embeddings:
|
|
29
|
+
"""
|
|
30
|
+
Encode the given prompt into embeddings.
|
|
31
|
+
"""
|
|
32
|
+
...
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO: This is unnecessary, but maybe we want
|
|
3
|
+
each ModelExecutor being capable of performing
|
|
4
|
+
single operations. Check and remove if not.
|
|
5
|
+
"""
|
|
6
|
+
from ai_simple_engine_diffusion.types.latents import Latents
|
|
7
|
+
from ai_simple_engine_diffusion.types.noise_prediction import NoisePrediction
|
|
8
|
+
from ai_simple_engine_diffusion.types.embeddings import Embeddings
|
|
9
|
+
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
10
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class UNetModelExecutorAbstract(
|
|
15
|
+
ModelExecutor,
|
|
16
|
+
ABC
|
|
17
|
+
):
|
|
18
|
+
"""
|
|
19
|
+
*Abstract class*
|
|
20
|
+
|
|
21
|
+
To predict the noise.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
async def predict_noise(
|
|
26
|
+
self,
|
|
27
|
+
model: LoadedModel,
|
|
28
|
+
latents: Latents,
|
|
29
|
+
embeddings: Embeddings,
|
|
30
|
+
timestep: int
|
|
31
|
+
) -> NoisePrediction:
|
|
32
|
+
"""
|
|
33
|
+
Predict the noise residual for the given
|
|
34
|
+
latents at the specified timestep.
|
|
35
|
+
"""
|
|
36
|
+
...
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO: This is unnecessary, but maybe we want
|
|
3
|
+
each ModelExecutor being capable of performing
|
|
4
|
+
single operations. Check and remove if not.
|
|
5
|
+
"""
|
|
6
|
+
from ai_simple_engine_diffusion.types.latents import Latents
|
|
7
|
+
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
8
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
9
|
+
from ai_simple_engine.types.image import Image
|
|
10
|
+
from abc import ABC, abstractmethod
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class VAEModelExecutorAbstract(
|
|
14
|
+
ModelExecutor,
|
|
15
|
+
ABC
|
|
16
|
+
):
|
|
17
|
+
"""
|
|
18
|
+
*Abstract class*
|
|
19
|
+
|
|
20
|
+
To transform images into latents and
|
|
21
|
+
latents into images.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
async def encode(
|
|
26
|
+
self,
|
|
27
|
+
model: LoadedModel,
|
|
28
|
+
image: Image
|
|
29
|
+
) -> Latents:
|
|
30
|
+
"""
|
|
31
|
+
Encode an image into latent space.
|
|
32
|
+
"""
|
|
33
|
+
...
|
|
34
|
+
|
|
35
|
+
@abstractmethod
|
|
36
|
+
async def decode(
|
|
37
|
+
self,
|
|
38
|
+
model: LoadedModel,
|
|
39
|
+
latents: Latents
|
|
40
|
+
) -> Image:
|
|
41
|
+
"""
|
|
42
|
+
Decode latent representations into an image.
|
|
43
|
+
"""
|
|
44
|
+
...
|
|
File without changes
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from ai_simple_engine_runtime_diffusers.models.runtime.diffusers_latent_diffusion_model import DiffusersLatentDiffusionModel
|
|
2
|
+
from ai_simple_engine_runtime_diffusers.models.runtime.utils import get_torch_dtype_for
|
|
3
|
+
from ai_simple_engine_diffusion.model.info.diffusion_model_info import DiffusionModelInfo
|
|
4
|
+
from ai_simple_engine_diffusion.scheduler.spec.base import SchedulerSpec
|
|
5
|
+
from ai_simple_engine.device.base import Device
|
|
6
|
+
from ai_simple_engine.models.loaders.abstract import ModelLoader
|
|
7
|
+
from ai_simple_engine.models.installed_model import InstalledModel
|
|
8
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
9
|
+
from transformers import CLIPTokenizer, CLIPTextModel
|
|
10
|
+
from diffusers import UNet2DConditionModel, AutoencoderKL
|
|
11
|
+
|
|
12
|
+
import torch
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class LatentDiffusionModelLoader(
|
|
16
|
+
ModelLoader
|
|
17
|
+
):
|
|
18
|
+
|
|
19
|
+
async def load(
|
|
20
|
+
self,
|
|
21
|
+
installed_model: InstalledModel,
|
|
22
|
+
device: Device
|
|
23
|
+
) -> LoadedModel:
|
|
24
|
+
path = str(installed_model.path)
|
|
25
|
+
|
|
26
|
+
tokenizer = CLIPTokenizer.from_pretrained(
|
|
27
|
+
path,
|
|
28
|
+
subfolder = 'tokenizer',
|
|
29
|
+
torch_dtype = get_torch_dtype_for(device)
|
|
30
|
+
).to(str(device))
|
|
31
|
+
|
|
32
|
+
text_encoder = CLIPTextModel.from_pretrained(
|
|
33
|
+
path,
|
|
34
|
+
subfolder = 'text_encoder',
|
|
35
|
+
torch_dtype = get_torch_dtype_for(device)
|
|
36
|
+
).to(str(device))
|
|
37
|
+
|
|
38
|
+
unet = UNet2DConditionModel.from_pretrained(
|
|
39
|
+
path,
|
|
40
|
+
subfolder = 'unet',
|
|
41
|
+
torch_dtype = get_torch_dtype_for(device)
|
|
42
|
+
).to(str(device))
|
|
43
|
+
|
|
44
|
+
vae = AutoencoderKL.from_pretrained(
|
|
45
|
+
path,
|
|
46
|
+
subfolder = 'vae',
|
|
47
|
+
torch_dtype = get_torch_dtype_for(device)
|
|
48
|
+
).to(str(device))
|
|
49
|
+
|
|
50
|
+
scheduler_config = AutoencoderKL.load_config(
|
|
51
|
+
path,
|
|
52
|
+
subfolder = 'scheduler',
|
|
53
|
+
torch_dtype = get_torch_dtype_for(device)
|
|
54
|
+
).to(str(device))
|
|
55
|
+
|
|
56
|
+
runtime_model = DiffusersLatentDiffusionModel(
|
|
57
|
+
tokenizer = tokenizer,
|
|
58
|
+
text_encoder = text_encoder,
|
|
59
|
+
unet = unet,
|
|
60
|
+
vae = vae,
|
|
61
|
+
scheduler_config = scheduler_config
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
info = DiffusionModelInfo(
|
|
65
|
+
latent_channels = unet.config.in_channels,
|
|
66
|
+
vae_scale_factor = 2 ** (len(vae.config.block_out_channels) - 1),
|
|
67
|
+
# TODO: Make dynamic depending on the 'installed_model'
|
|
68
|
+
scheduler = self._default_scheduler(installed_model)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
return LoadedModel(
|
|
72
|
+
installed_model = installed_model,
|
|
73
|
+
instance = runtime_model,
|
|
74
|
+
info = info
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
def _default_scheduler(
|
|
78
|
+
self,
|
|
79
|
+
installed_model: InstalledModel
|
|
80
|
+
) -> SchedulerSpec:
|
|
81
|
+
"""
|
|
82
|
+
*For internal use only*
|
|
83
|
+
|
|
84
|
+
Get the scheduler by default for the
|
|
85
|
+
`installed_model` provided.
|
|
86
|
+
"""
|
|
87
|
+
# TODO: Make dynamic based on the 'installed_model'
|
|
88
|
+
return SchedulerSpec(
|
|
89
|
+
identifier = 'euler'
|
|
90
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from transformers import CLIPTokenizer, CLIPTextModel
|
|
2
|
+
from diffusers import UNet2DConditionModel, AutoencoderKL
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass(frozen = True)
|
|
7
|
+
class DiffusersLatentDiffusionModel:
|
|
8
|
+
"""
|
|
9
|
+
Runtime representation of a Latent Diffusion
|
|
10
|
+
model loaded through Diffusers.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
tokenizer: CLIPTokenizer
|
|
14
|
+
text_encoder: CLIPTextModel
|
|
15
|
+
unet: UNet2DConditionModel
|
|
16
|
+
vae: AutoencoderKL
|
|
17
|
+
scheduler_config: dict
|
|
18
|
+
|
|
19
|
+
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
from ai_simple_engine.device.base import Device
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def get_torch_dtype_for(
|
|
7
|
+
device: Device
|
|
8
|
+
):
|
|
9
|
+
"""
|
|
10
|
+
Get the `torch` `dtype` for the `device`
|
|
11
|
+
provided.
|
|
12
|
+
"""
|
|
13
|
+
if device.type == 'cuda':
|
|
14
|
+
return torch.float16
|
|
15
|
+
|
|
16
|
+
if device.type == 'mps':
|
|
17
|
+
return torch.float16
|
|
18
|
+
|
|
19
|
+
return torch.float32
|
|
File without changes
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
from ai_simple_engine_diffusion.scheduler.abstract import Scheduler
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class DiffusersScheduler(
|
|
5
|
+
Scheduler
|
|
6
|
+
):
|
|
7
|
+
|
|
8
|
+
def __init__(
|
|
9
|
+
self,
|
|
10
|
+
scheduler
|
|
11
|
+
):
|
|
12
|
+
self._scheduler = scheduler
|
|
13
|
+
|
|
14
|
+
@property
|
|
15
|
+
def timesteps(
|
|
16
|
+
self
|
|
17
|
+
):
|
|
18
|
+
return self._scheduler.timesteps
|
|
19
|
+
|
|
20
|
+
def set_timesteps(
|
|
21
|
+
self,
|
|
22
|
+
steps: int
|
|
23
|
+
):
|
|
24
|
+
self._scheduler.set_timesteps(steps)
|
|
25
|
+
|
|
26
|
+
def scale_model_input(
|
|
27
|
+
self,
|
|
28
|
+
latents,
|
|
29
|
+
timestep
|
|
30
|
+
):
|
|
31
|
+
return self._scheduler.scale_model_input(
|
|
32
|
+
latents,
|
|
33
|
+
timestep
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
def step(
|
|
37
|
+
self,
|
|
38
|
+
*,
|
|
39
|
+
model_output,
|
|
40
|
+
timestep,
|
|
41
|
+
sample
|
|
42
|
+
):
|
|
43
|
+
return self._scheduler.step(
|
|
44
|
+
model_output = model_output,
|
|
45
|
+
timestep = timestep,
|
|
46
|
+
sample = sample
|
|
47
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from ai_simple_engine.resolver_registry import ResolverRegistry
|
|
2
|
+
from diffusers.schedulers.scheduling_utils import SchedulerMixin
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class DiffusersSchedulerRegistry(
|
|
6
|
+
ResolverRegistry[
|
|
7
|
+
str,
|
|
8
|
+
type[SchedulerMixin]
|
|
9
|
+
]
|
|
10
|
+
):
|
|
11
|
+
"""
|
|
12
|
+
Class that will register the different
|
|
13
|
+
Scheduler classes that we have available
|
|
14
|
+
so we can obtain them using their
|
|
15
|
+
identifier.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def key_for(
|
|
19
|
+
self,
|
|
20
|
+
identifier: str
|
|
21
|
+
) -> str:
|
|
22
|
+
return identifier
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""
|
|
2
|
+
TODO: Maybe rename and move this module.
|
|
3
|
+
"""
|
|
4
|
+
from diffusers.schedulers import (
|
|
5
|
+
DDIMScheduler,
|
|
6
|
+
EulerDiscreteScheduler,
|
|
7
|
+
EulerAncestralDiscreteScheduler,
|
|
8
|
+
DPMSolverMultistepScheduler,
|
|
9
|
+
FlowMatchEulerDiscreteScheduler
|
|
10
|
+
)
|
|
11
|
+
|
|
12
|
+
SCHEDULERS = {
|
|
13
|
+
'ddim': DDIMScheduler,
|
|
14
|
+
'euler': EulerDiscreteScheduler,
|
|
15
|
+
'euler_a': EulerAncestralDiscreteScheduler,
|
|
16
|
+
'dpmpp_2m': DPMSolverMultistepScheduler,
|
|
17
|
+
'flow_match_euler': FlowMatchEulerDiscreteScheduler
|
|
18
|
+
}
|
|
19
|
+
"""
|
|
20
|
+
The list including all the schedulers we are
|
|
21
|
+
accepting.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def get_scheduler_class(
|
|
26
|
+
scheduler: str,
|
|
27
|
+
model: LoadedModel
|
|
28
|
+
) -> Union[SchedulerMixin, None]:
|
|
29
|
+
scheduler_class = SCHEDULERS.get(scheduler, None)
|
|
30
|
+
|
|
31
|
+
if scheduler_class is None:
|
|
32
|
+
raise Exception(f'The scheduler "{scheduler}" is not accepted by our system.')
|
|
33
|
+
|
|
34
|
+
return scheduler_class
|
|
35
|
+
|
|
36
|
+
scheduler_cls = registry.resolve(model.info.scheduler.identifier)
|
|
37
|
+
|
|
38
|
+
scheduler = scheduler_cls.from_config(...)
|
|
39
|
+
|
|
40
|
+
return scheduler
|
|
41
|
+
|
|
42
|
+
return scheduler_cls.from_config(
|
|
43
|
+
model.model.scheduler.config
|
|
44
|
+
)
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
from ai_simple_engine_runtime_diffusers.models.loader.latent_diffusion_model_loader import LatentDiffusionModelLoader
|
|
2
|
+
from ai_simple_engine_runtime_diffusers.models.scheduler.utils import SCHEDULERS
|
|
3
|
+
from ai_simple_engine_runtime_diffusers.models.scheduler.registry.diffusers_scheduler_registry import DiffusersSchedulerRegistry
|
|
4
|
+
from ai_simple_engine.engine_builder import EngineBuilder
|
|
5
|
+
from ai_simple_engine.plugins.plugin import Plugin
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class DiffusersRuntimeMusicgenPlugin(
|
|
9
|
+
Plugin
|
|
10
|
+
):
|
|
11
|
+
"""
|
|
12
|
+
The plugin to add the diffusers' models
|
|
13
|
+
functionality.
|
|
14
|
+
|
|
15
|
+
This plugin includes:
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
def register(
|
|
19
|
+
self,
|
|
20
|
+
builder: EngineBuilder
|
|
21
|
+
):
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
(
|
|
25
|
+
builder.add_model_loader(LatentDiffusionModelLoader)
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
# TODO: Register the SchedulerRegistry
|
|
29
|
+
"""
|
|
30
|
+
Dinamycally register all the scheduler classes
|
|
31
|
+
incldued in the 'SCHEDULERS' dict. The executor
|
|
32
|
+
will resolve the instance when trying to
|
|
33
|
+
execute:
|
|
34
|
+
|
|
35
|
+
scheduler_cls = registry.resolve(
|
|
36
|
+
scheduler_spec.identifier
|
|
37
|
+
)
|
|
38
|
+
"""
|
|
39
|
+
scheduler_registry = builder.get_or_add_service(DiffusersSchedulerRegistry)
|
|
40
|
+
|
|
41
|
+
for identifier, cls in SCHEDULERS.items():
|
|
42
|
+
scheduler_registry.register(
|
|
43
|
+
identifier,
|
|
44
|
+
cls
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
# scheduler_registry.register(
|
|
48
|
+
# 'euler',
|
|
49
|
+
# EulerDiscreteScheduler
|
|
50
|
+
# )
|
|
51
|
+
|
|
52
|
+
# TODO: This below was the 'musicgen'
|
|
53
|
+
# (
|
|
54
|
+
# builder
|
|
55
|
+
# .add_model_loader(MusicgenModelLoader())
|
|
56
|
+
# )
|
|
57
|
+
|
|
58
|
+
# """
|
|
59
|
+
# We obtain the registry that handles the
|
|
60
|
+
# model executors by the model's family,
|
|
61
|
+
# and we register our specific model
|
|
62
|
+
# executor that uses transformers.
|
|
63
|
+
# """
|
|
64
|
+
# registry = builder.get_or_add_service(FamilyModelExecutorRegistry)
|
|
65
|
+
|
|
66
|
+
# registry.register(
|
|
67
|
+
# MUSICGEN_MODEL_FAMILY,
|
|
68
|
+
# TransformersMusicgenModelExecutor()
|
|
69
|
+
# )
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: ai-simple-engine-runtime-diffusers
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: AI Simple Engine Runtime Diffusers Module
|
|
5
|
+
License-File: LICENSE
|
|
6
|
+
Author: danialcala94
|
|
7
|
+
Author-email: danielalcalavalera@gmail.com
|
|
8
|
+
Requires-Python: >=3.10,<3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
+
Requires-Dist: ai-simple-engine (>=0.2.6,<1.0.0)
|
|
13
|
+
Requires-Dist: ai-simple-engine-common (>=0.0.3,<1.0.0)
|
|
14
|
+
Requires-Dist: ai-simple-engine-diffusion (>=0.0.2,<1.0.0)
|
|
15
|
+
Requires-Dist: diffusers (>=0.38.0,<1.0.0)
|
|
16
|
+
Requires-Dist: transformers (>=0.0.1,<1.0.0)
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# AI Simple Engine Runtime Diffusers Module
|
|
20
|
+
|
|
21
|
+
The module that implements the Runtime Diffusers module for the AI Simple Engine `ai-simple-engine`.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
ai_simple_engine_runtime_diffusers/__init__.py,sha256=oeCwDniRRNaeu1i8oSoIGl24B_le57R-tjJfbQIcaTA,101
|
|
2
|
+
ai_simple_engine_runtime_diffusers/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
3
|
+
ai_simple_engine_runtime_diffusers/models/executor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
4
|
+
ai_simple_engine_runtime_diffusers/models/executor/diffusers_latent_diffusion_model_executor.py,sha256=-kVvNpBZLbcWuHiXo6cj3-Y6mVXCn9YIN9S_7opKbj0,3701
|
|
5
|
+
ai_simple_engine_runtime_diffusers/models/executor/noise/__init__.py,sha256=gtBak1TwMesgIsuc56HdYmCwrTjdysO68FO6XzAs-ME,69
|
|
6
|
+
ai_simple_engine_runtime_diffusers/models/executor/noise/pytorch_noise_generator.py,sha256=kFL42BiG2odZp7hAA-7FZakihakOzc8BpYorysDvWx0,508
|
|
7
|
+
ai_simple_engine_runtime_diffusers/models/executor/prompt_embeddings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
ai_simple_engine_runtime_diffusers/models/executor/prompt_embeddings/prompt_embeddings.py,sha256=Y-Krd8UkaaZofSTZoY3Iq1lYetNBBfa_x8ivkplrpBA,321
|
|
9
|
+
ai_simple_engine_runtime_diffusers/models/executor/text_encoder_model_executor_abstract.py,sha256=DdGE73KVvPZOjD1ulk3ffBC3RjAx4MIf9vcsyatgUyE,805
|
|
10
|
+
ai_simple_engine_runtime_diffusers/models/executor/unet_model_executor_abstract.py,sha256=xj5Fw8J1jAwJZRoE5dEnH4Ewf92jIyEh6BJEOHOY1n0,1004
|
|
11
|
+
ai_simple_engine_runtime_diffusers/models/executor/vae_model_executor_abstract.py,sha256=t3j4jYHKbo7bSjMhLLOJYsY2IZm7uQGXXK-c7QDCTv8,1045
|
|
12
|
+
ai_simple_engine_runtime_diffusers/models/loader/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
13
|
+
ai_simple_engine_runtime_diffusers/models/loader/latent_diffusion_model_loader.py,sha256=fROYlfzuJCqmneCm7xr0OY2PRtKjVeL5JCVi1c-SWP0,3057
|
|
14
|
+
ai_simple_engine_runtime_diffusers/models/runtime/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
15
|
+
ai_simple_engine_runtime_diffusers/models/runtime/diffusers_latent_diffusion_model.py,sha256=UTgR2mWW3rDtJXyRATpR_lfcWS6wMugeKHlvKneXTbw,478
|
|
16
|
+
ai_simple_engine_runtime_diffusers/models/runtime/utils.py,sha256=niSrKW6RdZwtRPQVEFCArTyYdGgWWk-rlFSd2ledSPo,347
|
|
17
|
+
ai_simple_engine_runtime_diffusers/models/scheduler/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
18
|
+
ai_simple_engine_runtime_diffusers/models/scheduler/diffusers_scheduler.py,sha256=TiirOtBVX9Op8VeYEjmh3F-rxCiJoEkM6ezyKFjQ_74,896
|
|
19
|
+
ai_simple_engine_runtime_diffusers/models/scheduler/registry/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
20
|
+
ai_simple_engine_runtime_diffusers/models/scheduler/registry/diffusers_scheduler_registry.py,sha256=i9nAd_T30oFFT65awnIw_Z6cDtctmN1NWxkQArXDugY,516
|
|
21
|
+
ai_simple_engine_runtime_diffusers/models/scheduler/utils.py,sha256=VSVeypJkrwaCTRZyYrmscE8g9_pufQ-AlZAhUD_3OKE,1111
|
|
22
|
+
ai_simple_engine_runtime_diffusers/plugin.py,sha256=rPl2BEW9ow6R0KouaFvvBpetG5Opz92E-Rq-6xH5iLc,2101
|
|
23
|
+
ai_simple_engine_runtime_diffusers-0.0.1.dist-info/licenses/LICENSE,sha256=6kbiFSfobTZ7beWiKnHpN902HgBx-Jzgcme0SvKqhKY,1091
|
|
24
|
+
ai_simple_engine_runtime_diffusers-0.0.1.dist-info/METADATA,sha256=p_wUk1JdE8bWcJGNq3m2tDWCbvaN_N2IptU7HbU3vX8,834
|
|
25
|
+
ai_simple_engine_runtime_diffusers-0.0.1.dist-info/WHEEL,sha256=M5asmiAlL6HEcOq52Yi5mmk9KmTVjY2RDPtO4p9DMrc,88
|
|
26
|
+
ai_simple_engine_runtime_diffusers-0.0.1.dist-info/RECORD,,
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright (c) 2018 The Python Packaging Authority
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|