ai-simple-engine-diffusion 0.0.1__tar.gz → 0.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/PKG-INFO +2 -2
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/pyproject.toml +2 -2
- ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/graph/operation/generate_image.py +67 -0
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/diffusion_loop.py → ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/graph/operation/generate_image_from_latents.py +19 -18
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/sample_latents.py +9 -9
- ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/model/executor/abstract/latent_diffusion_model_executor_abstract.py +155 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/info/diffusion_model_info.py +1 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/spec/base.py +2 -2
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/latent_space.py +3 -2
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/decode_latents.py +0 -46
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/encode_prompt.py +0 -46
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/generate_latents.py +0 -74
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/predict_noise.py +0 -53
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/abstract/diffusion_model_executor_abstract.py +0 -66
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/diffusers_stable_diffusion_model_executor.py +0 -43
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/text_encoder_model_executor_abstract.py +0 -27
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/unet_model_executor_abstract.py +0 -31
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/vae_model_executor_abstract.py +0 -39
- ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/scheduler/utils.py +0 -20
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/LICENSE +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/README.md +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/consts.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/encode_image.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/executor/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/executor/abstract/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/info/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/noise/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/noise/noise_generator_abstract.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/prompt_embeddings/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/prompt_embeddings/abstract.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/abstract.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/spec/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/__init__.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/data_type.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/embeddings.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/latents.py +0 -0
- {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/noise_prediction.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ai-simple-engine-diffusion
|
|
3
|
-
Version: 0.0.
|
|
3
|
+
Version: 0.0.2
|
|
4
4
|
Summary: AI Simple Engine Diffusion Module
|
|
5
5
|
License-File: LICENSE
|
|
6
6
|
Author: danialcala94
|
|
@@ -9,7 +9,7 @@ Requires-Python: >=3.10,<3.12
|
|
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
|
10
10
|
Classifier: Programming Language :: Python :: 3.10
|
|
11
11
|
Classifier: Programming Language :: Python :: 3.11
|
|
12
|
-
Requires-Dist: ai-simple-engine (>=0.2.
|
|
12
|
+
Requires-Dist: ai-simple-engine (>=0.2.6,<1.0.0)
|
|
13
13
|
Requires-Dist: ai-simple-engine-common (>=0.0.3,<1.0.0)
|
|
14
14
|
Description-Content-Type: text/markdown
|
|
15
15
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "ai-simple-engine-diffusion"
|
|
3
|
-
version = "0.0.
|
|
3
|
+
version = "0.0.2"
|
|
4
4
|
description = "AI Simple Engine Diffusion Module"
|
|
5
5
|
authors = [
|
|
6
6
|
{name = "danialcala94", email = "danielalcalavalera@gmail.com"}
|
|
@@ -10,7 +10,7 @@ requires-python = ">=3.10,<3.12"
|
|
|
10
10
|
|
|
11
11
|
[tool.poetry.dependencies]
|
|
12
12
|
# Mandatory
|
|
13
|
-
ai-simple-engine = { version = ">=0.2.
|
|
13
|
+
ai-simple-engine = { version = ">=0.2.6,<1.0.0", optional = false }
|
|
14
14
|
ai-simple-engine-common = { version = ">=0.0.3,<1.0.0", optional = false }
|
|
15
15
|
# Optional
|
|
16
16
|
|
ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/graph/operation/generate_image.py
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
from ai_simple_engine_diffusion.graph.operation.sample_latents import SampleLatents
|
|
2
|
+
from ai_simple_engine_diffusion.graph.operation.generate_image_from_latents import GenerateImageFromLatents
|
|
3
|
+
from ai_simple_engine.graph.operation.abstract.composite_operation import CompositeOperation
|
|
4
|
+
from ai_simple_engine.graph.input import Input
|
|
5
|
+
from ai_simple_engine.graph.output import Output
|
|
6
|
+
from ai_simple_engine.types.data_type.base import INT, LOADED_MODEL, STRING, FLOAT, IMAGE
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class GenerateImage(
|
|
10
|
+
CompositeOperation
|
|
11
|
+
):
|
|
12
|
+
"""
|
|
13
|
+
*Composite Operation*
|
|
14
|
+
|
|
15
|
+
Generate an image from the given `prompt`,
|
|
16
|
+
`negative_prompt` and using the `model`.
|
|
17
|
+
|
|
18
|
+
Inputs:
|
|
19
|
+
- `model` (`LOADED_MODEL`)
|
|
20
|
+
- `prompt` (`STRING`)
|
|
21
|
+
- `negative_prompt` (`STRING`)
|
|
22
|
+
- `width` (`INT`)
|
|
23
|
+
- `height` (`INT`)
|
|
24
|
+
- `steps` (`INT`)
|
|
25
|
+
- `guidance_scale` (`FLOAT`)
|
|
26
|
+
- `seed` (`INT`)
|
|
27
|
+
|
|
28
|
+
Outputs:
|
|
29
|
+
- `image` (`IMAGE`)
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
model = Input(LOADED_MODEL)
|
|
33
|
+
prompt = Input(STRING)
|
|
34
|
+
negative_prompt = Input(
|
|
35
|
+
STRING,
|
|
36
|
+
default = ''
|
|
37
|
+
)
|
|
38
|
+
width = Input(INT)
|
|
39
|
+
height = Input(INT)
|
|
40
|
+
steps = Input(INT)
|
|
41
|
+
guidance_scale = Input(FLOAT)
|
|
42
|
+
seed = Input(INT)
|
|
43
|
+
|
|
44
|
+
image = Output(IMAGE)
|
|
45
|
+
|
|
46
|
+
def expand(
|
|
47
|
+
self
|
|
48
|
+
):
|
|
49
|
+
latents = SampleLatents(
|
|
50
|
+
model = self.model,
|
|
51
|
+
width = self.width,
|
|
52
|
+
height = self.height,
|
|
53
|
+
seed = self.seed
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
image = GenerateImageFromLatents(
|
|
57
|
+
model = self.model,
|
|
58
|
+
latents = latents.latents,
|
|
59
|
+
prompt = self.prompt,
|
|
60
|
+
negative_prompt = self.negative_prompt,
|
|
61
|
+
steps = self.steps,
|
|
62
|
+
guidance_scale = self.guidance_scale
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
return {
|
|
66
|
+
'image': image.image
|
|
67
|
+
}
|
|
@@ -1,42 +1,42 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.data_type import
|
|
1
|
+
from ai_simple_engine_diffusion.types.data_type import LATENTS
|
|
2
2
|
from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
|
|
3
|
-
from ai_simple_engine.types.data_type.base import INT, FLOAT, LOADED_MODEL
|
|
4
3
|
from ai_simple_engine.graph.input import Input
|
|
5
4
|
from ai_simple_engine.graph.output import Output
|
|
5
|
+
from ai_simple_engine.types.data_type.base import LOADED_MODEL, STRING, INT, FLOAT, IMAGE
|
|
6
6
|
|
|
7
7
|
|
|
8
|
-
class
|
|
8
|
+
class GenerateImageFromLatents(
|
|
9
9
|
AtomicOperation
|
|
10
10
|
):
|
|
11
11
|
"""
|
|
12
12
|
*Atomic Operation*
|
|
13
13
|
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
This operation encapsulates the whole diffusion
|
|
18
|
-
loop. Internally it repeatedly asks the model
|
|
19
|
-
to predict the noise and delegates the latent
|
|
20
|
-
update to the scheduler.
|
|
14
|
+
Generate an image from an initial latent
|
|
15
|
+
tensor.
|
|
21
16
|
|
|
22
17
|
Inputs:
|
|
23
18
|
- `model` (`LOADED_MODEL`)
|
|
24
19
|
- `latents` (`LATENTS`)
|
|
25
|
-
- `
|
|
26
|
-
- `
|
|
20
|
+
- `prompt` (`STRING`)
|
|
21
|
+
- `negative_prompt` (`STRING`)
|
|
22
|
+
- `steps` (`INT`)
|
|
27
23
|
- `guidance_scale` (`FLOAT`)
|
|
28
24
|
|
|
29
25
|
Outputs:
|
|
30
|
-
- `
|
|
26
|
+
- `image` (`IMAGE`)
|
|
31
27
|
"""
|
|
32
28
|
|
|
33
29
|
model = Input(LOADED_MODEL)
|
|
34
30
|
latents = Input(LATENTS)
|
|
35
|
-
|
|
31
|
+
prompt = Input(STRING)
|
|
32
|
+
negative_prompt = Input(
|
|
33
|
+
STRING,
|
|
34
|
+
default = ''
|
|
35
|
+
)
|
|
36
36
|
steps = Input(INT)
|
|
37
37
|
guidance_scale = Input(FLOAT)
|
|
38
38
|
|
|
39
|
-
|
|
39
|
+
image = Output(IMAGE)
|
|
40
40
|
|
|
41
41
|
async def execute(
|
|
42
42
|
self,
|
|
@@ -44,14 +44,15 @@ class DiffusionLoop(
|
|
|
44
44
|
):
|
|
45
45
|
executor = context.model_executors.resolve(self.model)
|
|
46
46
|
|
|
47
|
-
|
|
47
|
+
image = await executor.generate(
|
|
48
48
|
model = self.model,
|
|
49
49
|
latents = self.latents,
|
|
50
|
-
|
|
50
|
+
prompt = self.prompt,
|
|
51
|
+
negative_prompt = self.negative_prompt,
|
|
51
52
|
steps = self.steps,
|
|
52
53
|
guidance_scale = self.guidance_scale
|
|
53
54
|
)
|
|
54
55
|
|
|
55
56
|
return {
|
|
56
|
-
'
|
|
57
|
+
'image': image
|
|
57
58
|
}
|
|
@@ -3,7 +3,7 @@ from ai_simple_engine_diffusion.noise.noise_generator_abstract import NoiseGener
|
|
|
3
3
|
from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
|
|
4
4
|
from ai_simple_engine.graph.input import Input
|
|
5
5
|
from ai_simple_engine.graph.output import Output
|
|
6
|
-
from ai_simple_engine.types.data_type.base import
|
|
6
|
+
from ai_simple_engine.types.data_type.base import INT, LOADED_MODEL
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class SampleLatents(
|
|
@@ -17,18 +17,18 @@ class SampleLatents(
|
|
|
17
17
|
|
|
18
18
|
Inputs:
|
|
19
19
|
- `model` (`LOADED_MODEL`)
|
|
20
|
-
- `width` (`
|
|
21
|
-
- `height` (`
|
|
22
|
-
- `seed` (`
|
|
20
|
+
- `width` (`INT`)
|
|
21
|
+
- `height` (`INT`)
|
|
22
|
+
- `seed` (`INT`)
|
|
23
23
|
|
|
24
24
|
Outputs:
|
|
25
25
|
- `latents` (`LATENTS`)
|
|
26
26
|
"""
|
|
27
27
|
|
|
28
28
|
model = Input(LOADED_MODEL)
|
|
29
|
-
width = Input(
|
|
30
|
-
height = Input(
|
|
31
|
-
seed = Input(
|
|
29
|
+
width = Input(INT)
|
|
30
|
+
height = Input(INT)
|
|
31
|
+
seed = Input(INT)
|
|
32
32
|
|
|
33
33
|
latents = Output(LATENTS)
|
|
34
34
|
|
|
@@ -36,14 +36,14 @@ class SampleLatents(
|
|
|
36
36
|
self,
|
|
37
37
|
context
|
|
38
38
|
):
|
|
39
|
-
|
|
39
|
+
noise_generator = context.services.get(NoiseGenerator)
|
|
40
40
|
|
|
41
41
|
space = self.model.info.latent_space(
|
|
42
42
|
width = self.width,
|
|
43
43
|
height = self.height
|
|
44
44
|
)
|
|
45
45
|
|
|
46
|
-
latents = await
|
|
46
|
+
latents = await noise_generator.generate(
|
|
47
47
|
space = space,
|
|
48
48
|
seed = self.seed
|
|
49
49
|
)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
2
|
+
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
3
|
+
from ai_simple_engine.execution.execution_context import ExecutionContext
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class LatentDiffusionModelExecutorAbstract(
|
|
8
|
+
ModelExecutor,
|
|
9
|
+
ABC
|
|
10
|
+
):
|
|
11
|
+
"""
|
|
12
|
+
*Abstract class*
|
|
13
|
+
|
|
14
|
+
Abstract executor for latent diffusion models.
|
|
15
|
+
|
|
16
|
+
Defines the common latent diffusion pipeline:
|
|
17
|
+
|
|
18
|
+
1. Encode the prompt.
|
|
19
|
+
2. Generate an initial latent tensor.
|
|
20
|
+
3. Perform the diffusion process.
|
|
21
|
+
4. Decode the final latents into an image.
|
|
22
|
+
|
|
23
|
+
Concrete implementations only need to provide the
|
|
24
|
+
model-specific operations, while the diffusion loop
|
|
25
|
+
is shared.
|
|
26
|
+
|
|
27
|
+
Models that should use this executor:
|
|
28
|
+
- Stable Diffusion 1.x
|
|
29
|
+
- SD 2.x
|
|
30
|
+
- SDXL
|
|
31
|
+
- FLUX
|
|
32
|
+
- SD3
|
|
33
|
+
- AuraFlow
|
|
34
|
+
- HunyuanDiT
|
|
35
|
+
- PixArt-α
|
|
36
|
+
- Sana
|
|
37
|
+
- Wan (imagen)
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
def create_scheduler(
|
|
42
|
+
self,
|
|
43
|
+
context: ExecutionContext,
|
|
44
|
+
model: LoadedModel
|
|
45
|
+
):
|
|
46
|
+
...
|
|
47
|
+
|
|
48
|
+
@abstractmethod
|
|
49
|
+
async def predict_noise(
|
|
50
|
+
self,
|
|
51
|
+
*,
|
|
52
|
+
model: LoadedModel,
|
|
53
|
+
latents,
|
|
54
|
+
embeddings,
|
|
55
|
+
timestep,
|
|
56
|
+
guidance_scale
|
|
57
|
+
):
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
@abstractmethod
|
|
61
|
+
async def encode_prompt(
|
|
62
|
+
self,
|
|
63
|
+
*,
|
|
64
|
+
model: LoadedModel,
|
|
65
|
+
prompt: str,
|
|
66
|
+
negative_prompt: str
|
|
67
|
+
):
|
|
68
|
+
...
|
|
69
|
+
|
|
70
|
+
@abstractmethod
|
|
71
|
+
async def decode_latents(
|
|
72
|
+
self,
|
|
73
|
+
*,
|
|
74
|
+
model: LoadedModel,
|
|
75
|
+
latents
|
|
76
|
+
):
|
|
77
|
+
...
|
|
78
|
+
|
|
79
|
+
async def generate(
|
|
80
|
+
self,
|
|
81
|
+
*,
|
|
82
|
+
context = ExecutionContext,
|
|
83
|
+
model: LoadedModel,
|
|
84
|
+
latents,
|
|
85
|
+
prompt: str,
|
|
86
|
+
negative_prompt: str,
|
|
87
|
+
steps: int,
|
|
88
|
+
guidance_scale: float
|
|
89
|
+
):
|
|
90
|
+
"""
|
|
91
|
+
Generate an image with the given `prompt`,
|
|
92
|
+
`negative_prompt` and `model`.
|
|
93
|
+
"""
|
|
94
|
+
embeddings = await self.encode_prompt(
|
|
95
|
+
model = model,
|
|
96
|
+
prompt = prompt,
|
|
97
|
+
negative_prompt = negative_prompt
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
latents = await self._diffuse(
|
|
101
|
+
context = context,
|
|
102
|
+
model = model,
|
|
103
|
+
latents = latents,
|
|
104
|
+
embeddings = embeddings,
|
|
105
|
+
steps = steps,
|
|
106
|
+
guidance_scale = guidance_scale
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
image = await self.decode_latents(
|
|
110
|
+
model = model,
|
|
111
|
+
latents = latents
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
return image
|
|
115
|
+
|
|
116
|
+
async def _diffuse(
|
|
117
|
+
self,
|
|
118
|
+
*,
|
|
119
|
+
context: ExecutionContext,
|
|
120
|
+
model: LoadedModel,
|
|
121
|
+
latents,
|
|
122
|
+
embeddings,
|
|
123
|
+
steps: int,
|
|
124
|
+
guidance_scale: float
|
|
125
|
+
):
|
|
126
|
+
scheduler = self.create_scheduler(
|
|
127
|
+
context = context,
|
|
128
|
+
model = model
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
scheduler.set_timesteps(steps)
|
|
132
|
+
|
|
133
|
+
for timestep in scheduler.timesteps:
|
|
134
|
+
latent_input = scheduler.scale_model_input(
|
|
135
|
+
latents,
|
|
136
|
+
timestep
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
noise = await self.predict_noise(
|
|
140
|
+
model = model,
|
|
141
|
+
latents = latent_input,
|
|
142
|
+
embeddings = embeddings,
|
|
143
|
+
timestep = timestep,
|
|
144
|
+
guidance_scale = guidance_scale
|
|
145
|
+
)
|
|
146
|
+
|
|
147
|
+
step = scheduler.step(
|
|
148
|
+
model_output = noise,
|
|
149
|
+
timestep = timestep,
|
|
150
|
+
sample = latents
|
|
151
|
+
)
|
|
152
|
+
|
|
153
|
+
latents = step.prev_sample
|
|
154
|
+
|
|
155
|
+
return latents
|
|
@@ -9,7 +9,7 @@ class SchedulerSpec:
|
|
|
9
9
|
be provided to the scheduler when creating it.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
identifier: str
|
|
13
13
|
kwargs: dict[str, object] = field(default_factory = dict)
|
|
14
14
|
|
|
15
15
|
|
|
@@ -17,7 +17,7 @@ class SchedulerSpec:
|
|
|
17
17
|
Example:
|
|
18
18
|
|
|
19
19
|
SchedulerSpec(
|
|
20
|
-
|
|
20
|
+
identifier="euler",
|
|
21
21
|
kwargs={
|
|
22
22
|
"use_karras_sigmas": True
|
|
23
23
|
}
|
ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/decode_latents.py
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.data_type import LATENTS
|
|
2
|
-
from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
|
|
3
|
-
from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
|
|
4
|
-
from ai_simple_engine.graph.input import Input
|
|
5
|
-
from ai_simple_engine.graph.output import Output
|
|
6
|
-
from ai_simple_engine.types.data_type.base import IMAGE, LOADED_MODEL
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class DecodeLatents(
|
|
10
|
-
AtomicOperation
|
|
11
|
-
):
|
|
12
|
-
"""
|
|
13
|
-
*Atomic Operation*
|
|
14
|
-
|
|
15
|
-
Decode the `latents` provided with the `model`
|
|
16
|
-
given, obtaining an `image`.
|
|
17
|
-
|
|
18
|
-
Inputs:
|
|
19
|
-
- `model` (`LOADED_MODEL`)
|
|
20
|
-
- `latents` (`LATENTS`)
|
|
21
|
-
|
|
22
|
-
Outputs:
|
|
23
|
-
- `image` (`IMAGE`)
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
model = Input(LOADED_MODEL)
|
|
27
|
-
latents = Input(LATENTS)
|
|
28
|
-
|
|
29
|
-
image = Output(IMAGE)
|
|
30
|
-
|
|
31
|
-
async def execute(
|
|
32
|
-
self,
|
|
33
|
-
context
|
|
34
|
-
):
|
|
35
|
-
registry = context.services.get(FamilyModelExecutorRegistry)
|
|
36
|
-
|
|
37
|
-
executor = registry.resolve(self.model)
|
|
38
|
-
|
|
39
|
-
image = await executor.decode(
|
|
40
|
-
model = self.model,
|
|
41
|
-
latents = self.latents
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
return {
|
|
45
|
-
'image': image
|
|
46
|
-
}
|
ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/encode_prompt.py
DELETED
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.data_type import EMBEDDINGS
|
|
2
|
-
from ai_simple_engine.types.data_type.base import LOADED_MODEL, STRING
|
|
3
|
-
from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
|
|
4
|
-
from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
|
|
5
|
-
from ai_simple_engine.graph.input import Input
|
|
6
|
-
from ai_simple_engine.graph.output import Output
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class EncodePrompt(
|
|
10
|
-
AtomicOperation
|
|
11
|
-
):
|
|
12
|
-
"""
|
|
13
|
-
*Atomic Operation*
|
|
14
|
-
|
|
15
|
-
Encode the `prompt` given with the `text_encoder`
|
|
16
|
-
also provided.
|
|
17
|
-
|
|
18
|
-
Inputs:
|
|
19
|
-
- `text_encoder` (`LOADED_MODEL`)
|
|
20
|
-
- `prompt` (`STRING`)
|
|
21
|
-
|
|
22
|
-
Outputs:
|
|
23
|
-
- `embeddings` (`EMBEDDINGS`)
|
|
24
|
-
"""
|
|
25
|
-
|
|
26
|
-
text_encoder = Input(LOADED_MODEL)
|
|
27
|
-
prompt = Input(STRING)
|
|
28
|
-
|
|
29
|
-
embeddings = Output(EMBEDDINGS)
|
|
30
|
-
|
|
31
|
-
async def execute(
|
|
32
|
-
self,
|
|
33
|
-
context
|
|
34
|
-
):
|
|
35
|
-
registry = context.services.get(FamilyModelExecutorRegistry)
|
|
36
|
-
|
|
37
|
-
runner = registry.resolve(self.text_encoder)
|
|
38
|
-
|
|
39
|
-
embeddings = await runner.encode(
|
|
40
|
-
model = self.text_encoder,
|
|
41
|
-
prompt = self.prompt
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
return {
|
|
45
|
-
'embeddings': embeddings
|
|
46
|
-
}
|
ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/generate_latents.py
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.data_type import LATENTS
|
|
2
|
-
from ai_simple_engine_diffusion.graph.operation.encode_prompt import EncodePrompt
|
|
3
|
-
from ai_simple_engine_diffusion.graph.operation.sample_latents import SampleLatents
|
|
4
|
-
from ai_simple_engine_diffusion.graph.operation.diffusion_loop import DiffusionLoop
|
|
5
|
-
from ai_simple_engine.graph.operation.abstract.composite_operation import CompositeOperation
|
|
6
|
-
from ai_simple_engine.graph.input import Input
|
|
7
|
-
from ai_simple_engine.graph.output import Output
|
|
8
|
-
from ai_simple_engine.types.data_type.base import INTEGER, LOADED_MODEL, STRING, FLOAT
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class GenerateLatents(
|
|
12
|
-
CompositeOperation
|
|
13
|
-
):
|
|
14
|
-
"""
|
|
15
|
-
*Composite Operation*
|
|
16
|
-
|
|
17
|
-
Generate `latents` for the given `prompt`
|
|
18
|
-
and instructions.
|
|
19
|
-
|
|
20
|
-
Inputs:
|
|
21
|
-
- `model` (`LOADED_MODEL`)
|
|
22
|
-
- `prompt` (`STRING`)
|
|
23
|
-
- `negative_prompt` (`STRING`)
|
|
24
|
-
- `width` (`INTEGER`)
|
|
25
|
-
- `height` (`INTEGER`)
|
|
26
|
-
- `steps` (`INTEGER`)
|
|
27
|
-
- `guidance_scale` (`FLOAT`)
|
|
28
|
-
- `seed` (`INTEGER`)
|
|
29
|
-
|
|
30
|
-
Outputs:
|
|
31
|
-
- `latents` (`LATENTS`)
|
|
32
|
-
"""
|
|
33
|
-
|
|
34
|
-
model = Input(LOADED_MODEL)
|
|
35
|
-
prompt = Input(STRING)
|
|
36
|
-
negative_prompt = Input(
|
|
37
|
-
STRING,
|
|
38
|
-
default = ''
|
|
39
|
-
)
|
|
40
|
-
width = Input(INTEGER)
|
|
41
|
-
height = Input(INTEGER)
|
|
42
|
-
steps = Input(INTEGER)
|
|
43
|
-
guidance_scale = Input(FLOAT)
|
|
44
|
-
seed = Input(INTEGER)
|
|
45
|
-
|
|
46
|
-
latents = Output(LATENTS)
|
|
47
|
-
|
|
48
|
-
def expand(
|
|
49
|
-
self
|
|
50
|
-
):
|
|
51
|
-
embeddings = EncodePrompt(
|
|
52
|
-
model = self.model,
|
|
53
|
-
prompt = self.prompt,
|
|
54
|
-
negative_prompt = self.negative_prompt
|
|
55
|
-
)
|
|
56
|
-
|
|
57
|
-
sampled_latents = SampleLatents(
|
|
58
|
-
model = self.model,
|
|
59
|
-
width = self.width,
|
|
60
|
-
height = self.height,
|
|
61
|
-
seed = self.seed
|
|
62
|
-
)
|
|
63
|
-
|
|
64
|
-
latents = DiffusionLoop(
|
|
65
|
-
model = self.model,
|
|
66
|
-
latents = sampled_latents.latents,
|
|
67
|
-
embeddings = embeddings.embeddings,
|
|
68
|
-
steps = self.steps,
|
|
69
|
-
guidance_scale = self.guidance_scale
|
|
70
|
-
)
|
|
71
|
-
|
|
72
|
-
return {
|
|
73
|
-
'latents': latents.latents
|
|
74
|
-
}
|
ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/predict_noise.py
DELETED
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.data_type import EMBEDDINGS, LATENTS, NOISE_PREDICTION
|
|
2
|
-
from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
|
|
3
|
-
from ai_simple_engine.types.data_type.base import LOADED_MODEL, INT
|
|
4
|
-
from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
|
|
5
|
-
from ai_simple_engine.graph.input import Input
|
|
6
|
-
from ai_simple_engine.graph.output import Output
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class PredictNoise(
|
|
10
|
-
AtomicOperation
|
|
11
|
-
):
|
|
12
|
-
"""
|
|
13
|
-
*Atomic Operation*
|
|
14
|
-
|
|
15
|
-
Predict the `noise` from the given
|
|
16
|
-
`latents` and `embeddings`, using the
|
|
17
|
-
`model` provided.
|
|
18
|
-
|
|
19
|
-
Inputs:
|
|
20
|
-
- `model` (`LOADED_MODEL`)
|
|
21
|
-
- `latents` (`LATENTS`)
|
|
22
|
-
- `embeddings` (`EMBEDDINGS`)
|
|
23
|
-
- `timestep` (`INT`)
|
|
24
|
-
|
|
25
|
-
Outputs:
|
|
26
|
-
- `noise` (`NOISE_PREDICTION`)
|
|
27
|
-
"""
|
|
28
|
-
|
|
29
|
-
model = Input(LOADED_MODEL)
|
|
30
|
-
latents = Input(LATENTS)
|
|
31
|
-
embeddings = Input(EMBEDDINGS)
|
|
32
|
-
timestep = Input(INT)
|
|
33
|
-
|
|
34
|
-
noise = Output(NOISE_PREDICTION)
|
|
35
|
-
|
|
36
|
-
async def execute(
|
|
37
|
-
self,
|
|
38
|
-
context
|
|
39
|
-
):
|
|
40
|
-
registry = context.services.get(FamilyModelExecutorRegistry)
|
|
41
|
-
|
|
42
|
-
executor = registry.resolve(self.model)
|
|
43
|
-
|
|
44
|
-
noise = await executor.predict_noise(
|
|
45
|
-
model = self.model,
|
|
46
|
-
latents = self.latents,
|
|
47
|
-
embeddings = self.embeddings,
|
|
48
|
-
timestep = self.timestep
|
|
49
|
-
)
|
|
50
|
-
|
|
51
|
-
return {
|
|
52
|
-
'noise': noise
|
|
53
|
-
}
|
|
@@ -1,66 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.model.info.diffusion_model_info import DiffusionModelInfo
|
|
2
|
-
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
3
|
-
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
4
|
-
from abc import ABC, abstractmethod
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
class DiffusionModelExecutorAbstract(
|
|
8
|
-
ModelExecutor,
|
|
9
|
-
ABC
|
|
10
|
-
):
|
|
11
|
-
|
|
12
|
-
@abstractmethod
|
|
13
|
-
def info(
|
|
14
|
-
self,
|
|
15
|
-
model: LoadedModel
|
|
16
|
-
) -> DiffusionModelInfo:
|
|
17
|
-
...
|
|
18
|
-
|
|
19
|
-
@abstractmethod
|
|
20
|
-
def create_scheduler(
|
|
21
|
-
self,
|
|
22
|
-
model: LoadedModel
|
|
23
|
-
):
|
|
24
|
-
...
|
|
25
|
-
|
|
26
|
-
@abstractmethod
|
|
27
|
-
async def predict_noise(
|
|
28
|
-
self,
|
|
29
|
-
*,
|
|
30
|
-
model: LoadedModel,
|
|
31
|
-
latents,
|
|
32
|
-
embeddings,
|
|
33
|
-
timestep,
|
|
34
|
-
guidance_scale
|
|
35
|
-
):
|
|
36
|
-
...
|
|
37
|
-
|
|
38
|
-
async def diffuse(
|
|
39
|
-
self,
|
|
40
|
-
*,
|
|
41
|
-
model: LoadedModel,
|
|
42
|
-
latents,
|
|
43
|
-
embeddings,
|
|
44
|
-
steps: int,
|
|
45
|
-
guidance_scale: float
|
|
46
|
-
):
|
|
47
|
-
scheduler = self.create_scheduler(model)
|
|
48
|
-
|
|
49
|
-
scheduler.set_timesteps(steps)
|
|
50
|
-
|
|
51
|
-
for timestep in scheduler.timesteps:
|
|
52
|
-
noise = await self.predict_noise(
|
|
53
|
-
model = model,
|
|
54
|
-
latents = latents,
|
|
55
|
-
embeddings = embeddings,
|
|
56
|
-
timestep = timestep,
|
|
57
|
-
guidance_scale = guidance_scale
|
|
58
|
-
)
|
|
59
|
-
|
|
60
|
-
latents = scheduler.step(
|
|
61
|
-
noise_prediction = noise,
|
|
62
|
-
latents = latents,
|
|
63
|
-
timestep = timestep
|
|
64
|
-
)
|
|
65
|
-
|
|
66
|
-
return latents
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.model.executor.abstract.diffusion_model_executor_abstract import DiffusionModelExecutorAbstract
|
|
2
|
-
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
class DiffusersStableDiffusionModelExecutor(
|
|
6
|
-
DiffusionModelExecutorAbstract
|
|
7
|
-
):
|
|
8
|
-
|
|
9
|
-
def create_scheduler(
|
|
10
|
-
self,
|
|
11
|
-
model: LoadedModel
|
|
12
|
-
):
|
|
13
|
-
# TODO: Move this
|
|
14
|
-
from ai_simple_engine_diffusion.scheduler.utils import SCHEDULERS
|
|
15
|
-
|
|
16
|
-
scheduler_cls = SCHEDULERS[model.info.scheduler]
|
|
17
|
-
|
|
18
|
-
return scheduler_cls.from_config(
|
|
19
|
-
model.model.scheduler.config
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
async def predict_noise(
|
|
23
|
-
self,
|
|
24
|
-
*,
|
|
25
|
-
model: LoadedModel,
|
|
26
|
-
latents,
|
|
27
|
-
embeddings,
|
|
28
|
-
timestep,
|
|
29
|
-
guidance_scale
|
|
30
|
-
):
|
|
31
|
-
pipeline = model.model
|
|
32
|
-
|
|
33
|
-
noise_prediction = pipeline.unet(
|
|
34
|
-
latents,
|
|
35
|
-
timestep,
|
|
36
|
-
encoder_hidden_states = embeddings
|
|
37
|
-
).sample
|
|
38
|
-
|
|
39
|
-
if guidance_scale > 1:
|
|
40
|
-
# TODO: Apply CFG (?)
|
|
41
|
-
pass
|
|
42
|
-
|
|
43
|
-
return noise_prediction
|
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
2
|
-
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
3
|
-
from ai_simple_engine_diffusion.types.data_type import Embeddings
|
|
4
|
-
from abc import ABC, abstractmethod
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
# This will be to perform 'EncodePrompt()'
|
|
8
|
-
class TextEncoderModelExecutorAbstract(
|
|
9
|
-
ModelExecutor,
|
|
10
|
-
ABC
|
|
11
|
-
):
|
|
12
|
-
"""
|
|
13
|
-
*Abstract class*
|
|
14
|
-
|
|
15
|
-
To convert text into embeddings
|
|
16
|
-
"""
|
|
17
|
-
|
|
18
|
-
@abstractmethod
|
|
19
|
-
async def encode(
|
|
20
|
-
self,
|
|
21
|
-
model: LoadedModel,
|
|
22
|
-
prompt: str
|
|
23
|
-
) -> Embeddings:
|
|
24
|
-
"""
|
|
25
|
-
Encode the given prompt into embeddings.
|
|
26
|
-
"""
|
|
27
|
-
...
|
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.latents import Latents
|
|
2
|
-
from ai_simple_engine_diffusion.types.noise_prediction import NoisePrediction
|
|
3
|
-
from ai_simple_engine_diffusion.types.embeddings import Embeddings
|
|
4
|
-
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
5
|
-
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
6
|
-
from abc import ABC, abstractmethod
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class UNetModelExecutorAbstract(
|
|
10
|
-
ModelExecutor,
|
|
11
|
-
ABC
|
|
12
|
-
):
|
|
13
|
-
"""
|
|
14
|
-
*Abstract class*
|
|
15
|
-
|
|
16
|
-
To predict the noise.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
@abstractmethod
|
|
20
|
-
async def predict_noise(
|
|
21
|
-
self,
|
|
22
|
-
model: LoadedModel,
|
|
23
|
-
latents: Latents,
|
|
24
|
-
embeddings: Embeddings,
|
|
25
|
-
timestep: int
|
|
26
|
-
) -> NoisePrediction:
|
|
27
|
-
"""
|
|
28
|
-
Predict the noise residual for the given
|
|
29
|
-
latents at the specified timestep.
|
|
30
|
-
"""
|
|
31
|
-
...
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
from ai_simple_engine_diffusion.types.latents import Latents
|
|
2
|
-
from ai_simple_engine.models.executor.abstract import ModelExecutor
|
|
3
|
-
from ai_simple_engine.models.loaded_model import LoadedModel
|
|
4
|
-
from ai_simple_engine.types.image import Image
|
|
5
|
-
from abc import ABC, abstractmethod
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class VAEModelExecutorAbstract(
|
|
9
|
-
ModelExecutor,
|
|
10
|
-
ABC
|
|
11
|
-
):
|
|
12
|
-
"""
|
|
13
|
-
*Abstract class*
|
|
14
|
-
|
|
15
|
-
To transform images into latents and
|
|
16
|
-
latents into images.
|
|
17
|
-
"""
|
|
18
|
-
|
|
19
|
-
@abstractmethod
|
|
20
|
-
async def encode(
|
|
21
|
-
self,
|
|
22
|
-
model: LoadedModel,
|
|
23
|
-
image: Image
|
|
24
|
-
) -> Latents:
|
|
25
|
-
"""
|
|
26
|
-
Encode an image into latent space.
|
|
27
|
-
"""
|
|
28
|
-
...
|
|
29
|
-
|
|
30
|
-
@abstractmethod
|
|
31
|
-
async def decode(
|
|
32
|
-
self,
|
|
33
|
-
model: LoadedModel,
|
|
34
|
-
latents: Latents
|
|
35
|
-
) -> Image:
|
|
36
|
-
"""
|
|
37
|
-
Decode latent representations into an image.
|
|
38
|
-
"""
|
|
39
|
-
...
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
TODO: Maybe rename and move this module.
|
|
3
|
-
"""
|
|
4
|
-
SCHEDULERS = {
|
|
5
|
-
'ddim': DDIMScheduler,
|
|
6
|
-
'euler': EulerDiscreteScheduler,
|
|
7
|
-
'euler_a': EulerAncestralDiscreteScheduler,
|
|
8
|
-
'dpmpp_2m': DPMSolverMultistepScheduler,
|
|
9
|
-
}
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def get_scheduler_class(
|
|
13
|
-
scheduler: str,
|
|
14
|
-
model: LoadedModel
|
|
15
|
-
) -> object:
|
|
16
|
-
scheduler_cls = SCHEDULERS[model.info.scheduler]
|
|
17
|
-
|
|
18
|
-
return scheduler_cls.from_config(
|
|
19
|
-
model.model.scheduler.config
|
|
20
|
-
)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|