ai-simple-engine-diffusion 0.0.1__tar.gz → 0.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/PKG-INFO +2 -2
  2. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/pyproject.toml +2 -2
  3. ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/graph/operation/generate_image.py +67 -0
  4. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/diffusion_loop.py → ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/graph/operation/generate_image_from_latents.py +19 -18
  5. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/sample_latents.py +9 -9
  6. ai_simple_engine_diffusion-0.0.2/src/ai_simple_engine_diffusion/model/executor/abstract/latent_diffusion_model_executor_abstract.py +155 -0
  7. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/info/diffusion_model_info.py +1 -0
  8. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/spec/base.py +2 -2
  9. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/latent_space.py +3 -2
  10. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/decode_latents.py +0 -46
  11. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/encode_prompt.py +0 -46
  12. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/generate_latents.py +0 -74
  13. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/graph/operation/predict_noise.py +0 -53
  14. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/abstract/diffusion_model_executor_abstract.py +0 -66
  15. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/diffusers_stable_diffusion_model_executor.py +0 -43
  16. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/text_encoder_model_executor_abstract.py +0 -27
  17. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/unet_model_executor_abstract.py +0 -31
  18. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/model/executor/vae_model_executor_abstract.py +0 -39
  19. ai_simple_engine_diffusion-0.0.1/src/ai_simple_engine_diffusion/scheduler/utils.py +0 -20
  20. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/LICENSE +0 -0
  21. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/README.md +0 -0
  22. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/__init__.py +0 -0
  23. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/consts.py +0 -0
  24. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/__init__.py +0 -0
  25. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/__init__.py +0 -0
  26. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/graph/operation/encode_image.py +0 -0
  27. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/__init__.py +0 -0
  28. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/executor/__init__.py +0 -0
  29. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/executor/abstract/__init__.py +0 -0
  30. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/model/info/__init__.py +0 -0
  31. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/noise/__init__.py +0 -0
  32. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/noise/noise_generator_abstract.py +0 -0
  33. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/prompt_embeddings/__init__.py +0 -0
  34. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/prompt_embeddings/abstract.py +0 -0
  35. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/__init__.py +0 -0
  36. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/abstract.py +0 -0
  37. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/scheduler/spec/__init__.py +0 -0
  38. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/__init__.py +0 -0
  39. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/data_type.py +0 -0
  40. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/embeddings.py +0 -0
  41. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/latents.py +0 -0
  42. {ai_simple_engine_diffusion-0.0.1 → ai_simple_engine_diffusion-0.0.2}/src/ai_simple_engine_diffusion/types/noise_prediction.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ai-simple-engine-diffusion
3
- Version: 0.0.1
3
+ Version: 0.0.2
4
4
  Summary: AI Simple Engine Diffusion Module
5
5
  License-File: LICENSE
6
6
  Author: danialcala94
@@ -9,7 +9,7 @@ Requires-Python: >=3.10,<3.12
9
9
  Classifier: Programming Language :: Python :: 3
10
10
  Classifier: Programming Language :: Python :: 3.10
11
11
  Classifier: Programming Language :: Python :: 3.11
12
- Requires-Dist: ai-simple-engine (>=0.2.0,<1.0.0)
12
+ Requires-Dist: ai-simple-engine (>=0.2.6,<1.0.0)
13
13
  Requires-Dist: ai-simple-engine-common (>=0.0.3,<1.0.0)
14
14
  Description-Content-Type: text/markdown
15
15
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "ai-simple-engine-diffusion"
3
- version = "0.0.1"
3
+ version = "0.0.2"
4
4
  description = "AI Simple Engine Diffusion Module"
5
5
  authors = [
6
6
  {name = "danialcala94", email = "danielalcalavalera@gmail.com"}
@@ -10,7 +10,7 @@ requires-python = ">=3.10,<3.12"
10
10
 
11
11
  [tool.poetry.dependencies]
12
12
  # Mandatory
13
- ai-simple-engine = { version = ">=0.2.0,<1.0.0", optional = false }
13
+ ai-simple-engine = { version = ">=0.2.6,<1.0.0", optional = false }
14
14
  ai-simple-engine-common = { version = ">=0.0.3,<1.0.0", optional = false }
15
15
  # Optional
16
16
 
@@ -0,0 +1,67 @@
1
+ from ai_simple_engine_diffusion.graph.operation.sample_latents import SampleLatents
2
+ from ai_simple_engine_diffusion.graph.operation.generate_image_from_latents import GenerateImageFromLatents
3
+ from ai_simple_engine.graph.operation.abstract.composite_operation import CompositeOperation
4
+ from ai_simple_engine.graph.input import Input
5
+ from ai_simple_engine.graph.output import Output
6
+ from ai_simple_engine.types.data_type.base import INT, LOADED_MODEL, STRING, FLOAT, IMAGE
7
+
8
+
9
+ class GenerateImage(
10
+ CompositeOperation
11
+ ):
12
+ """
13
+ *Composite Operation*
14
+
15
+ Generate an image from the given `prompt`,
16
+ `negative_prompt` and using the `model`.
17
+
18
+ Inputs:
19
+ - `model` (`LOADED_MODEL`)
20
+ - `prompt` (`STRING`)
21
+ - `negative_prompt` (`STRING`)
22
+ - `width` (`INT`)
23
+ - `height` (`INT`)
24
+ - `steps` (`INT`)
25
+ - `guidance_scale` (`FLOAT`)
26
+ - `seed` (`INT`)
27
+
28
+ Outputs:
29
+ - `image` (`IMAGE`)
30
+ """
31
+
32
+ model = Input(LOADED_MODEL)
33
+ prompt = Input(STRING)
34
+ negative_prompt = Input(
35
+ STRING,
36
+ default = ''
37
+ )
38
+ width = Input(INT)
39
+ height = Input(INT)
40
+ steps = Input(INT)
41
+ guidance_scale = Input(FLOAT)
42
+ seed = Input(INT)
43
+
44
+ image = Output(IMAGE)
45
+
46
+ def expand(
47
+ self
48
+ ):
49
+ latents = SampleLatents(
50
+ model = self.model,
51
+ width = self.width,
52
+ height = self.height,
53
+ seed = self.seed
54
+ )
55
+
56
+ image = GenerateImageFromLatents(
57
+ model = self.model,
58
+ latents = latents.latents,
59
+ prompt = self.prompt,
60
+ negative_prompt = self.negative_prompt,
61
+ steps = self.steps,
62
+ guidance_scale = self.guidance_scale
63
+ )
64
+
65
+ return {
66
+ 'image': image.image
67
+ }
@@ -1,42 +1,42 @@
1
- from ai_simple_engine_diffusion.types.data_type import EMBEDDINGS, LATENTS
1
+ from ai_simple_engine_diffusion.types.data_type import LATENTS
2
2
  from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
3
- from ai_simple_engine.types.data_type.base import INT, FLOAT, LOADED_MODEL
4
3
  from ai_simple_engine.graph.input import Input
5
4
  from ai_simple_engine.graph.output import Output
5
+ from ai_simple_engine.types.data_type.base import LOADED_MODEL, STRING, INT, FLOAT, IMAGE
6
6
 
7
7
 
8
- class DiffusionLoop(
8
+ class GenerateImageFromLatents(
9
9
  AtomicOperation
10
10
  ):
11
11
  """
12
12
  *Atomic Operation*
13
13
 
14
- Performs the complete denoising process,
15
- iteratively updating the latent tensor.
16
-
17
- This operation encapsulates the whole diffusion
18
- loop. Internally it repeatedly asks the model
19
- to predict the noise and delegates the latent
20
- update to the scheduler.
14
+ Generate an image from an initial latent
15
+ tensor.
21
16
 
22
17
  Inputs:
23
18
  - `model` (`LOADED_MODEL`)
24
19
  - `latents` (`LATENTS`)
25
- - `embeddings` (`EMBEDDINGS`)
26
- - `steps` (`INT)
20
+ - `prompt` (`STRING`)
21
+ - `negative_prompt` (`STRING`)
22
+ - `steps` (`INT`)
27
23
  - `guidance_scale` (`FLOAT`)
28
24
 
29
25
  Outputs:
30
- - `latents_output` (`LATENTS`)
26
+ - `image` (`IMAGE`)
31
27
  """
32
28
 
33
29
  model = Input(LOADED_MODEL)
34
30
  latents = Input(LATENTS)
35
- embeddings = Input(EMBEDDINGS)
31
+ prompt = Input(STRING)
32
+ negative_prompt = Input(
33
+ STRING,
34
+ default = ''
35
+ )
36
36
  steps = Input(INT)
37
37
  guidance_scale = Input(FLOAT)
38
38
 
39
- latents_output = Output(LATENTS)
39
+ image = Output(IMAGE)
40
40
 
41
41
  async def execute(
42
42
  self,
@@ -44,14 +44,15 @@ class DiffusionLoop(
44
44
  ):
45
45
  executor = context.model_executors.resolve(self.model)
46
46
 
47
- latents = await executor.diffuse(
47
+ image = await executor.generate(
48
48
  model = self.model,
49
49
  latents = self.latents,
50
- embeddings = self.embeddings,
50
+ prompt = self.prompt,
51
+ negative_prompt = self.negative_prompt,
51
52
  steps = self.steps,
52
53
  guidance_scale = self.guidance_scale
53
54
  )
54
55
 
55
56
  return {
56
- 'latents': latents
57
+ 'image': image
57
58
  }
@@ -3,7 +3,7 @@ from ai_simple_engine_diffusion.noise.noise_generator_abstract import NoiseGener
3
3
  from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
4
4
  from ai_simple_engine.graph.input import Input
5
5
  from ai_simple_engine.graph.output import Output
6
- from ai_simple_engine.types.data_type.base import INTEGER, LOADED_MODEL
6
+ from ai_simple_engine.types.data_type.base import INT, LOADED_MODEL
7
7
 
8
8
 
9
9
  class SampleLatents(
@@ -17,18 +17,18 @@ class SampleLatents(
17
17
 
18
18
  Inputs:
19
19
  - `model` (`LOADED_MODEL`)
20
- - `width` (`INTEGER`)
21
- - `height` (`INTEGER`)
22
- - `seed` (`INTEGER`)
20
+ - `width` (`INT`)
21
+ - `height` (`INT`)
22
+ - `seed` (`INT`)
23
23
 
24
24
  Outputs:
25
25
  - `latents` (`LATENTS`)
26
26
  """
27
27
 
28
28
  model = Input(LOADED_MODEL)
29
- width = Input(INTEGER)
30
- height = Input(INTEGER)
31
- seed = Input(INTEGER)
29
+ width = Input(INT)
30
+ height = Input(INT)
31
+ seed = Input(INT)
32
32
 
33
33
  latents = Output(LATENTS)
34
34
 
@@ -36,14 +36,14 @@ class SampleLatents(
36
36
  self,
37
37
  context
38
38
  ):
39
- generator = context.services.get(NoiseGenerator)
39
+ noise_generator = context.services.get(NoiseGenerator)
40
40
 
41
41
  space = self.model.info.latent_space(
42
42
  width = self.width,
43
43
  height = self.height
44
44
  )
45
45
 
46
- latents = await generator.generate(
46
+ latents = await noise_generator.generate(
47
47
  space = space,
48
48
  seed = self.seed
49
49
  )
@@ -0,0 +1,155 @@
1
+ from ai_simple_engine.models.executor.abstract import ModelExecutor
2
+ from ai_simple_engine.models.loaded_model import LoadedModel
3
+ from ai_simple_engine.execution.execution_context import ExecutionContext
4
+ from abc import ABC, abstractmethod
5
+
6
+
7
+ class LatentDiffusionModelExecutorAbstract(
8
+ ModelExecutor,
9
+ ABC
10
+ ):
11
+ """
12
+ *Abstract class*
13
+
14
+ Abstract executor for latent diffusion models.
15
+
16
+ Defines the common latent diffusion pipeline:
17
+
18
+ 1. Encode the prompt.
19
+ 2. Generate an initial latent tensor.
20
+ 3. Perform the diffusion process.
21
+ 4. Decode the final latents into an image.
22
+
23
+ Concrete implementations only need to provide the
24
+ model-specific operations, while the diffusion loop
25
+ is shared.
26
+
27
+ Models that should use this executor:
28
+ - Stable Diffusion 1.x
29
+ - SD 2.x
30
+ - SDXL
31
+ - FLUX
32
+ - SD3
33
+ - AuraFlow
34
+ - HunyuanDiT
35
+ - PixArt-α
36
+ - Sana
37
+ - Wan (imagen)
38
+ """
39
+
40
+ @abstractmethod
41
+ def create_scheduler(
42
+ self,
43
+ context: ExecutionContext,
44
+ model: LoadedModel
45
+ ):
46
+ ...
47
+
48
+ @abstractmethod
49
+ async def predict_noise(
50
+ self,
51
+ *,
52
+ model: LoadedModel,
53
+ latents,
54
+ embeddings,
55
+ timestep,
56
+ guidance_scale
57
+ ):
58
+ ...
59
+
60
+ @abstractmethod
61
+ async def encode_prompt(
62
+ self,
63
+ *,
64
+ model: LoadedModel,
65
+ prompt: str,
66
+ negative_prompt: str
67
+ ):
68
+ ...
69
+
70
+ @abstractmethod
71
+ async def decode_latents(
72
+ self,
73
+ *,
74
+ model: LoadedModel,
75
+ latents
76
+ ):
77
+ ...
78
+
79
+ async def generate(
80
+ self,
81
+ *,
82
+ context = ExecutionContext,
83
+ model: LoadedModel,
84
+ latents,
85
+ prompt: str,
86
+ negative_prompt: str,
87
+ steps: int,
88
+ guidance_scale: float
89
+ ):
90
+ """
91
+ Generate an image with the given `prompt`,
92
+ `negative_prompt` and `model`.
93
+ """
94
+ embeddings = await self.encode_prompt(
95
+ model = model,
96
+ prompt = prompt,
97
+ negative_prompt = negative_prompt
98
+ )
99
+
100
+ latents = await self._diffuse(
101
+ context = context,
102
+ model = model,
103
+ latents = latents,
104
+ embeddings = embeddings,
105
+ steps = steps,
106
+ guidance_scale = guidance_scale
107
+ )
108
+
109
+ image = await self.decode_latents(
110
+ model = model,
111
+ latents = latents
112
+ )
113
+
114
+ return image
115
+
116
+ async def _diffuse(
117
+ self,
118
+ *,
119
+ context: ExecutionContext,
120
+ model: LoadedModel,
121
+ latents,
122
+ embeddings,
123
+ steps: int,
124
+ guidance_scale: float
125
+ ):
126
+ scheduler = self.create_scheduler(
127
+ context = context,
128
+ model = model
129
+ )
130
+
131
+ scheduler.set_timesteps(steps)
132
+
133
+ for timestep in scheduler.timesteps:
134
+ latent_input = scheduler.scale_model_input(
135
+ latents,
136
+ timestep
137
+ )
138
+
139
+ noise = await self.predict_noise(
140
+ model = model,
141
+ latents = latent_input,
142
+ embeddings = embeddings,
143
+ timestep = timestep,
144
+ guidance_scale = guidance_scale
145
+ )
146
+
147
+ step = scheduler.step(
148
+ model_output = noise,
149
+ timestep = timestep,
150
+ sample = latents
151
+ )
152
+
153
+ latents = step.prev_sample
154
+
155
+ return latents
@@ -25,6 +25,7 @@ class DiffusionModelInfo(
25
25
  height: int
26
26
  ) -> LatentSpace:
27
27
  return LatentSpace(
28
+ batch_size = 1,
28
29
  width = width // self.vae_scale_factor,
29
30
  height = height // self.vae_scale_factor,
30
31
  channels = self.latent_channels
@@ -9,7 +9,7 @@ class SchedulerSpec:
9
9
  be provided to the scheduler when creating it.
10
10
  """
11
11
 
12
- name: str
12
+ identifier: str
13
13
  kwargs: dict[str, object] = field(default_factory = dict)
14
14
 
15
15
 
@@ -17,7 +17,7 @@ class SchedulerSpec:
17
17
  Example:
18
18
 
19
19
  SchedulerSpec(
20
- name="euler",
20
+ identifier="euler",
21
21
  kwargs={
22
22
  "use_karras_sigmas": True
23
23
  }
@@ -4,6 +4,7 @@ from dataclasses import dataclass
4
4
  @dataclass(frozen = True)
5
5
  class LatentSpace:
6
6
 
7
+ batch_size: int = 1
8
+ channels: int
7
9
  width: int
8
- height: int
9
- channels: int
10
+ height: int
@@ -1,46 +0,0 @@
1
- from ai_simple_engine_diffusion.types.data_type import LATENTS
2
- from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
3
- from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
4
- from ai_simple_engine.graph.input import Input
5
- from ai_simple_engine.graph.output import Output
6
- from ai_simple_engine.types.data_type.base import IMAGE, LOADED_MODEL
7
-
8
-
9
- class DecodeLatents(
10
- AtomicOperation
11
- ):
12
- """
13
- *Atomic Operation*
14
-
15
- Decode the `latents` provided with the `model`
16
- given, obtaining an `image`.
17
-
18
- Inputs:
19
- - `model` (`LOADED_MODEL`)
20
- - `latents` (`LATENTS`)
21
-
22
- Outputs:
23
- - `image` (`IMAGE`)
24
- """
25
-
26
- model = Input(LOADED_MODEL)
27
- latents = Input(LATENTS)
28
-
29
- image = Output(IMAGE)
30
-
31
- async def execute(
32
- self,
33
- context
34
- ):
35
- registry = context.services.get(FamilyModelExecutorRegistry)
36
-
37
- executor = registry.resolve(self.model)
38
-
39
- image = await executor.decode(
40
- model = self.model,
41
- latents = self.latents
42
- )
43
-
44
- return {
45
- 'image': image
46
- }
@@ -1,46 +0,0 @@
1
- from ai_simple_engine_diffusion.types.data_type import EMBEDDINGS
2
- from ai_simple_engine.types.data_type.base import LOADED_MODEL, STRING
3
- from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
4
- from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
5
- from ai_simple_engine.graph.input import Input
6
- from ai_simple_engine.graph.output import Output
7
-
8
-
9
- class EncodePrompt(
10
- AtomicOperation
11
- ):
12
- """
13
- *Atomic Operation*
14
-
15
- Encode the `prompt` given with the `text_encoder`
16
- also provided.
17
-
18
- Inputs:
19
- - `text_encoder` (`LOADED_MODEL`)
20
- - `prompt` (`STRING`)
21
-
22
- Outputs:
23
- - `embeddings` (`EMBEDDINGS`)
24
- """
25
-
26
- text_encoder = Input(LOADED_MODEL)
27
- prompt = Input(STRING)
28
-
29
- embeddings = Output(EMBEDDINGS)
30
-
31
- async def execute(
32
- self,
33
- context
34
- ):
35
- registry = context.services.get(FamilyModelExecutorRegistry)
36
-
37
- runner = registry.resolve(self.text_encoder)
38
-
39
- embeddings = await runner.encode(
40
- model = self.text_encoder,
41
- prompt = self.prompt
42
- )
43
-
44
- return {
45
- 'embeddings': embeddings
46
- }
@@ -1,74 +0,0 @@
1
- from ai_simple_engine_diffusion.types.data_type import LATENTS
2
- from ai_simple_engine_diffusion.graph.operation.encode_prompt import EncodePrompt
3
- from ai_simple_engine_diffusion.graph.operation.sample_latents import SampleLatents
4
- from ai_simple_engine_diffusion.graph.operation.diffusion_loop import DiffusionLoop
5
- from ai_simple_engine.graph.operation.abstract.composite_operation import CompositeOperation
6
- from ai_simple_engine.graph.input import Input
7
- from ai_simple_engine.graph.output import Output
8
- from ai_simple_engine.types.data_type.base import INTEGER, LOADED_MODEL, STRING, FLOAT
9
-
10
-
11
- class GenerateLatents(
12
- CompositeOperation
13
- ):
14
- """
15
- *Composite Operation*
16
-
17
- Generate `latents` for the given `prompt`
18
- and instructions.
19
-
20
- Inputs:
21
- - `model` (`LOADED_MODEL`)
22
- - `prompt` (`STRING`)
23
- - `negative_prompt` (`STRING`)
24
- - `width` (`INTEGER`)
25
- - `height` (`INTEGER`)
26
- - `steps` (`INTEGER`)
27
- - `guidance_scale` (`FLOAT`)
28
- - `seed` (`INTEGER`)
29
-
30
- Outputs:
31
- - `latents` (`LATENTS`)
32
- """
33
-
34
- model = Input(LOADED_MODEL)
35
- prompt = Input(STRING)
36
- negative_prompt = Input(
37
- STRING,
38
- default = ''
39
- )
40
- width = Input(INTEGER)
41
- height = Input(INTEGER)
42
- steps = Input(INTEGER)
43
- guidance_scale = Input(FLOAT)
44
- seed = Input(INTEGER)
45
-
46
- latents = Output(LATENTS)
47
-
48
- def expand(
49
- self
50
- ):
51
- embeddings = EncodePrompt(
52
- model = self.model,
53
- prompt = self.prompt,
54
- negative_prompt = self.negative_prompt
55
- )
56
-
57
- sampled_latents = SampleLatents(
58
- model = self.model,
59
- width = self.width,
60
- height = self.height,
61
- seed = self.seed
62
- )
63
-
64
- latents = DiffusionLoop(
65
- model = self.model,
66
- latents = sampled_latents.latents,
67
- embeddings = embeddings.embeddings,
68
- steps = self.steps,
69
- guidance_scale = self.guidance_scale
70
- )
71
-
72
- return {
73
- 'latents': latents.latents
74
- }
@@ -1,53 +0,0 @@
1
- from ai_simple_engine_diffusion.types.data_type import EMBEDDINGS, LATENTS, NOISE_PREDICTION
2
- from ai_simple_engine.models.executor.registry.family_model_executor_registry import FamilyModelExecutorRegistry
3
- from ai_simple_engine.types.data_type.base import LOADED_MODEL, INT
4
- from ai_simple_engine.graph.operation.abstract.atomic_operation import AtomicOperation
5
- from ai_simple_engine.graph.input import Input
6
- from ai_simple_engine.graph.output import Output
7
-
8
-
9
- class PredictNoise(
10
- AtomicOperation
11
- ):
12
- """
13
- *Atomic Operation*
14
-
15
- Predict the `noise` from the given
16
- `latents` and `embeddings`, using the
17
- `model` provided.
18
-
19
- Inputs:
20
- - `model` (`LOADED_MODEL`)
21
- - `latents` (`LATENTS`)
22
- - `embeddings` (`EMBEDDINGS`)
23
- - `timestep` (`INT`)
24
-
25
- Outputs:
26
- - `noise` (`NOISE_PREDICTION`)
27
- """
28
-
29
- model = Input(LOADED_MODEL)
30
- latents = Input(LATENTS)
31
- embeddings = Input(EMBEDDINGS)
32
- timestep = Input(INT)
33
-
34
- noise = Output(NOISE_PREDICTION)
35
-
36
- async def execute(
37
- self,
38
- context
39
- ):
40
- registry = context.services.get(FamilyModelExecutorRegistry)
41
-
42
- executor = registry.resolve(self.model)
43
-
44
- noise = await executor.predict_noise(
45
- model = self.model,
46
- latents = self.latents,
47
- embeddings = self.embeddings,
48
- timestep = self.timestep
49
- )
50
-
51
- return {
52
- 'noise': noise
53
- }
@@ -1,66 +0,0 @@
1
- from ai_simple_engine_diffusion.model.info.diffusion_model_info import DiffusionModelInfo
2
- from ai_simple_engine.models.executor.abstract import ModelExecutor
3
- from ai_simple_engine.models.loaded_model import LoadedModel
4
- from abc import ABC, abstractmethod
5
-
6
-
7
- class DiffusionModelExecutorAbstract(
8
- ModelExecutor,
9
- ABC
10
- ):
11
-
12
- @abstractmethod
13
- def info(
14
- self,
15
- model: LoadedModel
16
- ) -> DiffusionModelInfo:
17
- ...
18
-
19
- @abstractmethod
20
- def create_scheduler(
21
- self,
22
- model: LoadedModel
23
- ):
24
- ...
25
-
26
- @abstractmethod
27
- async def predict_noise(
28
- self,
29
- *,
30
- model: LoadedModel,
31
- latents,
32
- embeddings,
33
- timestep,
34
- guidance_scale
35
- ):
36
- ...
37
-
38
- async def diffuse(
39
- self,
40
- *,
41
- model: LoadedModel,
42
- latents,
43
- embeddings,
44
- steps: int,
45
- guidance_scale: float
46
- ):
47
- scheduler = self.create_scheduler(model)
48
-
49
- scheduler.set_timesteps(steps)
50
-
51
- for timestep in scheduler.timesteps:
52
- noise = await self.predict_noise(
53
- model = model,
54
- latents = latents,
55
- embeddings = embeddings,
56
- timestep = timestep,
57
- guidance_scale = guidance_scale
58
- )
59
-
60
- latents = scheduler.step(
61
- noise_prediction = noise,
62
- latents = latents,
63
- timestep = timestep
64
- )
65
-
66
- return latents
@@ -1,43 +0,0 @@
1
- from ai_simple_engine_diffusion.model.executor.abstract.diffusion_model_executor_abstract import DiffusionModelExecutorAbstract
2
- from ai_simple_engine.models.loaded_model import LoadedModel
3
-
4
-
5
- class DiffusersStableDiffusionModelExecutor(
6
- DiffusionModelExecutorAbstract
7
- ):
8
-
9
- def create_scheduler(
10
- self,
11
- model: LoadedModel
12
- ):
13
- # TODO: Move this
14
- from ai_simple_engine_diffusion.scheduler.utils import SCHEDULERS
15
-
16
- scheduler_cls = SCHEDULERS[model.info.scheduler]
17
-
18
- return scheduler_cls.from_config(
19
- model.model.scheduler.config
20
- )
21
-
22
- async def predict_noise(
23
- self,
24
- *,
25
- model: LoadedModel,
26
- latents,
27
- embeddings,
28
- timestep,
29
- guidance_scale
30
- ):
31
- pipeline = model.model
32
-
33
- noise_prediction = pipeline.unet(
34
- latents,
35
- timestep,
36
- encoder_hidden_states = embeddings
37
- ).sample
38
-
39
- if guidance_scale > 1:
40
- # TODO: Apply CFG (?)
41
- pass
42
-
43
- return noise_prediction
@@ -1,27 +0,0 @@
1
- from ai_simple_engine.models.executor.abstract import ModelExecutor
2
- from ai_simple_engine.models.loaded_model import LoadedModel
3
- from ai_simple_engine_diffusion.types.data_type import Embeddings
4
- from abc import ABC, abstractmethod
5
-
6
-
7
- # This will be to perform 'EncodePrompt()'
8
- class TextEncoderModelExecutorAbstract(
9
- ModelExecutor,
10
- ABC
11
- ):
12
- """
13
- *Abstract class*
14
-
15
- To convert text into embeddings
16
- """
17
-
18
- @abstractmethod
19
- async def encode(
20
- self,
21
- model: LoadedModel,
22
- prompt: str
23
- ) -> Embeddings:
24
- """
25
- Encode the given prompt into embeddings.
26
- """
27
- ...
@@ -1,31 +0,0 @@
1
- from ai_simple_engine_diffusion.types.latents import Latents
2
- from ai_simple_engine_diffusion.types.noise_prediction import NoisePrediction
3
- from ai_simple_engine_diffusion.types.embeddings import Embeddings
4
- from ai_simple_engine.models.executor.abstract import ModelExecutor
5
- from ai_simple_engine.models.loaded_model import LoadedModel
6
- from abc import ABC, abstractmethod
7
-
8
-
9
- class UNetModelExecutorAbstract(
10
- ModelExecutor,
11
- ABC
12
- ):
13
- """
14
- *Abstract class*
15
-
16
- To predict the noise.
17
- """
18
-
19
- @abstractmethod
20
- async def predict_noise(
21
- self,
22
- model: LoadedModel,
23
- latents: Latents,
24
- embeddings: Embeddings,
25
- timestep: int
26
- ) -> NoisePrediction:
27
- """
28
- Predict the noise residual for the given
29
- latents at the specified timestep.
30
- """
31
- ...
@@ -1,39 +0,0 @@
1
- from ai_simple_engine_diffusion.types.latents import Latents
2
- from ai_simple_engine.models.executor.abstract import ModelExecutor
3
- from ai_simple_engine.models.loaded_model import LoadedModel
4
- from ai_simple_engine.types.image import Image
5
- from abc import ABC, abstractmethod
6
-
7
-
8
- class VAEModelExecutorAbstract(
9
- ModelExecutor,
10
- ABC
11
- ):
12
- """
13
- *Abstract class*
14
-
15
- To transform images into latents and
16
- latents into images.
17
- """
18
-
19
- @abstractmethod
20
- async def encode(
21
- self,
22
- model: LoadedModel,
23
- image: Image
24
- ) -> Latents:
25
- """
26
- Encode an image into latent space.
27
- """
28
- ...
29
-
30
- @abstractmethod
31
- async def decode(
32
- self,
33
- model: LoadedModel,
34
- latents: Latents
35
- ) -> Image:
36
- """
37
- Decode latent representations into an image.
38
- """
39
- ...
@@ -1,20 +0,0 @@
1
- """
2
- TODO: Maybe rename and move this module.
3
- """
4
- SCHEDULERS = {
5
- 'ddim': DDIMScheduler,
6
- 'euler': EulerDiscreteScheduler,
7
- 'euler_a': EulerAncestralDiscreteScheduler,
8
- 'dpmpp_2m': DPMSolverMultistepScheduler,
9
- }
10
-
11
-
12
- def get_scheduler_class(
13
- scheduler: str,
14
- model: LoadedModel
15
- ) -> object:
16
- scheduler_cls = SCHEDULERS[model.info.scheduler]
17
-
18
- return scheduler_cls.from_config(
19
- model.model.scheduler.config
20
- )