diffusion-model-framework 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_model_framework-0.1.1/PKG-INFO +84 -0
- diffusion_model_framework-0.1.1/README.md +61 -0
- diffusion_model_framework-0.1.1/diffusion_models/__init__.py +1 -0
- diffusion_model_framework-0.1.1/diffusion_models/diffusion_inference.py +127 -0
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/diffusion_trainer.py +46 -11
- diffusion_model_framework-0.1.1/diffusion_models/gaussian_diffusion/base_diffuser.py +78 -0
- diffusion_model_framework-0.1.1/diffusion_models/gaussian_diffusion/beta_schedulers.py +263 -0
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/gaussian_diffusion/gaussian_diffuser.py +64 -4
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/models/SimpleUnet.py +6 -4
- diffusion_model_framework-0.1.1/diffusion_models/models/base_diffusion_model.py +99 -0
- diffusion_model_framework-0.1.1/diffusion_models/utils/__init__.py +0 -0
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/utils/schemas.py +39 -5
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/utils/tensorboard.py +21 -0
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/pyproject.toml +1 -1
- diffusion_model_framework-0.1.0/PKG-INFO +0 -43
- diffusion_model_framework-0.1.0/README.md +0 -20
- diffusion_model_framework-0.1.0/diffusion_models/__init__.py +0 -1
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/base_diffuser.py +0 -28
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/beta_schedulers.py +0 -136
- {diffusion_model_framework-0.1.0 → diffusion_model_framework-0.1.1}/diffusion_models/gaussian_diffusion/__init__.py +0 -0
- {diffusion_model_framework-0.1.0/diffusion_models/utils → diffusion_model_framework-0.1.1/diffusion_models/models}/__init__.py +0 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: diffusion-model-framework
|
|
3
|
+
Version: 0.1.1
|
|
4
|
+
Summary:
|
|
5
|
+
Home-page: https://github.com/sachahu1/Diffusion-Model-Framework
|
|
6
|
+
Author: Sacha Hu
|
|
7
|
+
Author-email: sachahu@mindsparkai.com
|
|
8
|
+
Requires-Python: >=3.9,<3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Requires-Dist: numpy (>=1.26.4,<2.0.0)
|
|
14
|
+
Requires-Dist: pillow-simd (>=9.0.0.post1,<10.0.0)
|
|
15
|
+
Requires-Dist: tensorboard (>=2.17.0,<3.0.0)
|
|
16
|
+
Requires-Dist: torch (>=2.4.0,<3.0.0)
|
|
17
|
+
Requires-Dist: torchvision (>=0.19.0,<0.20.0)
|
|
18
|
+
Requires-Dist: tqdm (>=4.66.2,<5.0.0)
|
|
19
|
+
Project-URL: Documentation, https://www.sachahu.com/docs/diffusion-model-framework/latest/index.html
|
|
20
|
+
Project-URL: Repository, https://github.com/sachahu1/Diffusion-Model-Framework
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Diffusion Model Playground
|
|
24
|
+
|
|
25
|
+

|
|
26
|
+

|
|
27
|
+

|
|
28
|
+

|
|
29
|
+
|
|
30
|
+
This is a simplified framework to train and run diffusion models.
|
|
31
|
+

|
|
32
|
+
|
|
33
|
+
## Getting Started
|
|
34
|
+
To get started, simply install the python package from PyPI:
|
|
35
|
+
```shell
|
|
36
|
+
pip install diffusion-model-framework
|
|
37
|
+
```
|
|
38
|
+
Then, you're ready to start setting up your own training process. For help
|
|
39
|
+
getting started with that, have a look at some of the [examples](https://github.com/sachahu1/Diffusion-Model-Framework/blob/main/examples).
|
|
40
|
+
|
|
41
|
+
### Installing the package
|
|
42
|
+
Thanks to poetry, installing this package is very simple and can be done in a single command. Simply run:
|
|
43
|
+
```shell
|
|
44
|
+
poetry install
|
|
45
|
+
```
|
|
46
|
+
That's it, the package is installed. Move to the next section to learn how to use this package.
|
|
47
|
+
|
|
48
|
+
## Building from source
|
|
49
|
+
### Installing Poetry
|
|
50
|
+
This tool uses poetry. If you already have poetry installed,
|
|
51
|
+
please skip to the next section. Otherwise, let's first setup poetry.
|
|
52
|
+
|
|
53
|
+
To install poetry, simply run this command:
|
|
54
|
+
```shell
|
|
55
|
+
curl -sSL https://install.python-poetry.org | python3 -
|
|
56
|
+
```
|
|
57
|
+
You can find out more about poetry installation [here](https://python-poetry.org/docs/master/#installation).
|
|
58
|
+
|
|
59
|
+
That's it, poetry is set up.
|
|
60
|
+
### Installing and building locally
|
|
61
|
+
Installing the package via poetry is very simple. Simply run:
|
|
62
|
+
```shell
|
|
63
|
+
poetry install
|
|
64
|
+
```
|
|
65
|
+
You can now start using the package to train your diffusion model. For example, try running:
|
|
66
|
+
```shell
|
|
67
|
+
poetry run python3 examples/train_model.py
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Building a wheel is also possible via:
|
|
71
|
+
```shell
|
|
72
|
+
poetry build
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
## Building the documentation
|
|
76
|
+
First you will need to install the dependency group used for documentation.
|
|
77
|
+
```shell
|
|
78
|
+
poetry install --with documentation
|
|
79
|
+
```
|
|
80
|
+
Now you can build the documentation using Sphinx by running:
|
|
81
|
+
```shell
|
|
82
|
+
poetry run sphinx-build -M html docs/source/ docs/build
|
|
83
|
+
```
|
|
84
|
+
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Diffusion Model Playground
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
This is a simplified framework to train and run diffusion models.
|
|
9
|
+

|
|
10
|
+
|
|
11
|
+
## Getting Started
|
|
12
|
+
To get started, simply install the python package from PyPI:
|
|
13
|
+
```shell
|
|
14
|
+
pip install diffusion-model-framework
|
|
15
|
+
```
|
|
16
|
+
Then, you're ready to start setting up your own training process. For help
|
|
17
|
+
getting started with that, have a look at some of the [examples](https://github.com/sachahu1/Diffusion-Model-Framework/blob/main/examples).
|
|
18
|
+
|
|
19
|
+
### Installing the package
|
|
20
|
+
Thanks to poetry, installing this package is very simple and can be done in a single command. Simply run:
|
|
21
|
+
```shell
|
|
22
|
+
poetry install
|
|
23
|
+
```
|
|
24
|
+
That's it, the package is installed. Move to the next section to learn how to use this package.
|
|
25
|
+
|
|
26
|
+
## Building from source
|
|
27
|
+
### Installing Poetry
|
|
28
|
+
This tool uses poetry. If you already have poetry installed,
|
|
29
|
+
please skip to the next section. Otherwise, let's first setup poetry.
|
|
30
|
+
|
|
31
|
+
To install poetry, simply run this command:
|
|
32
|
+
```shell
|
|
33
|
+
curl -sSL https://install.python-poetry.org | python3 -
|
|
34
|
+
```
|
|
35
|
+
You can find out more about poetry installation [here](https://python-poetry.org/docs/master/#installation).
|
|
36
|
+
|
|
37
|
+
That's it, poetry is set up.
|
|
38
|
+
### Installing and building locally
|
|
39
|
+
Installing the package via poetry is very simple. Simply run:
|
|
40
|
+
```shell
|
|
41
|
+
poetry install
|
|
42
|
+
```
|
|
43
|
+
You can now start using the package to train your diffusion model. For example, try running:
|
|
44
|
+
```shell
|
|
45
|
+
poetry run python3 examples/train_model.py
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Building a wheel is also possible via:
|
|
49
|
+
```shell
|
|
50
|
+
poetry build
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Building the documentation
|
|
54
|
+
First you will need to install the dependency group used for documentation.
|
|
55
|
+
```shell
|
|
56
|
+
poetry install --with documentation
|
|
57
|
+
```
|
|
58
|
+
Now you can build the documentation using Sphinx by running:
|
|
59
|
+
```shell
|
|
60
|
+
poetry run sphinx-build -M html docs/source/ docs/build
|
|
61
|
+
```
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.1"
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
from typing import Callable, Tuple
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from PIL.Image import Image
|
|
6
|
+
from torchvision import transforms
|
|
7
|
+
from torchvision.utils import make_grid
|
|
8
|
+
|
|
9
|
+
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class DiffusionInference:
|
|
13
|
+
def __init__(
|
|
14
|
+
self,
|
|
15
|
+
model: BaseDiffusionModel,
|
|
16
|
+
reverse_transforms: Callable = lambda x: x,
|
|
17
|
+
image_shape: Tuple[int, int] = (3, 64),
|
|
18
|
+
device: str = "cuda",
|
|
19
|
+
):
|
|
20
|
+
"""A diffusion inference framework.
|
|
21
|
+
|
|
22
|
+
This is a simplified inference framework to easily start using your
|
|
23
|
+
diffusion model.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
model: The trained diffusion model.
|
|
27
|
+
reverse_transforms: A set of reverse transforms.
|
|
28
|
+
image_shape: The shape of the image to produce. This is a tuple where the
|
|
29
|
+
first value is the number of channels and the second is the size of the
|
|
30
|
+
image. Images are expected to be square.
|
|
31
|
+
device: The device to run the inference on.
|
|
32
|
+
"""
|
|
33
|
+
self.image_channels = image_shape[0]
|
|
34
|
+
"""The number of channels of the image."""
|
|
35
|
+
self.image_size = image_shape[1]
|
|
36
|
+
"""The size of the image."""
|
|
37
|
+
|
|
38
|
+
self.model = model.to(device)
|
|
39
|
+
"""The trained diffusion model."""
|
|
40
|
+
|
|
41
|
+
self.reverse_transforms = reverse_transforms
|
|
42
|
+
"""The set of reverse transforms."""
|
|
43
|
+
self.device = device
|
|
44
|
+
"""The device to run the inference on."""
|
|
45
|
+
|
|
46
|
+
def _visualise_images(self, denoised_images: torch.Tensor):
|
|
47
|
+
reverse_transformed_images = self.reverse_transforms(denoised_images)
|
|
48
|
+
image_grid = make_grid(reverse_transformed_images, nrow=5)
|
|
49
|
+
pil_images = transforms.ToPILImage()(image_grid)
|
|
50
|
+
return pil_images
|
|
51
|
+
|
|
52
|
+
def generate(
|
|
53
|
+
self, number_of_images: int, save_gif: bool = False
|
|
54
|
+
) -> Image:
|
|
55
|
+
"""Generate a batch of images.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
number_of_images: The number of images to generate.
|
|
59
|
+
save_gif: Whether to save the generation process as a GIF.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
A PIL image of the generated images stacked together.
|
|
63
|
+
"""
|
|
64
|
+
images = torch.randn(
|
|
65
|
+
(
|
|
66
|
+
number_of_images,
|
|
67
|
+
self.image_channels,
|
|
68
|
+
self.image_size,
|
|
69
|
+
self.image_size,
|
|
70
|
+
),
|
|
71
|
+
device=self.device,
|
|
72
|
+
)
|
|
73
|
+
|
|
74
|
+
denoised_images = self.model.denoise(images)
|
|
75
|
+
|
|
76
|
+
if save_gif:
|
|
77
|
+
pil_images = []
|
|
78
|
+
for timestep, images in enumerate(denoised_images):
|
|
79
|
+
pil_image_grid = self._visualise_images(images)
|
|
80
|
+
pil_images.append(pil_image_grid)
|
|
81
|
+
|
|
82
|
+
images = pil_images[1::2]
|
|
83
|
+
frame_one = pil_images[0]
|
|
84
|
+
|
|
85
|
+
logging.info("Saving GIF")
|
|
86
|
+
frame_one.save(
|
|
87
|
+
"generated.gif",
|
|
88
|
+
format="GIF",
|
|
89
|
+
append_images=images,
|
|
90
|
+
save_all=True,
|
|
91
|
+
duration=[10 for i in range(len(images))] + [2000],
|
|
92
|
+
loop=0,
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
else:
|
|
96
|
+
pil_image_grid = self._visualise_images(denoised_images[-1])
|
|
97
|
+
return pil_image_grid
|
|
98
|
+
|
|
99
|
+
def get_generator(self, number_of_images: int = 1):
|
|
100
|
+
"""An image generator.
|
|
101
|
+
|
|
102
|
+
This method is a generator that will generate a batch of images. At
|
|
103
|
+
every call, the generator denoises the images by one more step until the
|
|
104
|
+
image is fully generated. This can be particularly useful for running the
|
|
105
|
+
image generation step by step or for a streaming API.
|
|
106
|
+
|
|
107
|
+
Args:
|
|
108
|
+
number_of_images: The number of images the generator should generate.
|
|
109
|
+
"""
|
|
110
|
+
images = torch.randn(
|
|
111
|
+
(
|
|
112
|
+
number_of_images,
|
|
113
|
+
self.image_channels,
|
|
114
|
+
self.image_size,
|
|
115
|
+
self.image_size,
|
|
116
|
+
),
|
|
117
|
+
device=self.device,
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
for i in range(self.model.diffuser.beta_scheduler.steps)[::-1]:
|
|
121
|
+
timestep = torch.full((images.shape[0],), i, device=self.device)
|
|
122
|
+
images = self.model.diffuser._denoise_step(
|
|
123
|
+
images, model=self.model, timestep=timestep
|
|
124
|
+
)
|
|
125
|
+
images = torch.clamp(images, -1.0, 1.0)
|
|
126
|
+
|
|
127
|
+
yield self._visualise_images(images)
|
|
@@ -1,19 +1,14 @@
|
|
|
1
1
|
import pathlib
|
|
2
|
-
from typing import Callable
|
|
3
|
-
from typing import Dict
|
|
4
|
-
from typing import Optional
|
|
2
|
+
from typing import Callable, Dict
|
|
5
3
|
|
|
6
4
|
import torch
|
|
7
5
|
from torch.nn import functional as F
|
|
8
|
-
from torch.utils.data import DataLoader
|
|
9
|
-
from torch.utils.data import Dataset
|
|
6
|
+
from torch.utils.data import DataLoader, Dataset
|
|
10
7
|
from tqdm import tqdm
|
|
11
8
|
|
|
12
9
|
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
13
|
-
from diffusion_models.utils.schemas import BetaSchedulerConfiguration
|
|
14
|
-
|
|
15
|
-
from diffusion_models.utils.schemas import LogConfiguration
|
|
16
|
-
from diffusion_models.utils.schemas import TrainingConfiguration
|
|
10
|
+
from diffusion_models.utils.schemas import BetaSchedulerConfiguration, \
|
|
11
|
+
Checkpoint, LogConfiguration, TrainingConfiguration
|
|
17
12
|
from diffusion_models.utils.tensorboard import TensorboardManager
|
|
18
13
|
|
|
19
14
|
|
|
@@ -25,17 +20,36 @@ class DiffusionTrainer:
|
|
|
25
20
|
optimizer: torch.optim.Optimizer,
|
|
26
21
|
training_configuration: TrainingConfiguration,
|
|
27
22
|
loss_function: Callable = F.l1_loss,
|
|
28
|
-
scheduler: Optional[torch.optim.lr_scheduler.
|
|
23
|
+
# scheduler: Optional[torch.optim.lr_scheduler.StepLR] = None,
|
|
29
24
|
log_configuration: LogConfiguration = LogConfiguration(),
|
|
30
25
|
reverse_transforms: Callable = lambda x: x,
|
|
31
26
|
device: str = "cuda",
|
|
32
27
|
):
|
|
28
|
+
"""A diffusion trainer framework.
|
|
29
|
+
|
|
30
|
+
This is a simplified framework for training a diffusion models.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
model: A diffusion model.
|
|
34
|
+
dataset: A dataset to train on.
|
|
35
|
+
optimizer: The optimizer to use.
|
|
36
|
+
training_configuration: The training configuration to use.
|
|
37
|
+
loss_function: The loss function to use.
|
|
38
|
+
log_configuration: The logging configuration to use.
|
|
39
|
+
reverse_transforms: The reverse transforms to use.
|
|
40
|
+
device: The device to use.
|
|
41
|
+
"""
|
|
33
42
|
self.model = model.to(device)
|
|
43
|
+
"""The diffusion model to use."""
|
|
34
44
|
self.optimizer = optimizer
|
|
45
|
+
"""The optimizer to use."""
|
|
35
46
|
self.loss_function = loss_function
|
|
47
|
+
"""The loss function to use."""
|
|
36
48
|
self.training_configuration = training_configuration
|
|
37
|
-
|
|
49
|
+
"""The training configuration to use."""
|
|
50
|
+
# self.scheduler = scheduler
|
|
38
51
|
self.device = device
|
|
52
|
+
"""The device to use."""
|
|
39
53
|
|
|
40
54
|
self.dataloader = DataLoader(
|
|
41
55
|
dataset=dataset,
|
|
@@ -46,6 +60,7 @@ class DiffusionTrainer:
|
|
|
46
60
|
pin_memory=True,
|
|
47
61
|
persistent_workers=True,
|
|
48
62
|
)
|
|
63
|
+
"""A torch dataloader."""
|
|
49
64
|
|
|
50
65
|
self._image_shape = dataset[0][0].shape
|
|
51
66
|
|
|
@@ -53,24 +68,35 @@ class DiffusionTrainer:
|
|
|
53
68
|
device=device
|
|
54
69
|
# init_scale=8192,
|
|
55
70
|
)
|
|
71
|
+
"""A torch GradScaler object."""
|
|
56
72
|
|
|
57
73
|
self.log_configuration = log_configuration
|
|
74
|
+
"""A LogConfiguration object."""
|
|
58
75
|
|
|
59
76
|
self.checkpoint_path = (
|
|
60
77
|
pathlib.Path("../checkpoints")
|
|
61
78
|
/ self.training_configuration.training_name
|
|
62
79
|
)
|
|
80
|
+
"""The path to save checkpoints."""
|
|
63
81
|
|
|
64
82
|
self.checkpoint_path.mkdir(exist_ok=True)
|
|
65
83
|
self.tensorboard_manager = TensorboardManager(
|
|
66
84
|
log_name=self.training_configuration.training_name,
|
|
67
85
|
)
|
|
86
|
+
"""A tensorboard manager instance."""
|
|
68
87
|
|
|
69
88
|
self.reverse_transforms = reverse_transforms
|
|
89
|
+
"""A set of reverse transforms."""
|
|
70
90
|
|
|
71
91
|
torch.backends.cudnn.benchmark = True
|
|
72
92
|
|
|
73
93
|
def save_checkpoint(self, epoch: int, checkpoint_name: str):
|
|
94
|
+
"""Save a checkpoint.
|
|
95
|
+
|
|
96
|
+
Args:
|
|
97
|
+
epoch: The current epoch.
|
|
98
|
+
checkpoint_name: The name of the checkpoint.
|
|
99
|
+
"""
|
|
74
100
|
checkpoint = Checkpoint(
|
|
75
101
|
epoch=epoch,
|
|
76
102
|
model_state_dict=self.model.state_dict(),
|
|
@@ -89,6 +115,7 @@ class DiffusionTrainer:
|
|
|
89
115
|
checkpoint.to_file(self.checkpoint_path / checkpoint_name)
|
|
90
116
|
|
|
91
117
|
def train(self):
|
|
118
|
+
"""Start the diffusion training."""
|
|
92
119
|
self.model.train()
|
|
93
120
|
for epoch in range(self.training_configuration.number_of_epochs):
|
|
94
121
|
for step, batch in enumerate(
|
|
@@ -140,6 +167,14 @@ class DiffusionTrainer:
|
|
|
140
167
|
|
|
141
168
|
@torch.no_grad()
|
|
142
169
|
def log_to_tensorboard(self, metrics: Dict[str, float], global_step: int):
|
|
170
|
+
"""Log to tensorboard.
|
|
171
|
+
|
|
172
|
+
This method logs some useful metrics and visualizations to tensorboard.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
metrics: A dictionary mapping metric names to values.
|
|
176
|
+
global_step: The current global step.
|
|
177
|
+
"""
|
|
143
178
|
self.model.eval()
|
|
144
179
|
if global_step % self.log_configuration.log_rate == 0:
|
|
145
180
|
self.tensorboard_manager.log_metrics(
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from typing import List, TYPE_CHECKING
|
|
3
|
+
from typing import Tuple
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
8
|
+
BaseBetaScheduler,
|
|
9
|
+
)
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class BaseDiffuser(abc.ABC):
|
|
15
|
+
|
|
16
|
+
def __init__(self, beta_scheduler: BaseBetaScheduler):
|
|
17
|
+
"""Initializes the object with the specified beta scheduler.
|
|
18
|
+
|
|
19
|
+
BaseDiffuser is an abstract base class for different diffuser
|
|
20
|
+
implementations. It defines the interface that all diffusers should adhere
|
|
21
|
+
to.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
beta_scheduler: The beta scheduler used by the diffuser.
|
|
25
|
+
|
|
26
|
+
Warnings:
|
|
27
|
+
Do not instantiate this class directly. Instead, build your own Diffuser
|
|
28
|
+
by inheriting from BaseDiffuser.
|
|
29
|
+
(see :class:`~.gaussian_diffuser.GaussianDiffuser`)
|
|
30
|
+
|
|
31
|
+
"""
|
|
32
|
+
self.beta_scheduler: BaseBetaScheduler = beta_scheduler
|
|
33
|
+
"""The beta scheduler used by the diffuser."""
|
|
34
|
+
|
|
35
|
+
@abc.abstractmethod
|
|
36
|
+
def diffuse_batch(
|
|
37
|
+
self, images: torch.Tensor
|
|
38
|
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
39
|
+
"""Diffuse a batch of images.
|
|
40
|
+
|
|
41
|
+
Args:
|
|
42
|
+
images: A tensor containing a batch of images.
|
|
43
|
+
|
|
44
|
+
Returns:
|
|
45
|
+
A tuple containing three tensors
|
|
46
|
+
|
|
47
|
+
- images: Diffused batch of images.
|
|
48
|
+
- noise: Noise added to the images.
|
|
49
|
+
- timesteps: Timesteps used for diffusion.
|
|
50
|
+
"""
|
|
51
|
+
raise NotImplementedError()
|
|
52
|
+
|
|
53
|
+
@abc.abstractmethod
|
|
54
|
+
def denoise_batch(self, images: torch.Tensor, model: "BaseDiffusionModel") -> List[torch.Tensor]:
|
|
55
|
+
"""Denoise a batch of images.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
images: A tensor containing a batch of images to denoise.
|
|
59
|
+
model: The model to be used for denoising.
|
|
60
|
+
|
|
61
|
+
Returns:
|
|
62
|
+
A list of tensors containing a batch of denoised images.
|
|
63
|
+
"""
|
|
64
|
+
raise NotImplementedError()
|
|
65
|
+
|
|
66
|
+
@abc.abstractmethod
|
|
67
|
+
def to(self, device: str = "cpu"):
|
|
68
|
+
"""Moves the data to the specified device.
|
|
69
|
+
|
|
70
|
+
This performs a similar behaviour to the `to` method of PyTorch.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
device: The device to which the method should move the data.
|
|
74
|
+
It should be a string indicating the desired device.
|
|
75
|
+
Default is "cpu".
|
|
76
|
+
"""
|
|
77
|
+
raise NotImplementedError()
|
|
78
|
+
|
|
@@ -0,0 +1,263 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BaseBetaScheduler:
|
|
9
|
+
def __init__(self, steps: int, enforce_zero_terminal_snr: bool = False):
|
|
10
|
+
"""Initializes a beta scheduler.
|
|
11
|
+
|
|
12
|
+
BaseBetaScheduler is an abstract base class for different beta scheduler
|
|
13
|
+
implementations. It defines the interface that all beta schedulers should
|
|
14
|
+
adhere to.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
steps: The number of steps for the beta.
|
|
18
|
+
enforce_zero_terminal_snr: Whether to enforce zero terminal SNR inline
|
|
19
|
+
with `"Common Diffusion Noise Schedules and Sample Steps are Flawed"
|
|
20
|
+
<https://arxiv.org/abs/2305.08891>`_.\n
|
|
21
|
+
Defaults to ``False``.
|
|
22
|
+
|
|
23
|
+
Warnings:
|
|
24
|
+
Do not instantiate this class directly. Instead, build your own Beta
|
|
25
|
+
scheduler by inheriting from BaseBetaScheduler.
|
|
26
|
+
(see :class:`~.LinearBetaScheduler`)
|
|
27
|
+
"""
|
|
28
|
+
super().__init__()
|
|
29
|
+
self.steps: int = steps
|
|
30
|
+
"""The number of steps for the beta scheduler."""
|
|
31
|
+
self.betas = self.sample_betas()
|
|
32
|
+
"""The :math:`\\beta` computed according to :meth:`~.BaseBetaScheduler.sample_betas`."""
|
|
33
|
+
self.alpha_bars = self.compute_alpha_bar()
|
|
34
|
+
"""The :math:`\\bar{\\alpha}` computed according to :meth:`~.BaseBetaScheduler.compute_alpha_bar`."""
|
|
35
|
+
|
|
36
|
+
if enforce_zero_terminal_snr:
|
|
37
|
+
self.enforce_zero_terminal_snr()
|
|
38
|
+
|
|
39
|
+
def enforce_zero_terminal_snr(self):
|
|
40
|
+
"""Enforce terminal SNR by adjusting :math:`\\beta` and :math:`\\bar{\\alpha}`.
|
|
41
|
+
|
|
42
|
+
This method enforces zero terminal SNR according to
|
|
43
|
+
`"Common Diffusion Noise Schedules and Sample Steps are Flawed"
|
|
44
|
+
<https://arxiv.org/abs/2305.08891>`_.
|
|
45
|
+
"""
|
|
46
|
+
alpha_bar_length = len(self.alpha_bars)
|
|
47
|
+
|
|
48
|
+
# Convert betas to alphas_bar_sqrt
|
|
49
|
+
alphas = 1 - self.betas
|
|
50
|
+
alphas_bar = alphas.cumprod(0)
|
|
51
|
+
alphas_bar_sqrt = alphas_bar.sqrt()
|
|
52
|
+
|
|
53
|
+
# Store old values.
|
|
54
|
+
alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
|
|
55
|
+
alphas_bar_sqrt_t = alphas_bar_sqrt[-1].clone()
|
|
56
|
+
# Shift so last timestep is zero.
|
|
57
|
+
alphas_bar_sqrt -= alphas_bar_sqrt_t
|
|
58
|
+
# Scale so first timestep is back to old value.
|
|
59
|
+
alphas_bar_sqrt *= alphas_bar_sqrt_0 / (
|
|
60
|
+
alphas_bar_sqrt_0 - alphas_bar_sqrt_t
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Convert alphas_bar_sqrt to betas
|
|
64
|
+
alphas_bar = alphas_bar_sqrt**2
|
|
65
|
+
alphas = alphas_bar[1:] / alphas_bar[:-1]
|
|
66
|
+
alphas = torch.cat([alphas_bar[0:1], alphas])
|
|
67
|
+
betas = 1 - alphas
|
|
68
|
+
if len(alphas) == alpha_bar_length:
|
|
69
|
+
self.betas = betas
|
|
70
|
+
self.alpha_bars = alphas_bar
|
|
71
|
+
else:
|
|
72
|
+
logging.warning(
|
|
73
|
+
"Got different alpha_bar length after enforcing zero SNR. Please check your beta scheduler"
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
@abc.abstractmethod
|
|
77
|
+
def sample_betas(self) -> torch.Tensor:
|
|
78
|
+
"""Compute :math:`\\beta` for noise scheduling.
|
|
79
|
+
|
|
80
|
+
Returns:
|
|
81
|
+
A torch tensor of the :math:`\\beta` values.
|
|
82
|
+
"""
|
|
83
|
+
raise NotImplementedError()
|
|
84
|
+
|
|
85
|
+
@abc.abstractmethod
|
|
86
|
+
def compute_alpha_bar(self) -> torch.Tensor:
|
|
87
|
+
"""Compute :math:`\\bar{\\alpha}` for noise scheduling.
|
|
88
|
+
|
|
89
|
+
Returns:
|
|
90
|
+
A torch tensor of the :math:`\\bar{\\alpha}` values.
|
|
91
|
+
"""
|
|
92
|
+
raise NotImplementedError()
|
|
93
|
+
|
|
94
|
+
def to(self, device: str):
|
|
95
|
+
"""Moves the beta scheduler to the given device.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
device: The device to which the method should move the object.
|
|
99
|
+
Default is "cpu".
|
|
100
|
+
|
|
101
|
+
"""
|
|
102
|
+
self.betas = self.betas.to(device)
|
|
103
|
+
self.alpha_bars = self.alpha_bars.to(device)
|
|
104
|
+
return self
|
|
105
|
+
|
|
106
|
+
@classmethod
|
|
107
|
+
def from_tensors(
|
|
108
|
+
cls, steps: int, betas: torch.Tensor, alpha_bars: torch.Tensor
|
|
109
|
+
):
|
|
110
|
+
"""Instantiate a beta scheduler from tensors.
|
|
111
|
+
|
|
112
|
+
Instantiate a beta scheduler from tensors. This is particularly useful for
|
|
113
|
+
loading checkpoints.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
steps: The number of steps for the beta scheduler.
|
|
117
|
+
betas: The pre-computed beta values for the noise scheduler.
|
|
118
|
+
alpha_bars: The pre-computed alpha bar values for the noise scheduler.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
|
|
122
|
+
"""
|
|
123
|
+
generic_beta_scheduler = cls(0)
|
|
124
|
+
generic_beta_scheduler.steps = steps
|
|
125
|
+
generic_beta_scheduler.betas = betas
|
|
126
|
+
generic_beta_scheduler.alpha_bars = alpha_bars
|
|
127
|
+
return generic_beta_scheduler
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class LinearBetaScheduler(BaseBetaScheduler):
|
|
131
|
+
def __init__(
|
|
132
|
+
self,
|
|
133
|
+
beta_start: float = 0.0001,
|
|
134
|
+
beta_end: float = 0.02,
|
|
135
|
+
steps: int = 1000,
|
|
136
|
+
enforce_zero_terminal_snr: bool = True,
|
|
137
|
+
):
|
|
138
|
+
"""A Linear Beta scheduler.
|
|
139
|
+
|
|
140
|
+
A simple linear beta scheduler with betas linearly spaced between
|
|
141
|
+
``beta_start`` and ``beta_end``.
|
|
142
|
+
|
|
143
|
+
Args:
|
|
144
|
+
beta_start: The starting value of the betas.
|
|
145
|
+
beta_end: The end value of the betas.
|
|
146
|
+
steps: The number of steps for the beta scheduler. This is also the number
|
|
147
|
+
of betas.
|
|
148
|
+
enforce_zero_terminal_snr: Whether to enforce zero terminal SNR.
|
|
149
|
+
"""
|
|
150
|
+
self.beta_start: int = beta_start
|
|
151
|
+
"""The starting value of the betas."""
|
|
152
|
+
self.beta_end: int = beta_end
|
|
153
|
+
"""The end value of the betas."""
|
|
154
|
+
super().__init__(
|
|
155
|
+
steps=steps,
|
|
156
|
+
enforce_zero_terminal_snr=enforce_zero_terminal_snr,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
def sample_betas(self) -> torch.Tensor:
|
|
160
|
+
"""Return linearly spaced betas between ``self.beta_start`` and ``self.beta_end``."""
|
|
161
|
+
return torch.linspace(self.beta_start, self.beta_end, self.steps)
|
|
162
|
+
|
|
163
|
+
def compute_alpha_bar(self):
|
|
164
|
+
"""Return :math:`\\bar{\\alpha}` computed from the beta values."""
|
|
165
|
+
alphas = 1 - self.betas
|
|
166
|
+
alpha_bar = torch.cumprod(alphas, dim=0)
|
|
167
|
+
return alpha_bar
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class CosineBetaScheduler(BaseBetaScheduler):
|
|
171
|
+
def __init__(
|
|
172
|
+
self,
|
|
173
|
+
offset: float = 0.008,
|
|
174
|
+
steps: int = 1000,
|
|
175
|
+
max_beta: Optional[float] = 0.999,
|
|
176
|
+
):
|
|
177
|
+
"""A Cosine Beta scheduler.
|
|
178
|
+
|
|
179
|
+
A Cosine Beta Scheduler based on the following formulas:
|
|
180
|
+
|
|
181
|
+
.. math::
|
|
182
|
+
:nowrap:
|
|
183
|
+
|
|
184
|
+
\\begin{equation}
|
|
185
|
+
\\left\\{ \\begin{aligned}
|
|
186
|
+
\\bar{\\alpha}_t &= \\frac{f(t)}{f(0)} \\\\
|
|
187
|
+
\\beta_t &= 1 - \\frac{\\bar{\\alpha}_t}{\\bar{\\alpha}_t -1}
|
|
188
|
+
\\end{aligned} \\right.
|
|
189
|
+
\\end{equation}
|
|
190
|
+
|
|
191
|
+
where
|
|
192
|
+
|
|
193
|
+
.. math::
|
|
194
|
+
|
|
195
|
+
f(t) = \\cos(\\frac{t/T + s}{1 + s} * \\frac{\\pi}{2})^2
|
|
196
|
+
|
|
197
|
+
where
|
|
198
|
+
|
|
199
|
+
.. math::
|
|
200
|
+
:nowrap:
|
|
201
|
+
|
|
202
|
+
\\begin{equation}
|
|
203
|
+
\\left\\{ \\begin{aligned}
|
|
204
|
+
|
|
205
|
+
s & \\text{ is the offset} \\\\
|
|
206
|
+
T & \\text{ is the number of steps}
|
|
207
|
+
|
|
208
|
+
\\end{aligned} \\right.
|
|
209
|
+
\\end{equation}
|
|
210
|
+
|
|
211
|
+
Args:
|
|
212
|
+
offset: The offset :math:`s` defined above.
|
|
213
|
+
steps: The number of steps for the beta scheduler.
|
|
214
|
+
max_beta: The maximum beta values. Higher values will be clipped.
|
|
215
|
+
"""
|
|
216
|
+
self.offset: float = offset
|
|
217
|
+
"""The offset :math:`s` defined above."""
|
|
218
|
+
self.max_beta: Optional[float] = max_beta
|
|
219
|
+
"""The maximum beta values. Higher values will be clipped."""
|
|
220
|
+
self.steps: int = steps
|
|
221
|
+
"""The number of steps for the beta scheduler."""
|
|
222
|
+
self._alpha_bars = self._compute_alpha_bar()
|
|
223
|
+
self._betas = self._compute_betas()
|
|
224
|
+
|
|
225
|
+
super().__init__(
|
|
226
|
+
steps=steps,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
def f(self, t: torch.Tensor) -> torch.Tensor:
|
|
230
|
+
"""A helper function to compute the :math:`\\bar{\\alpha}_t`.
|
|
231
|
+
|
|
232
|
+
Args:
|
|
233
|
+
t: The timestep to compute.
|
|
234
|
+
|
|
235
|
+
Returns:
|
|
236
|
+
|
|
237
|
+
.. math::
|
|
238
|
+
|
|
239
|
+
f(t) = \\cos(\\frac{t/T + s}{1 + s} * \\frac{\\pi}{2})^2
|
|
240
|
+
|
|
241
|
+
"""
|
|
242
|
+
return (
|
|
243
|
+
torch.cos(
|
|
244
|
+
(((t / self.steps) + self.offset) / (1 + self.offset)) * (torch.pi / 2)
|
|
245
|
+
)
|
|
246
|
+
** 2
|
|
247
|
+
)
|
|
248
|
+
|
|
249
|
+
def _compute_betas(self):
|
|
250
|
+
betas = 1 - self._alpha_bars[1:] / self._alpha_bars[:-1]
|
|
251
|
+
if self.max_beta:
|
|
252
|
+
betas = torch.clip(betas, max=self.max_beta)
|
|
253
|
+
return betas
|
|
254
|
+
|
|
255
|
+
def _compute_alpha_bar(self):
|
|
256
|
+
t = torch.linspace(0, self.steps, self.steps, dtype=torch.float32)
|
|
257
|
+
return self.f(t) / self.f(torch.tensor([0], dtype=torch.float32))
|
|
258
|
+
|
|
259
|
+
def sample_betas(self):
|
|
260
|
+
return self._betas
|
|
261
|
+
|
|
262
|
+
def compute_alpha_bar(self):
|
|
263
|
+
return self._alpha_bars
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import List
|
|
1
|
+
from typing import List, Tuple, TYPE_CHECKING
|
|
2
2
|
|
|
3
3
|
import torch
|
|
4
4
|
from tqdm import tqdm
|
|
@@ -9,14 +9,33 @@ from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
|
9
9
|
)
|
|
10
10
|
from diffusion_models.utils.schemas import Checkpoint
|
|
11
11
|
|
|
12
|
+
if TYPE_CHECKING:
|
|
13
|
+
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
12
14
|
|
|
13
15
|
class GaussianDiffuser(BaseDiffuser):
|
|
14
16
|
def __init__(self, beta_scheduler: BaseBetaScheduler):
|
|
17
|
+
"""Initializes the class instance.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
beta_scheduler (BaseBetaScheduler): The beta scheduler instance to be used.
|
|
21
|
+
|
|
22
|
+
"""
|
|
15
23
|
super().__init__(beta_scheduler)
|
|
16
|
-
self.device = "
|
|
24
|
+
self.device: str = "cpu"
|
|
25
|
+
"""The device to use. Defaults to cpu."""
|
|
17
26
|
|
|
18
27
|
@classmethod
|
|
19
28
|
def from_checkpoint(cls, checkpoint: Checkpoint) -> "GaussianDiffuser":
|
|
29
|
+
"""Instantiate a Gaussian Diffuser from a training checkpoint.
|
|
30
|
+
|
|
31
|
+
Args:
|
|
32
|
+
checkpoint: The training checkpoint object containing
|
|
33
|
+
the trained model's parameters and configuration.
|
|
34
|
+
|
|
35
|
+
Returns:
|
|
36
|
+
An instance of the GaussianDiffuser class initialized with the parameters
|
|
37
|
+
loaded from the given checkpoint.
|
|
38
|
+
"""
|
|
20
39
|
return cls(
|
|
21
40
|
beta_scheduler=BaseBetaScheduler.from_tensors(
|
|
22
41
|
steps=checkpoint.beta_scheduler_config.steps,
|
|
@@ -26,11 +45,41 @@ class GaussianDiffuser(BaseDiffuser):
|
|
|
26
45
|
)
|
|
27
46
|
|
|
28
47
|
def to(self, device: str = "cpu"):
|
|
48
|
+
"""Moves the data to the specified device.
|
|
49
|
+
|
|
50
|
+
This performs a similar behaviour to the `to` method of PyTorch. moving the
|
|
51
|
+
GaussianDiffuser and the BetaScheduler to the specified device.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
device: The device to which the method should move the object.
|
|
55
|
+
Default is "cpu".
|
|
56
|
+
|
|
57
|
+
Example:
|
|
58
|
+
>>> gaussian_diffuser = GaussianDiffuser()
|
|
59
|
+
>>> gaussian_diffuser = gaussian_diffuser.to(device="cuda")
|
|
60
|
+
"""
|
|
29
61
|
self.device = device
|
|
30
62
|
self.beta_scheduler = self.beta_scheduler.to(self.device)
|
|
31
63
|
return self
|
|
32
64
|
|
|
33
|
-
def diffuse_batch(
|
|
65
|
+
def diffuse_batch(
|
|
66
|
+
self, images: torch.Tensor
|
|
67
|
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
68
|
+
"""Diffuse a batch of images.
|
|
69
|
+
|
|
70
|
+
Diffuse the given batch of images by adding noise based on the beta scheduler.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
images: Batch of images to diffuse.\n
|
|
74
|
+
Shape should be ``(B, C, H, W)``.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
A tuple containing three tensors
|
|
78
|
+
|
|
79
|
+
- images: Diffused batch of images.
|
|
80
|
+
- noise: Noise added to the images.
|
|
81
|
+
- timesteps: Timesteps used for diffusion.
|
|
82
|
+
"""
|
|
34
83
|
timesteps = torch.randint(
|
|
35
84
|
0, self.beta_scheduler.steps, (images.shape[0],), device=self.device
|
|
36
85
|
)
|
|
@@ -67,8 +116,19 @@ class GaussianDiffuser(BaseDiffuser):
|
|
|
67
116
|
def denoise_batch(
|
|
68
117
|
self,
|
|
69
118
|
images: torch.Tensor,
|
|
70
|
-
model:
|
|
119
|
+
model: "BaseDiffusionModel",
|
|
71
120
|
) -> List[torch.Tensor]:
|
|
121
|
+
"""Denoise a batch of images.
|
|
122
|
+
|
|
123
|
+
This denoises a batch images. This is the image generation process.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
images: A batch of noisy images.
|
|
127
|
+
model: The model to be used for denoising.
|
|
128
|
+
|
|
129
|
+
Returns:
|
|
130
|
+
A list of tensors containing a batch of denoised images.
|
|
131
|
+
"""
|
|
72
132
|
denoised_images = []
|
|
73
133
|
for i in tqdm(range(self.beta_scheduler.steps)[::-1], desc="Denoising"):
|
|
74
134
|
timestep = torch.full((images.shape[0],), i, device=self.device)
|
|
@@ -61,15 +61,17 @@ class SinusoidalPositionEmbeddings(nn.Module):
|
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
class SimpleUnet(BaseDiffusionModel):
|
|
64
|
-
"""
|
|
65
|
-
A simplified variant of the Unet architecture.
|
|
66
|
-
"""
|
|
67
|
-
|
|
68
64
|
def __init__(
|
|
69
65
|
self,
|
|
70
66
|
diffuser: GaussianDiffuser,
|
|
71
67
|
image_channels: int,
|
|
72
68
|
):
|
|
69
|
+
"""A Simplified variant of the Unet architecture used in DDPM.
|
|
70
|
+
|
|
71
|
+
Args:
|
|
72
|
+
diffuser: A gaussian diffuser.
|
|
73
|
+
image_channels: The number of image channels.
|
|
74
|
+
"""
|
|
73
75
|
super().__init__(diffuser=diffuser)
|
|
74
76
|
image_channels = image_channels
|
|
75
77
|
down_channels = (64, 128, 256, 512, 1024)
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from typing import List, Tuple
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch import nn
|
|
6
|
+
|
|
7
|
+
from diffusion_models.gaussian_diffusion.base_diffuser import BaseDiffuser
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class BaseDiffusionModel(nn.Module, abc.ABC):
|
|
11
|
+
def __init__(self, diffuser: BaseDiffuser):
|
|
12
|
+
"""Initializes the object with the specified diffuser.
|
|
13
|
+
|
|
14
|
+
BaseDiffusionModel is an abstract base class for different diffusion models
|
|
15
|
+
implementations. It defines the interface that all diffusion models should
|
|
16
|
+
adhere to.
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
diffuser: The diffuser to use for the diffusion model.
|
|
20
|
+
|
|
21
|
+
Warnings:
|
|
22
|
+
Do not instantiate this class directly. Instead, build your own diffusion
|
|
23
|
+
model by inheriting from BaseDiffusionModel.
|
|
24
|
+
(see :class:`~.SimpleUnet.SimpleUnet`)
|
|
25
|
+
|
|
26
|
+
"""
|
|
27
|
+
super().__init__()
|
|
28
|
+
self.diffuser: BaseDiffuser = diffuser
|
|
29
|
+
"""A diffuser to be used by the diffusion model."""
|
|
30
|
+
|
|
31
|
+
def diffuse(
|
|
32
|
+
self, images: torch.Tensor
|
|
33
|
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
34
|
+
"""Diffuse a batch of images.
|
|
35
|
+
|
|
36
|
+
Args:
|
|
37
|
+
images: A tensor containing a batch of images.
|
|
38
|
+
|
|
39
|
+
Returns:
|
|
40
|
+
A tuple containing three tensors
|
|
41
|
+
|
|
42
|
+
- images: Diffused batch of images.
|
|
43
|
+
- noise: Noise added to the images.
|
|
44
|
+
- timesteps: Timesteps used for diffusion.
|
|
45
|
+
"""
|
|
46
|
+
return self.diffuser.diffuse_batch(images=images)
|
|
47
|
+
|
|
48
|
+
def denoise(self, images: torch.Tensor) -> List[torch.Tensor]:
|
|
49
|
+
"""Denoise a batch of images.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
images: A tensor containing a batch of images to denoise.
|
|
53
|
+
|
|
54
|
+
Returns:
|
|
55
|
+
A list of tensors containing a batch of denoised images.
|
|
56
|
+
"""
|
|
57
|
+
return self.diffuser.denoise_batch(images=images, model=self)
|
|
58
|
+
|
|
59
|
+
@abc.abstractmethod
|
|
60
|
+
def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor:
|
|
61
|
+
"""Forward pass of the diffusion model.
|
|
62
|
+
|
|
63
|
+
The forward pass of the diffusion model, predicting the noise at a single
|
|
64
|
+
step.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
x: A batch of noisy images.
|
|
68
|
+
timestep: The timesteps of each image in the batch.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
A tensor representing the noise predicted for each image.
|
|
72
|
+
"""
|
|
73
|
+
raise NotImplementedError
|
|
74
|
+
|
|
75
|
+
def to(self, device: str = "cpu") -> "BaseDiffusionModel":
|
|
76
|
+
"""Moves the model to the specified device.
|
|
77
|
+
|
|
78
|
+
This performs a similar behaviour to the `to` method of PyTorch. moving the
|
|
79
|
+
DiffusionModel and all related artifacts to the specified device.
|
|
80
|
+
|
|
81
|
+
Args:
|
|
82
|
+
device: The device to which the method should move the object.
|
|
83
|
+
Default is "cpu".
|
|
84
|
+
"""
|
|
85
|
+
new_self = super(BaseDiffusionModel, self).to(device)
|
|
86
|
+
new_self.diffuser = new_self.diffuser.to(device)
|
|
87
|
+
return new_self
|
|
88
|
+
|
|
89
|
+
def compile(self, *args, **kwargs):
|
|
90
|
+
"""Compiles the diffusion model.
|
|
91
|
+
|
|
92
|
+
This performs a similar behaviour to the `compile` method of PyTorch.
|
|
93
|
+
|
|
94
|
+
Returns:
|
|
95
|
+
A compiled diffusion model.
|
|
96
|
+
"""
|
|
97
|
+
model = torch.compile(self, *args, **kwargs)
|
|
98
|
+
model.to = self.to
|
|
99
|
+
return model
|
|
File without changes
|
|
@@ -16,49 +16,82 @@ from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
|
16
16
|
|
|
17
17
|
@dataclass
|
|
18
18
|
class TrainingConfiguration:
|
|
19
|
+
"""A training configuration for simple experiment management."""
|
|
19
20
|
training_name: str
|
|
21
|
+
"""The name of the training."""
|
|
20
22
|
batch_size: int
|
|
23
|
+
"""The batch size used for training."""
|
|
21
24
|
learning_rate: float
|
|
25
|
+
"""The learning rate of the training."""
|
|
22
26
|
number_of_epochs: int
|
|
23
|
-
|
|
24
|
-
log_rate: int = 10
|
|
25
|
-
image_rate: int = 100
|
|
27
|
+
"""The number of epoch used for training."""
|
|
26
28
|
checkpoint_rate: int = 100
|
|
27
|
-
|
|
29
|
+
"""The rate at which checkpoints are saved.."""
|
|
28
30
|
mixed_precision_training: bool = False # TODO: This is not complete yet
|
|
31
|
+
"""Whether or not to use automatic mixed precision training."""
|
|
29
32
|
gradient_clip: Optional[float] = None # TODO: This is not complete yet
|
|
33
|
+
"""Whether or not to clip gradients."""
|
|
30
34
|
|
|
31
35
|
|
|
32
36
|
@dataclass
|
|
33
37
|
class LogConfiguration:
|
|
38
|
+
"""An object to manage logging configuration."""
|
|
34
39
|
log_rate: int = 10
|
|
40
|
+
"""The rate at which training metrics are logged."""
|
|
35
41
|
image_rate: int = 50
|
|
42
|
+
"""The rate at which images are generated for visualization. This can be used to validate model performance."""
|
|
36
43
|
number_of_images: int = 5
|
|
44
|
+
"""The number of images to generate."""
|
|
37
45
|
# metrics: Dict[str, float] # TODO: consider Dict[str, Callable]
|
|
38
46
|
|
|
39
47
|
|
|
40
48
|
@dataclass
|
|
41
49
|
class BetaSchedulerConfiguration:
|
|
50
|
+
"""A simplified beta scheduler configuration."""
|
|
42
51
|
steps: int
|
|
52
|
+
"""The number of steps in the beta scheduler."""
|
|
43
53
|
betas: torch.Tensor
|
|
54
|
+
"""The beta values."""
|
|
44
55
|
alpha_bars: torch.Tensor
|
|
45
|
-
|
|
56
|
+
"""The alpha bar values."""
|
|
46
57
|
|
|
47
58
|
@dataclass
|
|
48
59
|
class Checkpoint:
|
|
60
|
+
"""A simplified checkpoint framework for easy saving and loading."""
|
|
49
61
|
epoch: int
|
|
62
|
+
"""The current epoch."""
|
|
50
63
|
model_state_dict: Dict[str, Any]
|
|
64
|
+
"""The model state dict."""
|
|
51
65
|
optimizer_state_dict: Dict[str, Any]
|
|
66
|
+
"""The optimizer state dict."""
|
|
52
67
|
scaler: Optional[GradScaler]
|
|
68
|
+
"""The GradScaler instance."""
|
|
53
69
|
beta_scheduler_config: BetaSchedulerConfiguration
|
|
70
|
+
"""The beta scheduler configuration."""
|
|
54
71
|
tensorboard_run_name: Optional[str] = None
|
|
72
|
+
"""The name of the tensorboard run."""
|
|
55
73
|
image_channels: int = 3
|
|
74
|
+
"""The number of image channels used in the training."""
|
|
56
75
|
loss: Optional[float] = (
|
|
57
76
|
None # TODO: remove legacy parameter and resave models
|
|
58
77
|
)
|
|
78
|
+
"""The final loss value recorded.
|
|
79
|
+
|
|
80
|
+
Note:
|
|
81
|
+
This is a legacy parameter and will be removed in a future release.
|
|
82
|
+
|
|
83
|
+
"""
|
|
59
84
|
|
|
60
85
|
@classmethod
|
|
61
86
|
def from_file(cls, file_path: str) -> "Checkpoint":
|
|
87
|
+
"""Load and instantiate a checkpoint from a file.
|
|
88
|
+
|
|
89
|
+
Args:
|
|
90
|
+
file_path: The path to the checkpoint file.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
A checkpoint instance.
|
|
94
|
+
"""
|
|
62
95
|
checkpoint = torch.load(f=file_path)
|
|
63
96
|
checkpoint = cls(**checkpoint)
|
|
64
97
|
beta_scheduler_config = BetaSchedulerConfiguration(
|
|
@@ -68,6 +101,7 @@ class Checkpoint:
|
|
|
68
101
|
return checkpoint
|
|
69
102
|
|
|
70
103
|
def to_file(self, file_path: Union[str, pathlib.Path]) -> None:
|
|
104
|
+
"""Saves a checkpoint to a file."""
|
|
71
105
|
torch.save(dataclasses.asdict(self), file_path)
|
|
72
106
|
|
|
73
107
|
|
|
@@ -7,18 +7,39 @@ from torch.utils.tensorboard import SummaryWriter
|
|
|
7
7
|
|
|
8
8
|
class TensorboardManager:
|
|
9
9
|
def __init__(self, log_name: Optional[str] = None):
|
|
10
|
+
"""A tensorboard manager for simplified tensorboard logging.
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
log_name: The name of the tensorboard log run.
|
|
14
|
+
"""
|
|
10
15
|
if log_name is None:
|
|
11
16
|
log_name = "fill this" ##########
|
|
12
17
|
self.log_directory = f"../runs/{log_name}"
|
|
18
|
+
"""The directory where tensorboard logs are saved."""
|
|
13
19
|
self.summary_writer = SummaryWriter(log_dir=self.log_directory)
|
|
20
|
+
"""The tensorboard summary writer."""
|
|
14
21
|
|
|
15
22
|
def log_metrics(self, metrics: Dict[str, float], global_step: int):
|
|
23
|
+
"""Log metrics to tensorboard.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
metrics: A dictionary mapping metric names to values.
|
|
27
|
+
global_step: The step at which the metrics are recorded.
|
|
28
|
+
"""
|
|
16
29
|
for metric_name, value in metrics.items():
|
|
17
30
|
self.summary_writer.add_scalar(
|
|
18
31
|
metric_name, value, global_step=global_step
|
|
19
32
|
)
|
|
20
33
|
|
|
21
34
|
def log_images(self, tag: str, images: torch.Tensor, timestep: int):
|
|
35
|
+
"""Log images to tensorboard.
|
|
36
|
+
|
|
37
|
+
Args:
|
|
38
|
+
tag: The name to give the images in tensorboard.
|
|
39
|
+
images: A tensor representing the images to log.
|
|
40
|
+
timestep: The timestep at which the images are produced.
|
|
41
|
+
|
|
42
|
+
"""
|
|
22
43
|
self.summary_writer.add_images(
|
|
23
44
|
tag=tag, img_tensor=images, global_step=timestep
|
|
24
45
|
)
|
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.1
|
|
2
|
-
Name: diffusion-model-framework
|
|
3
|
-
Version: 0.1.0
|
|
4
|
-
Summary:
|
|
5
|
-
Home-page: https://github.com/sachahu1/Diffusion-Model-Framework
|
|
6
|
-
Author: Sacha Hu
|
|
7
|
-
Author-email: sachahu@mindsparkai.com
|
|
8
|
-
Requires-Python: >=3.9,<3.12
|
|
9
|
-
Classifier: Programming Language :: Python :: 3
|
|
10
|
-
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
-
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
-
Requires-Dist: numpy (>=1.26.4,<2.0.0)
|
|
14
|
-
Requires-Dist: pillow-simd (>=9.0.0.post1,<10.0.0)
|
|
15
|
-
Requires-Dist: tensorboard (>=2.17.0,<3.0.0)
|
|
16
|
-
Requires-Dist: torch (>=2.4.0,<3.0.0)
|
|
17
|
-
Requires-Dist: torchvision (>=0.19.0,<0.20.0)
|
|
18
|
-
Requires-Dist: tqdm (>=4.66.2,<5.0.0)
|
|
19
|
-
Project-URL: Documentation, https://www.sachahu.com/docs/diffusion-model-framework/latest/index.html
|
|
20
|
-
Project-URL: Repository, https://github.com/sachahu1/Diffusion-Model-Framework
|
|
21
|
-
Description-Content-Type: text/markdown
|
|
22
|
-
|
|
23
|
-
# Diffusion Model Playground
|
|
24
|
-
|
|
25
|
-

|
|
26
|
-

|
|
27
|
-

|
|
28
|
-

|
|
29
|
-
|
|
30
|
-
This repo is a playground for implementing, training and visualising diffusion
|
|
31
|
-
models.
|
|
32
|
-
|
|
33
|
-

|
|
34
|
-
|
|
35
|
-
# Getting Started
|
|
36
|
-
This tool uses the [poetry python manager](https://python-poetry.org/).
|
|
37
|
-
To install the dependencies, simply run `poetry install`.
|
|
38
|
-
|
|
39
|
-
## Examples
|
|
40
|
-
To get started with this codebase, head to the [examples](./examples) folder to
|
|
41
|
-
find a simple example to get started with training your own model along with
|
|
42
|
-
an example to easily run inference.
|
|
43
|
-
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
# Diffusion Model Playground
|
|
2
|
-
|
|
3
|
-

|
|
4
|
-

|
|
5
|
-

|
|
6
|
-

|
|
7
|
-
|
|
8
|
-
This repo is a playground for implementing, training and visualising diffusion
|
|
9
|
-
models.
|
|
10
|
-
|
|
11
|
-

|
|
12
|
-
|
|
13
|
-
# Getting Started
|
|
14
|
-
This tool uses the [poetry python manager](https://python-poetry.org/).
|
|
15
|
-
To install the dependencies, simply run `poetry install`.
|
|
16
|
-
|
|
17
|
-
## Examples
|
|
18
|
-
To get started with this codebase, head to the [examples](./examples) folder to
|
|
19
|
-
find a simple example to get started with training your own model along with
|
|
20
|
-
an example to easily run inference.
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
__version__ = "0.1.0"
|
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
import abc
|
|
2
|
-
from typing import List
|
|
3
|
-
from typing import Tuple
|
|
4
|
-
|
|
5
|
-
import torch
|
|
6
|
-
|
|
7
|
-
from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
8
|
-
BaseBetaScheduler,
|
|
9
|
-
)
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
class BaseDiffuser(abc.ABC):
|
|
13
|
-
def __init__(self, beta_scheduler: BaseBetaScheduler):
|
|
14
|
-
self.beta_scheduler = beta_scheduler
|
|
15
|
-
|
|
16
|
-
@abc.abstractmethod
|
|
17
|
-
def diffuse_batch(
|
|
18
|
-
self, images: torch.Tensor
|
|
19
|
-
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
20
|
-
pass
|
|
21
|
-
|
|
22
|
-
@abc.abstractmethod
|
|
23
|
-
def denoise_batch(self, images: torch.Tensor, model) -> List[torch.Tensor]:
|
|
24
|
-
pass
|
|
25
|
-
|
|
26
|
-
@abc.abstractmethod
|
|
27
|
-
def to(self, device: str = "cpu"):
|
|
28
|
-
pass
|
|
@@ -1,136 +0,0 @@
|
|
|
1
|
-
import abc
|
|
2
|
-
import logging
|
|
3
|
-
from typing import Optional
|
|
4
|
-
|
|
5
|
-
import torch
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
class BaseBetaScheduler:
|
|
9
|
-
def __init__(self, steps: int, enforce_zero_terminal_snr: bool = False):
|
|
10
|
-
super().__init__()
|
|
11
|
-
self.steps = steps
|
|
12
|
-
self.betas = self.sample_betas()
|
|
13
|
-
self.alpha_bars = self.compute_alpha_bar()
|
|
14
|
-
|
|
15
|
-
if enforce_zero_terminal_snr:
|
|
16
|
-
self.enforce_zero_terminal_snr()
|
|
17
|
-
|
|
18
|
-
def enforce_zero_terminal_snr(self):
|
|
19
|
-
alpha_bar_length = len(self.alpha_bars)
|
|
20
|
-
|
|
21
|
-
# Convert betas to alphas_bar_sqrt
|
|
22
|
-
alphas = 1 - self.betas
|
|
23
|
-
alphas_bar = alphas.cumprod(0)
|
|
24
|
-
alphas_bar_sqrt = alphas_bar.sqrt()
|
|
25
|
-
|
|
26
|
-
# Store old values.
|
|
27
|
-
alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
|
|
28
|
-
alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()
|
|
29
|
-
# Shift so last timestep is zero.
|
|
30
|
-
alphas_bar_sqrt -= alphas_bar_sqrt_T
|
|
31
|
-
# Scale so first timestep is back to old value.
|
|
32
|
-
alphas_bar_sqrt *= alphas_bar_sqrt_0 / (
|
|
33
|
-
alphas_bar_sqrt_0 - alphas_bar_sqrt_T
|
|
34
|
-
)
|
|
35
|
-
|
|
36
|
-
# Convert alphas_bar_sqrt to betas
|
|
37
|
-
alphas_bar = alphas_bar_sqrt**2
|
|
38
|
-
alphas = alphas_bar[1:] / alphas_bar[:-1]
|
|
39
|
-
alphas = torch.cat([alphas_bar[0:1], alphas])
|
|
40
|
-
betas = 1 - alphas
|
|
41
|
-
if len(alphas) == alpha_bar_length:
|
|
42
|
-
self.betas = betas
|
|
43
|
-
self.alpha_bars = alphas_bar
|
|
44
|
-
else:
|
|
45
|
-
logging.warning(
|
|
46
|
-
"Got different alpha_bar length after enforcing zero SNR. Please check your beta scheduler"
|
|
47
|
-
)
|
|
48
|
-
|
|
49
|
-
@abc.abstractmethod
|
|
50
|
-
def sample_betas(self):
|
|
51
|
-
pass
|
|
52
|
-
|
|
53
|
-
@abc.abstractmethod
|
|
54
|
-
def compute_alpha_bar(self):
|
|
55
|
-
pass
|
|
56
|
-
|
|
57
|
-
def to(self, device: str):
|
|
58
|
-
self.betas = self.betas.to(device)
|
|
59
|
-
self.alpha_bars = self.alpha_bars.to(device)
|
|
60
|
-
return self
|
|
61
|
-
|
|
62
|
-
@classmethod
|
|
63
|
-
def from_tensors(
|
|
64
|
-
cls, steps: int, betas: torch.Tensor, alpha_bars: torch.Tensor
|
|
65
|
-
):
|
|
66
|
-
generic_beta_scheduler = cls(0)
|
|
67
|
-
generic_beta_scheduler.steps = steps
|
|
68
|
-
generic_beta_scheduler.betas = betas
|
|
69
|
-
generic_beta_scheduler.alpha_bars = alpha_bars
|
|
70
|
-
return generic_beta_scheduler
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
class LinearBetaScheduler(BaseBetaScheduler):
|
|
74
|
-
def __init__(
|
|
75
|
-
self,
|
|
76
|
-
beta_start: float = 0.0001,
|
|
77
|
-
beta_end: float = 0.02,
|
|
78
|
-
steps: int = 1000,
|
|
79
|
-
enforce_zero_terminal_snr: bool = True,
|
|
80
|
-
):
|
|
81
|
-
self.beta_start = beta_start
|
|
82
|
-
self.beta_end = beta_end
|
|
83
|
-
super().__init__(
|
|
84
|
-
steps=steps,
|
|
85
|
-
enforce_zero_terminal_snr=enforce_zero_terminal_snr,
|
|
86
|
-
)
|
|
87
|
-
|
|
88
|
-
def sample_betas(self):
|
|
89
|
-
return torch.linspace(self.beta_start, self.beta_end, self.steps)
|
|
90
|
-
|
|
91
|
-
def compute_alpha_bar(self):
|
|
92
|
-
alphas = 1 - self.betas
|
|
93
|
-
alpha_bar = torch.cumprod(alphas, dim=0)
|
|
94
|
-
return alpha_bar
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
class CosineBetaScheduler(BaseBetaScheduler):
|
|
98
|
-
def __init__(
|
|
99
|
-
self,
|
|
100
|
-
offset: float = 0.008,
|
|
101
|
-
steps: int = 1000,
|
|
102
|
-
max_beta: Optional[float] = 0.999,
|
|
103
|
-
):
|
|
104
|
-
self.offset = offset
|
|
105
|
-
self.max_beta = max_beta
|
|
106
|
-
self.steps = steps
|
|
107
|
-
self._alpha_bars = self._compute_alpha_bar()
|
|
108
|
-
self._betas = self._compute_betas()
|
|
109
|
-
|
|
110
|
-
super().__init__(
|
|
111
|
-
steps=steps,
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
def f(self, t: torch.Tensor):
|
|
115
|
-
return (
|
|
116
|
-
torch.cos(
|
|
117
|
-
(((t / self.steps) + self.offset) / (1 + self.offset)) * (torch.pi / 2)
|
|
118
|
-
)
|
|
119
|
-
** 2
|
|
120
|
-
)
|
|
121
|
-
|
|
122
|
-
def _compute_betas(self):
|
|
123
|
-
betas = 1 - self._alpha_bars[1:] / self._alpha_bars[:-1]
|
|
124
|
-
if self.max_beta:
|
|
125
|
-
betas = torch.clip(betas, max=self.max_beta)
|
|
126
|
-
return betas
|
|
127
|
-
|
|
128
|
-
def _compute_alpha_bar(self):
|
|
129
|
-
t = torch.linspace(0, self.steps, self.steps, dtype=torch.float32)
|
|
130
|
-
return self.f(t) / self.f(torch.tensor([0], dtype=torch.float32))
|
|
131
|
-
|
|
132
|
-
def sample_betas(self):
|
|
133
|
-
return self._betas
|
|
134
|
-
|
|
135
|
-
def compute_alpha_bar(self):
|
|
136
|
-
return self._alpha_bars
|
|
File without changes
|