diffusion-model-framework 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffusion_model_framework-0.1.0/PKG-INFO +43 -0
- diffusion_model_framework-0.1.0/README.md +20 -0
- diffusion_model_framework-0.1.0/diffusion_models/__init__.py +1 -0
- diffusion_model_framework-0.1.0/diffusion_models/diffusion_trainer.py +168 -0
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/__init__.py +0 -0
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/base_diffuser.py +28 -0
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/beta_schedulers.py +136 -0
- diffusion_model_framework-0.1.0/diffusion_models/gaussian_diffusion/gaussian_diffuser.py +78 -0
- diffusion_model_framework-0.1.0/diffusion_models/models/SimpleUnet.py +126 -0
- diffusion_model_framework-0.1.0/diffusion_models/utils/__init__.py +0 -0
- diffusion_model_framework-0.1.0/diffusion_models/utils/schemas.py +102 -0
- diffusion_model_framework-0.1.0/diffusion_models/utils/tensorboard.py +24 -0
- diffusion_model_framework-0.1.0/pyproject.toml +124 -0
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: diffusion-model-framework
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary:
|
|
5
|
+
Home-page: https://github.com/sachahu1/Diffusion-Model-Framework
|
|
6
|
+
Author: Sacha Hu
|
|
7
|
+
Author-email: sachahu@mindsparkai.com
|
|
8
|
+
Requires-Python: >=3.9,<3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
13
|
+
Requires-Dist: numpy (>=1.26.4,<2.0.0)
|
|
14
|
+
Requires-Dist: pillow-simd (>=9.0.0.post1,<10.0.0)
|
|
15
|
+
Requires-Dist: tensorboard (>=2.17.0,<3.0.0)
|
|
16
|
+
Requires-Dist: torch (>=2.4.0,<3.0.0)
|
|
17
|
+
Requires-Dist: torchvision (>=0.19.0,<0.20.0)
|
|
18
|
+
Requires-Dist: tqdm (>=4.66.2,<5.0.0)
|
|
19
|
+
Project-URL: Documentation, https://www.sachahu.com/docs/diffusion-model-framework/latest/index.html
|
|
20
|
+
Project-URL: Repository, https://github.com/sachahu1/Diffusion-Model-Framework
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
|
|
23
|
+
# Diffusion Model Playground
|
|
24
|
+
|
|
25
|
+

|
|
26
|
+

|
|
27
|
+

|
|
28
|
+

|
|
29
|
+
|
|
30
|
+
This repo is a playground for implementing, training and visualising diffusion
|
|
31
|
+
models.
|
|
32
|
+
|
|
33
|
+

|
|
34
|
+
|
|
35
|
+
# Getting Started
|
|
36
|
+
This tool uses the [poetry python manager](https://python-poetry.org/).
|
|
37
|
+
To install the dependencies, simply run `poetry install`.
|
|
38
|
+
|
|
39
|
+
## Examples
|
|
40
|
+
To get started with this codebase, head to the [examples](./examples) folder to
|
|
41
|
+
find a simple example to get started with training your own model along with
|
|
42
|
+
an example to easily run inference.
|
|
43
|
+
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Diffusion Model Playground
|
|
2
|
+
|
|
3
|
+

|
|
4
|
+

|
|
5
|
+

|
|
6
|
+

|
|
7
|
+
|
|
8
|
+
This repo is a playground for implementing, training and visualising diffusion
|
|
9
|
+
models.
|
|
10
|
+
|
|
11
|
+

|
|
12
|
+
|
|
13
|
+
# Getting Started
|
|
14
|
+
This tool uses the [poetry python manager](https://python-poetry.org/).
|
|
15
|
+
To install the dependencies, simply run `poetry install`.
|
|
16
|
+
|
|
17
|
+
## Examples
|
|
18
|
+
To get started with this codebase, head to the [examples](./examples) folder to
|
|
19
|
+
find a simple example to get started with training your own model along with
|
|
20
|
+
an example to easily run inference.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import pathlib
|
|
2
|
+
from typing import Callable
|
|
3
|
+
from typing import Dict
|
|
4
|
+
from typing import Optional
|
|
5
|
+
|
|
6
|
+
import torch
|
|
7
|
+
from torch.nn import functional as F
|
|
8
|
+
from torch.utils.data import DataLoader
|
|
9
|
+
from torch.utils.data import Dataset
|
|
10
|
+
from tqdm import tqdm
|
|
11
|
+
|
|
12
|
+
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
13
|
+
from diffusion_models.utils.schemas import BetaSchedulerConfiguration
|
|
14
|
+
from diffusion_models.utils.schemas import Checkpoint
|
|
15
|
+
from diffusion_models.utils.schemas import LogConfiguration
|
|
16
|
+
from diffusion_models.utils.schemas import TrainingConfiguration
|
|
17
|
+
from diffusion_models.utils.tensorboard import TensorboardManager
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class DiffusionTrainer:
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
model: BaseDiffusionModel,
|
|
24
|
+
dataset: Dataset,
|
|
25
|
+
optimizer: torch.optim.Optimizer,
|
|
26
|
+
training_configuration: TrainingConfiguration,
|
|
27
|
+
loss_function: Callable = F.l1_loss,
|
|
28
|
+
scheduler: Optional[torch.optim.lr_scheduler.LRScheduler] = None,
|
|
29
|
+
log_configuration: LogConfiguration = LogConfiguration(),
|
|
30
|
+
reverse_transforms: Callable = lambda x: x,
|
|
31
|
+
device: str = "cuda",
|
|
32
|
+
):
|
|
33
|
+
self.model = model.to(device)
|
|
34
|
+
self.optimizer = optimizer
|
|
35
|
+
self.loss_function = loss_function
|
|
36
|
+
self.training_configuration = training_configuration
|
|
37
|
+
self.scheduler = scheduler
|
|
38
|
+
self.device = device
|
|
39
|
+
|
|
40
|
+
self.dataloader = DataLoader(
|
|
41
|
+
dataset=dataset,
|
|
42
|
+
batch_size=training_configuration.batch_size,
|
|
43
|
+
shuffle=True,
|
|
44
|
+
drop_last=True,
|
|
45
|
+
num_workers=16,
|
|
46
|
+
pin_memory=True,
|
|
47
|
+
persistent_workers=True,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
self._image_shape = dataset[0][0].shape
|
|
51
|
+
|
|
52
|
+
self.scaler = torch.amp.GradScaler(
|
|
53
|
+
device=device
|
|
54
|
+
# init_scale=8192,
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
self.log_configuration = log_configuration
|
|
58
|
+
|
|
59
|
+
self.checkpoint_path = (
|
|
60
|
+
pathlib.Path("../checkpoints")
|
|
61
|
+
/ self.training_configuration.training_name
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
self.checkpoint_path.mkdir(exist_ok=True)
|
|
65
|
+
self.tensorboard_manager = TensorboardManager(
|
|
66
|
+
log_name=self.training_configuration.training_name,
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
self.reverse_transforms = reverse_transforms
|
|
70
|
+
|
|
71
|
+
torch.backends.cudnn.benchmark = True
|
|
72
|
+
|
|
73
|
+
def save_checkpoint(self, epoch: int, checkpoint_name: str):
|
|
74
|
+
checkpoint = Checkpoint(
|
|
75
|
+
epoch=epoch,
|
|
76
|
+
model_state_dict=self.model.state_dict(),
|
|
77
|
+
optimizer_state_dict=self.optimizer.state_dict(),
|
|
78
|
+
scaler=self.scaler.state_dict()
|
|
79
|
+
if self.training_configuration.mixed_precision_training
|
|
80
|
+
else None,
|
|
81
|
+
image_channels=self._image_shape[0],
|
|
82
|
+
beta_scheduler_config=BetaSchedulerConfiguration(
|
|
83
|
+
steps=self.model.diffuser.beta_scheduler.steps,
|
|
84
|
+
betas=self.model.diffuser.beta_scheduler.betas,
|
|
85
|
+
alpha_bars=self.model.diffuser.beta_scheduler.alpha_bars,
|
|
86
|
+
),
|
|
87
|
+
tensorboard_run_name=self.tensorboard_manager.summary_writer.log_dir,
|
|
88
|
+
)
|
|
89
|
+
checkpoint.to_file(self.checkpoint_path / checkpoint_name)
|
|
90
|
+
|
|
91
|
+
def train(self):
|
|
92
|
+
self.model.train()
|
|
93
|
+
for epoch in range(self.training_configuration.number_of_epochs):
|
|
94
|
+
for step, batch in enumerate(
|
|
95
|
+
tqdm(self.dataloader, desc=f"Epoch={epoch}")
|
|
96
|
+
):
|
|
97
|
+
global_step = epoch * len(self.dataloader) + step
|
|
98
|
+
|
|
99
|
+
images, _ = batch
|
|
100
|
+
images = images.to(self.device)
|
|
101
|
+
|
|
102
|
+
noisy_images, noise, timesteps = self.model.diffuse(images=images)
|
|
103
|
+
|
|
104
|
+
self.optimizer.zero_grad(set_to_none=True)
|
|
105
|
+
|
|
106
|
+
with torch.autocast(
|
|
107
|
+
device_type=self.device,
|
|
108
|
+
enabled=self.training_configuration.mixed_precision_training,
|
|
109
|
+
):
|
|
110
|
+
prediction = self.model(noisy_images, timesteps)
|
|
111
|
+
loss = self.loss_function(noise, prediction)
|
|
112
|
+
|
|
113
|
+
self.scaler.scale(loss).backward()
|
|
114
|
+
|
|
115
|
+
if self.training_configuration.gradient_clip is not None:
|
|
116
|
+
# Unscales the gradients of optimizer's assigned params in-place
|
|
117
|
+
self.scaler.unscale_(self.optimizer)
|
|
118
|
+
|
|
119
|
+
# Since the gradients of optimizer's assigned params are unscaled, clips as usual:
|
|
120
|
+
torch.nn.utils.clip_grad_norm_(
|
|
121
|
+
self.model.parameters(),
|
|
122
|
+
max_norm=self.training_configuration.gradient_clip,
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
self.scaler.step(self.optimizer)
|
|
126
|
+
self.scaler.update()
|
|
127
|
+
|
|
128
|
+
self.log_to_tensorboard(
|
|
129
|
+
metrics={
|
|
130
|
+
"Loss": loss,
|
|
131
|
+
},
|
|
132
|
+
global_step=global_step,
|
|
133
|
+
)
|
|
134
|
+
if epoch % self.training_configuration.checkpoint_rate == 0:
|
|
135
|
+
self.save_checkpoint(epoch=epoch, checkpoint_name=f"epoch_{epoch}.pt")
|
|
136
|
+
self.save_checkpoint(
|
|
137
|
+
epoch=self.training_configuration.number_of_epochs,
|
|
138
|
+
checkpoint_name="final.pt",
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
@torch.no_grad()
|
|
142
|
+
def log_to_tensorboard(self, metrics: Dict[str, float], global_step: int):
|
|
143
|
+
self.model.eval()
|
|
144
|
+
if global_step % self.log_configuration.log_rate == 0:
|
|
145
|
+
self.tensorboard_manager.log_metrics(
|
|
146
|
+
metrics=metrics, global_step=global_step
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
if (global_step % self.log_configuration.image_rate == 0) and (
|
|
150
|
+
self.log_configuration.number_of_images > 0
|
|
151
|
+
):
|
|
152
|
+
image_channels, image_height, image_width = self._image_shape
|
|
153
|
+
images = torch.randn(
|
|
154
|
+
(
|
|
155
|
+
self.log_configuration.number_of_images,
|
|
156
|
+
image_channels,
|
|
157
|
+
image_height,
|
|
158
|
+
image_width,
|
|
159
|
+
),
|
|
160
|
+
device=self.device,
|
|
161
|
+
)
|
|
162
|
+
images = self.model.denoise(images)
|
|
163
|
+
for step, images in enumerate(images[::-1]):
|
|
164
|
+
self.tensorboard_manager.log_images(
|
|
165
|
+
tag=f"Images at timestep {global_step}",
|
|
166
|
+
images=self.reverse_transforms(images),
|
|
167
|
+
timestep=step,
|
|
168
|
+
)
|
|
File without changes
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
from typing import List
|
|
3
|
+
from typing import Tuple
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
8
|
+
BaseBetaScheduler,
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class BaseDiffuser(abc.ABC):
|
|
13
|
+
def __init__(self, beta_scheduler: BaseBetaScheduler):
|
|
14
|
+
self.beta_scheduler = beta_scheduler
|
|
15
|
+
|
|
16
|
+
@abc.abstractmethod
|
|
17
|
+
def diffuse_batch(
|
|
18
|
+
self, images: torch.Tensor
|
|
19
|
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
20
|
+
pass
|
|
21
|
+
|
|
22
|
+
@abc.abstractmethod
|
|
23
|
+
def denoise_batch(self, images: torch.Tensor, model) -> List[torch.Tensor]:
|
|
24
|
+
pass
|
|
25
|
+
|
|
26
|
+
@abc.abstractmethod
|
|
27
|
+
def to(self, device: str = "cpu"):
|
|
28
|
+
pass
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import abc
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Optional
|
|
4
|
+
|
|
5
|
+
import torch
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class BaseBetaScheduler:
|
|
9
|
+
def __init__(self, steps: int, enforce_zero_terminal_snr: bool = False):
|
|
10
|
+
super().__init__()
|
|
11
|
+
self.steps = steps
|
|
12
|
+
self.betas = self.sample_betas()
|
|
13
|
+
self.alpha_bars = self.compute_alpha_bar()
|
|
14
|
+
|
|
15
|
+
if enforce_zero_terminal_snr:
|
|
16
|
+
self.enforce_zero_terminal_snr()
|
|
17
|
+
|
|
18
|
+
def enforce_zero_terminal_snr(self):
|
|
19
|
+
alpha_bar_length = len(self.alpha_bars)
|
|
20
|
+
|
|
21
|
+
# Convert betas to alphas_bar_sqrt
|
|
22
|
+
alphas = 1 - self.betas
|
|
23
|
+
alphas_bar = alphas.cumprod(0)
|
|
24
|
+
alphas_bar_sqrt = alphas_bar.sqrt()
|
|
25
|
+
|
|
26
|
+
# Store old values.
|
|
27
|
+
alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone()
|
|
28
|
+
alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone()
|
|
29
|
+
# Shift so last timestep is zero.
|
|
30
|
+
alphas_bar_sqrt -= alphas_bar_sqrt_T
|
|
31
|
+
# Scale so first timestep is back to old value.
|
|
32
|
+
alphas_bar_sqrt *= alphas_bar_sqrt_0 / (
|
|
33
|
+
alphas_bar_sqrt_0 - alphas_bar_sqrt_T
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
# Convert alphas_bar_sqrt to betas
|
|
37
|
+
alphas_bar = alphas_bar_sqrt**2
|
|
38
|
+
alphas = alphas_bar[1:] / alphas_bar[:-1]
|
|
39
|
+
alphas = torch.cat([alphas_bar[0:1], alphas])
|
|
40
|
+
betas = 1 - alphas
|
|
41
|
+
if len(alphas) == alpha_bar_length:
|
|
42
|
+
self.betas = betas
|
|
43
|
+
self.alpha_bars = alphas_bar
|
|
44
|
+
else:
|
|
45
|
+
logging.warning(
|
|
46
|
+
"Got different alpha_bar length after enforcing zero SNR. Please check your beta scheduler"
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
@abc.abstractmethod
|
|
50
|
+
def sample_betas(self):
|
|
51
|
+
pass
|
|
52
|
+
|
|
53
|
+
@abc.abstractmethod
|
|
54
|
+
def compute_alpha_bar(self):
|
|
55
|
+
pass
|
|
56
|
+
|
|
57
|
+
def to(self, device: str):
|
|
58
|
+
self.betas = self.betas.to(device)
|
|
59
|
+
self.alpha_bars = self.alpha_bars.to(device)
|
|
60
|
+
return self
|
|
61
|
+
|
|
62
|
+
@classmethod
|
|
63
|
+
def from_tensors(
|
|
64
|
+
cls, steps: int, betas: torch.Tensor, alpha_bars: torch.Tensor
|
|
65
|
+
):
|
|
66
|
+
generic_beta_scheduler = cls(0)
|
|
67
|
+
generic_beta_scheduler.steps = steps
|
|
68
|
+
generic_beta_scheduler.betas = betas
|
|
69
|
+
generic_beta_scheduler.alpha_bars = alpha_bars
|
|
70
|
+
return generic_beta_scheduler
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class LinearBetaScheduler(BaseBetaScheduler):
|
|
74
|
+
def __init__(
|
|
75
|
+
self,
|
|
76
|
+
beta_start: float = 0.0001,
|
|
77
|
+
beta_end: float = 0.02,
|
|
78
|
+
steps: int = 1000,
|
|
79
|
+
enforce_zero_terminal_snr: bool = True,
|
|
80
|
+
):
|
|
81
|
+
self.beta_start = beta_start
|
|
82
|
+
self.beta_end = beta_end
|
|
83
|
+
super().__init__(
|
|
84
|
+
steps=steps,
|
|
85
|
+
enforce_zero_terminal_snr=enforce_zero_terminal_snr,
|
|
86
|
+
)
|
|
87
|
+
|
|
88
|
+
def sample_betas(self):
|
|
89
|
+
return torch.linspace(self.beta_start, self.beta_end, self.steps)
|
|
90
|
+
|
|
91
|
+
def compute_alpha_bar(self):
|
|
92
|
+
alphas = 1 - self.betas
|
|
93
|
+
alpha_bar = torch.cumprod(alphas, dim=0)
|
|
94
|
+
return alpha_bar
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class CosineBetaScheduler(BaseBetaScheduler):
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
offset: float = 0.008,
|
|
101
|
+
steps: int = 1000,
|
|
102
|
+
max_beta: Optional[float] = 0.999,
|
|
103
|
+
):
|
|
104
|
+
self.offset = offset
|
|
105
|
+
self.max_beta = max_beta
|
|
106
|
+
self.steps = steps
|
|
107
|
+
self._alpha_bars = self._compute_alpha_bar()
|
|
108
|
+
self._betas = self._compute_betas()
|
|
109
|
+
|
|
110
|
+
super().__init__(
|
|
111
|
+
steps=steps,
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
def f(self, t: torch.Tensor):
|
|
115
|
+
return (
|
|
116
|
+
torch.cos(
|
|
117
|
+
(((t / self.steps) + self.offset) / (1 + self.offset)) * (torch.pi / 2)
|
|
118
|
+
)
|
|
119
|
+
** 2
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
def _compute_betas(self):
|
|
123
|
+
betas = 1 - self._alpha_bars[1:] / self._alpha_bars[:-1]
|
|
124
|
+
if self.max_beta:
|
|
125
|
+
betas = torch.clip(betas, max=self.max_beta)
|
|
126
|
+
return betas
|
|
127
|
+
|
|
128
|
+
def _compute_alpha_bar(self):
|
|
129
|
+
t = torch.linspace(0, self.steps, self.steps, dtype=torch.float32)
|
|
130
|
+
return self.f(t) / self.f(torch.tensor([0], dtype=torch.float32))
|
|
131
|
+
|
|
132
|
+
def sample_betas(self):
|
|
133
|
+
return self._betas
|
|
134
|
+
|
|
135
|
+
def compute_alpha_bar(self):
|
|
136
|
+
return self._alpha_bars
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
|
|
3
|
+
import torch
|
|
4
|
+
from tqdm import tqdm
|
|
5
|
+
|
|
6
|
+
from diffusion_models.gaussian_diffusion.base_diffuser import BaseDiffuser
|
|
7
|
+
from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
8
|
+
BaseBetaScheduler,
|
|
9
|
+
)
|
|
10
|
+
from diffusion_models.utils.schemas import Checkpoint
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class GaussianDiffuser(BaseDiffuser):
|
|
14
|
+
def __init__(self, beta_scheduler: BaseBetaScheduler):
|
|
15
|
+
super().__init__(beta_scheduler)
|
|
16
|
+
self.device = "cuda"
|
|
17
|
+
|
|
18
|
+
@classmethod
|
|
19
|
+
def from_checkpoint(cls, checkpoint: Checkpoint) -> "GaussianDiffuser":
|
|
20
|
+
return cls(
|
|
21
|
+
beta_scheduler=BaseBetaScheduler.from_tensors(
|
|
22
|
+
steps=checkpoint.beta_scheduler_config.steps,
|
|
23
|
+
betas=checkpoint.beta_scheduler_config.betas,
|
|
24
|
+
alpha_bars=checkpoint.beta_scheduler_config.alpha_bars,
|
|
25
|
+
)
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
def to(self, device: str = "cpu"):
|
|
29
|
+
self.device = device
|
|
30
|
+
self.beta_scheduler = self.beta_scheduler.to(self.device)
|
|
31
|
+
return self
|
|
32
|
+
|
|
33
|
+
def diffuse_batch(self, images):
|
|
34
|
+
timesteps = torch.randint(
|
|
35
|
+
0, self.beta_scheduler.steps, (images.shape[0],), device=self.device
|
|
36
|
+
)
|
|
37
|
+
noise = torch.randn_like(images, device=self.device)
|
|
38
|
+
|
|
39
|
+
alpha_bar_t = self.beta_scheduler.alpha_bars.gather(dim=0, index=timesteps)
|
|
40
|
+
|
|
41
|
+
alpha_bar_t = alpha_bar_t.reshape((-1, *((1,) * (len(images.shape) - 1))))
|
|
42
|
+
|
|
43
|
+
mu = torch.sqrt(alpha_bar_t)
|
|
44
|
+
sigma = torch.sqrt(1 - alpha_bar_t)
|
|
45
|
+
images = mu * images + sigma * noise
|
|
46
|
+
return images, noise, timesteps
|
|
47
|
+
|
|
48
|
+
@torch.no_grad()
|
|
49
|
+
def _denoise_step(
|
|
50
|
+
self, images: torch.Tensor, model: torch.nn.Module, timestep: torch.Tensor
|
|
51
|
+
) -> torch.Tensor:
|
|
52
|
+
beta_t = self.beta_scheduler.betas[timestep].reshape(-1, 1, 1, 1)
|
|
53
|
+
alpha_t = 1 - beta_t
|
|
54
|
+
alpha_bar_t = self.beta_scheduler.alpha_bars.gather(
|
|
55
|
+
dim=0, index=timestep
|
|
56
|
+
).reshape(-1, 1, 1, 1)
|
|
57
|
+
mu = (1 / torch.sqrt(alpha_t)) * (
|
|
58
|
+
images - model(images, timestep) * (beta_t / torch.sqrt(1 - alpha_bar_t))
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
if timestep[0] == 0:
|
|
62
|
+
return mu
|
|
63
|
+
else:
|
|
64
|
+
sigma = torch.sqrt(beta_t) * torch.randn_like(images)
|
|
65
|
+
return mu + sigma
|
|
66
|
+
|
|
67
|
+
def denoise_batch(
|
|
68
|
+
self,
|
|
69
|
+
images: torch.Tensor,
|
|
70
|
+
model: torch.nn.Module,
|
|
71
|
+
) -> List[torch.Tensor]:
|
|
72
|
+
denoised_images = []
|
|
73
|
+
for i in tqdm(range(self.beta_scheduler.steps)[::-1], desc="Denoising"):
|
|
74
|
+
timestep = torch.full((images.shape[0],), i, device=self.device)
|
|
75
|
+
images = self._denoise_step(images, model=model, timestep=timestep)
|
|
76
|
+
images = torch.clamp(images, -1.0, 1.0)
|
|
77
|
+
denoised_images.append(images)
|
|
78
|
+
return denoised_images
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# Diffusion Model taken from DeepFindr (https://www.youtube.com/@DeepFindr)
|
|
2
|
+
import math
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch import nn
|
|
6
|
+
|
|
7
|
+
from diffusion_models.gaussian_diffusion.gaussian_diffuser import (
|
|
8
|
+
GaussianDiffuser,
|
|
9
|
+
)
|
|
10
|
+
from diffusion_models.models.base_diffusion_model import BaseDiffusionModel
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class Block(nn.Module):
|
|
14
|
+
def __init__(self, in_ch, out_ch, time_emb_dim, up=False):
|
|
15
|
+
super().__init__()
|
|
16
|
+
self.time_mlp = nn.Linear(time_emb_dim, out_ch)
|
|
17
|
+
if up:
|
|
18
|
+
self.conv1 = nn.Conv2d(2 * in_ch, out_ch, 3, padding=1)
|
|
19
|
+
self.transform = nn.ConvTranspose2d(out_ch, out_ch, 4, 2, 1)
|
|
20
|
+
else:
|
|
21
|
+
self.conv1 = nn.Conv2d(in_ch, out_ch, 3, padding=1)
|
|
22
|
+
self.transform = nn.Conv2d(out_ch, out_ch, 4, 2, 1)
|
|
23
|
+
self.conv2 = nn.Conv2d(out_ch, out_ch, 3, padding=1)
|
|
24
|
+
self.bnorm1 = nn.BatchNorm2d(out_ch)
|
|
25
|
+
self.bnorm2 = nn.BatchNorm2d(out_ch)
|
|
26
|
+
self.relu = nn.ReLU()
|
|
27
|
+
|
|
28
|
+
def forward(
|
|
29
|
+
self,
|
|
30
|
+
x,
|
|
31
|
+
t,
|
|
32
|
+
):
|
|
33
|
+
# First Conv
|
|
34
|
+
h = self.bnorm1(self.relu(self.conv1(x)))
|
|
35
|
+
# Time embedding
|
|
36
|
+
time_emb = self.relu(self.time_mlp(t))
|
|
37
|
+
# Extend last 2 dimensions
|
|
38
|
+
time_emb = time_emb[(...,) + (None,) * 2]
|
|
39
|
+
# Add time channel
|
|
40
|
+
h = h + time_emb
|
|
41
|
+
# Second Conv
|
|
42
|
+
h = self.bnorm2(self.relu(self.conv2(h)))
|
|
43
|
+
# Down or Upsample
|
|
44
|
+
return self.transform(h)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class SinusoidalPositionEmbeddings(nn.Module):
|
|
48
|
+
def __init__(self, dim):
|
|
49
|
+
super().__init__()
|
|
50
|
+
self.dim = dim
|
|
51
|
+
|
|
52
|
+
def forward(self, time):
|
|
53
|
+
device = time.device
|
|
54
|
+
half_dim = self.dim // 2
|
|
55
|
+
embeddings = math.log(10000) / (half_dim - 1)
|
|
56
|
+
embeddings = torch.exp(torch.arange(half_dim, device=device) * -embeddings)
|
|
57
|
+
embeddings = time[:, None] * embeddings[None, :]
|
|
58
|
+
embeddings = torch.cat((embeddings.sin(), embeddings.cos()), dim=-1)
|
|
59
|
+
# TODO: Double check the ordering here
|
|
60
|
+
return embeddings
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class SimpleUnet(BaseDiffusionModel):
|
|
64
|
+
"""
|
|
65
|
+
A simplified variant of the Unet architecture.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
def __init__(
|
|
69
|
+
self,
|
|
70
|
+
diffuser: GaussianDiffuser,
|
|
71
|
+
image_channels: int,
|
|
72
|
+
):
|
|
73
|
+
super().__init__(diffuser=diffuser)
|
|
74
|
+
image_channels = image_channels
|
|
75
|
+
down_channels = (64, 128, 256, 512, 1024)
|
|
76
|
+
up_channels = (1024, 512, 256, 128, 64)
|
|
77
|
+
out_dim = image_channels
|
|
78
|
+
time_emb_dim = 32
|
|
79
|
+
|
|
80
|
+
# Time embedding
|
|
81
|
+
self.time_mlp = nn.Sequential(
|
|
82
|
+
SinusoidalPositionEmbeddings(time_emb_dim),
|
|
83
|
+
nn.Linear(time_emb_dim, time_emb_dim),
|
|
84
|
+
nn.ReLU(),
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
# Initial projection
|
|
88
|
+
self.conv0 = nn.Conv2d(image_channels, down_channels[0], 3, padding=1)
|
|
89
|
+
|
|
90
|
+
# Downsample
|
|
91
|
+
self.downs = nn.ModuleList(
|
|
92
|
+
[
|
|
93
|
+
Block(down_channels[i], down_channels[i + 1], time_emb_dim)
|
|
94
|
+
for i in range(len(down_channels) - 1)
|
|
95
|
+
]
|
|
96
|
+
)
|
|
97
|
+
# Upsample
|
|
98
|
+
self.ups = nn.ModuleList(
|
|
99
|
+
[
|
|
100
|
+
Block(up_channels[i], up_channels[i + 1], time_emb_dim, up=True)
|
|
101
|
+
for i in range(len(up_channels) - 1)
|
|
102
|
+
]
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
# Edit: Corrected a bug found by Jakub C (see YouTube comment)
|
|
106
|
+
self.output = nn.Conv2d(up_channels[-1], out_dim, 1)
|
|
107
|
+
|
|
108
|
+
def forward(self, x, timestep):
|
|
109
|
+
# if self.training:
|
|
110
|
+
# x = self.augmentations(x)
|
|
111
|
+
|
|
112
|
+
# Embedd time
|
|
113
|
+
t = self.time_mlp(timestep)
|
|
114
|
+
# Initial conv
|
|
115
|
+
x = self.conv0(x)
|
|
116
|
+
# Unet
|
|
117
|
+
residual_inputs = []
|
|
118
|
+
for down in self.downs:
|
|
119
|
+
x = down(x, t)
|
|
120
|
+
residual_inputs.append(x)
|
|
121
|
+
for up in self.ups:
|
|
122
|
+
residual_x = residual_inputs.pop()
|
|
123
|
+
# Add residual x as additional channels
|
|
124
|
+
x = torch.cat((x, residual_x), dim=1)
|
|
125
|
+
x = up(x, t)
|
|
126
|
+
return self.output(x)
|
|
File without changes
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import dataclasses
|
|
2
|
+
import pathlib
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
from typing import Any
|
|
5
|
+
from typing import Dict
|
|
6
|
+
from typing import Optional
|
|
7
|
+
from typing import Union
|
|
8
|
+
|
|
9
|
+
import torch
|
|
10
|
+
from torch.cuda.amp import GradScaler
|
|
11
|
+
|
|
12
|
+
from diffusion_models.gaussian_diffusion.beta_schedulers import (
|
|
13
|
+
BaseBetaScheduler,
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class TrainingConfiguration:
|
|
19
|
+
training_name: str
|
|
20
|
+
batch_size: int
|
|
21
|
+
learning_rate: float
|
|
22
|
+
number_of_epochs: int
|
|
23
|
+
|
|
24
|
+
log_rate: int = 10
|
|
25
|
+
image_rate: int = 100
|
|
26
|
+
checkpoint_rate: int = 100
|
|
27
|
+
|
|
28
|
+
mixed_precision_training: bool = False # TODO: This is not complete yet
|
|
29
|
+
gradient_clip: Optional[float] = None # TODO: This is not complete yet
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass
|
|
33
|
+
class LogConfiguration:
|
|
34
|
+
log_rate: int = 10
|
|
35
|
+
image_rate: int = 50
|
|
36
|
+
number_of_images: int = 5
|
|
37
|
+
# metrics: Dict[str, float] # TODO: consider Dict[str, Callable]
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass
|
|
41
|
+
class BetaSchedulerConfiguration:
|
|
42
|
+
steps: int
|
|
43
|
+
betas: torch.Tensor
|
|
44
|
+
alpha_bars: torch.Tensor
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass
|
|
48
|
+
class Checkpoint:
|
|
49
|
+
epoch: int
|
|
50
|
+
model_state_dict: Dict[str, Any]
|
|
51
|
+
optimizer_state_dict: Dict[str, Any]
|
|
52
|
+
scaler: Optional[GradScaler]
|
|
53
|
+
beta_scheduler_config: BetaSchedulerConfiguration
|
|
54
|
+
tensorboard_run_name: Optional[str] = None
|
|
55
|
+
image_channels: int = 3
|
|
56
|
+
loss: Optional[float] = (
|
|
57
|
+
None # TODO: remove legacy parameter and resave models
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
@classmethod
|
|
61
|
+
def from_file(cls, file_path: str) -> "Checkpoint":
|
|
62
|
+
checkpoint = torch.load(f=file_path)
|
|
63
|
+
checkpoint = cls(**checkpoint)
|
|
64
|
+
beta_scheduler_config = BetaSchedulerConfiguration(
|
|
65
|
+
**checkpoint.beta_scheduler_config
|
|
66
|
+
)
|
|
67
|
+
checkpoint.beta_scheduler_config = beta_scheduler_config
|
|
68
|
+
return checkpoint
|
|
69
|
+
|
|
70
|
+
def to_file(self, file_path: Union[str, pathlib.Path]) -> None:
|
|
71
|
+
torch.save(dataclasses.asdict(self), file_path)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@dataclass
|
|
75
|
+
class OldCheckpoint:
|
|
76
|
+
epoch: int
|
|
77
|
+
model_state_dict: Dict[str, Any]
|
|
78
|
+
optimizer_state_dict: Dict[str, Any]
|
|
79
|
+
scaler: Optional[GradScaler]
|
|
80
|
+
# beta_scheduler_config: BetaSchedulerConfiguration
|
|
81
|
+
tensorboard_run_name: Optional[str] = None
|
|
82
|
+
loss: Optional[float] = (
|
|
83
|
+
None # TODO: remove legacy parameter and resave models
|
|
84
|
+
)
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_file(cls, file_path: str) -> "OldCheckpoint":
|
|
88
|
+
checkpoint = torch.load(f=file_path)
|
|
89
|
+
return cls(**checkpoint)
|
|
90
|
+
|
|
91
|
+
def to_file(self, file_path: Union[str, pathlib.Path]) -> None:
|
|
92
|
+
torch.save(dataclasses.asdict(self), file_path)
|
|
93
|
+
|
|
94
|
+
def to_new_checkpoint(self, beta_scheduler: BaseBetaScheduler) -> Checkpoint:
|
|
95
|
+
beta_scheduler_config = BetaSchedulerConfiguration(
|
|
96
|
+
steps=beta_scheduler.steps,
|
|
97
|
+
betas=beta_scheduler.betas,
|
|
98
|
+
alpha_bars=beta_scheduler.alpha_bars,
|
|
99
|
+
)
|
|
100
|
+
return Checkpoint(
|
|
101
|
+
**dataclasses.asdict(self), beta_scheduler_config=beta_scheduler_config
|
|
102
|
+
)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
from typing import Dict
|
|
2
|
+
from typing import Optional
|
|
3
|
+
|
|
4
|
+
import torch
|
|
5
|
+
from torch.utils.tensorboard import SummaryWriter
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TensorboardManager:
|
|
9
|
+
def __init__(self, log_name: Optional[str] = None):
|
|
10
|
+
if log_name is None:
|
|
11
|
+
log_name = "fill this" ##########
|
|
12
|
+
self.log_directory = f"../runs/{log_name}"
|
|
13
|
+
self.summary_writer = SummaryWriter(log_dir=self.log_directory)
|
|
14
|
+
|
|
15
|
+
def log_metrics(self, metrics: Dict[str, float], global_step: int):
|
|
16
|
+
for metric_name, value in metrics.items():
|
|
17
|
+
self.summary_writer.add_scalar(
|
|
18
|
+
metric_name, value, global_step=global_step
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
def log_images(self, tag: str, images: torch.Tensor, timestep: int):
|
|
22
|
+
self.summary_writer.add_images(
|
|
23
|
+
tag=tag, img_tensor=images, global_step=timestep
|
|
24
|
+
)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "diffusion-model-framework"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = ""
|
|
5
|
+
authors = ["Sacha Hu <sachahu@mindsparkai.com>"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
documentation = "https://www.sachahu.com/docs/diffusion-model-framework/latest/index.html"
|
|
8
|
+
repository = "https://github.com/sachahu1/Diffusion-Model-Framework"
|
|
9
|
+
packages = [
|
|
10
|
+
{ include = "diffusion_models" }
|
|
11
|
+
]
|
|
12
|
+
|
|
13
|
+
[tool.poetry.dependencies]
|
|
14
|
+
python = ">=3.9,<3.12"
|
|
15
|
+
pillow-simd = "^9.0.0.post1"
|
|
16
|
+
tqdm = "^4.66.2"
|
|
17
|
+
numpy = "^1.26.4"
|
|
18
|
+
torchvision = "^0.19.0"
|
|
19
|
+
torch = "^2.4.0"
|
|
20
|
+
tensorboard = "^2.17.0"
|
|
21
|
+
|
|
22
|
+
[tool.poetry.group.gdown]
|
|
23
|
+
optional = true
|
|
24
|
+
|
|
25
|
+
[tool.poetry.group.gdown.dependencies]
|
|
26
|
+
gdown = "^5.1.0"
|
|
27
|
+
|
|
28
|
+
[tool.poetry.group.dev]
|
|
29
|
+
optional = true
|
|
30
|
+
|
|
31
|
+
[tool.poetry.group.dev.dependencies]
|
|
32
|
+
ruff = "^0.5.5"
|
|
33
|
+
|
|
34
|
+
[tool.poetry.group.test]
|
|
35
|
+
optional = true
|
|
36
|
+
|
|
37
|
+
[tool.poetry.group.test.dependencies]
|
|
38
|
+
pytest = "^6.2.5"
|
|
39
|
+
pytest-mock = "^3.14.0"
|
|
40
|
+
coverage = {version = "^6.4.2", extras = ["toml"]}
|
|
41
|
+
|
|
42
|
+
[tool.poetry.group.documentation]
|
|
43
|
+
optional = true
|
|
44
|
+
|
|
45
|
+
[tool.poetry.group.documentation.dependencies]
|
|
46
|
+
Sphinx = "^7.3.7"
|
|
47
|
+
sphinxcontrib-napoleon = "^0.7"
|
|
48
|
+
sphinx-rtd-theme = "^2.0.0"
|
|
49
|
+
myst-parser = "^3.0.1"
|
|
50
|
+
sphinx-copybutton = "^0.5.2"
|
|
51
|
+
sphinx-autoapi = "^3.2.1"
|
|
52
|
+
gitpython = "^3.1.43"
|
|
53
|
+
|
|
54
|
+
[build-system]
|
|
55
|
+
requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
|
|
56
|
+
build-backend = "poetry_dynamic_versioning.backend"
|
|
57
|
+
|
|
58
|
+
[tool.ruff]
|
|
59
|
+
line-length = 79
|
|
60
|
+
indent-width = 2
|
|
61
|
+
exclude = [".venv", ".env", ".git", "tests"]
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
# Add the `line-too-long` rule to the enforced rule set. By default, Ruff omits rules that
|
|
65
|
+
# overlap with the use of a formatter, like Black, but we can override this behavior by
|
|
66
|
+
# explicitly adding the rule.
|
|
67
|
+
extend-select = [
|
|
68
|
+
"B", # flake8-bugbear
|
|
69
|
+
"C4", # flake8-comprehensions
|
|
70
|
+
"ERA", # flake8-eradicate/eradicate
|
|
71
|
+
"I", # isort
|
|
72
|
+
"N", # pep8-naming
|
|
73
|
+
"PIE", # flake8-pie
|
|
74
|
+
"PGH", # pygrep
|
|
75
|
+
"RUF", # ruff checks
|
|
76
|
+
"SIM", # flake8-simplify
|
|
77
|
+
"T20", # flake8-print
|
|
78
|
+
"TCH", # flake8-type-checking
|
|
79
|
+
"TID", # flake8-tidy-imports
|
|
80
|
+
"UP", # pyupgrade
|
|
81
|
+
]
|
|
82
|
+
fixable = ["ALL"]
|
|
83
|
+
ignore = ["SIM117"]
|
|
84
|
+
|
|
85
|
+
[tool.ruff.lint.isort]
|
|
86
|
+
force-single-line = true
|
|
87
|
+
lines-after-imports = 2
|
|
88
|
+
known-first-party = ["diffusion_model-framework"]
|
|
89
|
+
|
|
90
|
+
[tool.ruff.lint.pydocstyle]
|
|
91
|
+
convention = "google"
|
|
92
|
+
|
|
93
|
+
[tool.ruff.format]
|
|
94
|
+
quote-style = "double"
|
|
95
|
+
indent-style = "space"
|
|
96
|
+
skip-magic-trailing-comma = false
|
|
97
|
+
line-ending = "auto"
|
|
98
|
+
docstring-code-format = true
|
|
99
|
+
|
|
100
|
+
[tool.semantic_release]
|
|
101
|
+
version_variables = [
|
|
102
|
+
"diffusion_models/__init__.py:__version__",
|
|
103
|
+
"docs/source/conf.py:release",
|
|
104
|
+
"docs/source/conf.py:html_context.current_version",
|
|
105
|
+
]
|
|
106
|
+
version_toml = [
|
|
107
|
+
"pyproject.toml:tool.poetry.version"
|
|
108
|
+
]
|
|
109
|
+
upload_to_pypi = false
|
|
110
|
+
upload_to_release = true
|
|
111
|
+
|
|
112
|
+
[tool.semantic_release.branches.main]
|
|
113
|
+
match = "(main|master)"
|
|
114
|
+
prerelease = false
|
|
115
|
+
|
|
116
|
+
[tool.semantic_release.branches.dev]
|
|
117
|
+
match = "dev"
|
|
118
|
+
prerelease = true
|
|
119
|
+
prerelease_token = "rc"
|
|
120
|
+
|
|
121
|
+
[tool.semantic_release.branches."feature/*"]
|
|
122
|
+
match = "(feature/*)"
|
|
123
|
+
prerelease = true
|
|
124
|
+
prerelease_token = "alpha"
|