lesiondiffusion-mlx 0.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of lesiondiffusion-mlx might be problematic. Click here for more details.
- lesiondiffusion_mlx/__init__.py +66 -0
- lesiondiffusion_mlx/_arrays.py +24 -0
- lesiondiffusion_mlx/_nifti.py +73 -0
- lesiondiffusion_mlx/_schedules.py +38 -0
- lesiondiffusion_mlx/_source_half.py +541 -0
- lesiondiffusion_mlx/_spatial.py +57 -0
- lesiondiffusion_mlx/_typing.py +110 -0
- lesiondiffusion_mlx/medmlx.py +1764 -0
- lesiondiffusion_mlx/mlx_model.py +456 -0
- lesiondiffusion_mlx/mlx_pipeline.py +136 -0
- lesiondiffusion_mlx/mlx_sampling.py +132 -0
- lesiondiffusion_mlx/mlx_weights.py +216 -0
- lesiondiffusion_mlx/py.typed +0 -0
- lesiondiffusion_mlx/reference.json +141 -0
- lesiondiffusion_mlx-0.2.1.dist-info/METADATA +99 -0
- lesiondiffusion_mlx-0.2.1.dist-info/RECORD +20 -0
- lesiondiffusion_mlx-0.2.1.dist-info/WHEEL +4 -0
- lesiondiffusion_mlx-0.2.1.dist-info/entry_points.txt +2 -0
- lesiondiffusion_mlx-0.2.1.dist-info/licenses/LICENSE +202 -0
- lesiondiffusion_mlx-0.2.1.dist-info/licenses/THIRD_PARTY_NOTICES.md +147 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
"""DDIM sampler for the upstream mask and inpainting stages, executed on Metal."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import TYPE_CHECKING, cast
|
|
7
|
+
|
|
8
|
+
from lesiondiffusion_mlx._typing import ArrayConstructor
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING:
|
|
11
|
+
from lesiondiffusion_mlx._typing import Array, Mlx
|
|
12
|
+
|
|
13
|
+
from lesiondiffusion_mlx._schedules import lesion_diffusion_timesteps
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DdimSampler:
|
|
17
|
+
"""Uniform DDIM sampler with upstream's schedule, covering both pipeline stages.
|
|
18
|
+
|
|
19
|
+
One class serves LMNet mask generation (no ``mask`` argument) and LINet latent
|
|
20
|
+
inpainting (with ``mask``), mirroring upstream's separate mask and inpainting
|
|
21
|
+
samplers. All tensors are channels-first ``(B, C, D, H, W)``. Timestep indices
|
|
22
|
+
come from ``lesion_diffusion_timesteps`` and are visited from the largest down.
|
|
23
|
+
``eta=1`` (the default) matches upstream's stochastic sampling.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
def __init__(self, alphas_cumprod: Array, mx: Mlx, steps: int, *, eta: float = 1.0):
|
|
27
|
+
"""Precompute per-step coefficients.
|
|
28
|
+
|
|
29
|
+
Args:
|
|
30
|
+
alphas_cumprod: Cumulative product of the training noise schedule, shape
|
|
31
|
+
``(T,)``, as stored in the converted checkpoint (float16 for LINet).
|
|
32
|
+
mx: The imported ``mlx.core`` module.
|
|
33
|
+
steps: Requested DDIM step count; see ``lesion_diffusion_timesteps``.
|
|
34
|
+
eta: DDIM stochasticity; 0 is deterministic, 1 is the upstream setting.
|
|
35
|
+
|
|
36
|
+
Raises:
|
|
37
|
+
ValueError: If ``steps`` does not give a valid schedule for ``T``.
|
|
38
|
+
"""
|
|
39
|
+
self.mx = mx
|
|
40
|
+
self.timesteps = lesion_diffusion_timesteps(steps, alphas_cumprod.shape[0])
|
|
41
|
+
# Source LINet stores its schedule in fp16, then creates fp32 scalar
|
|
42
|
+
# broadcast tensors in p_sample_ddim. Preserve that rounded schedule.
|
|
43
|
+
alphas = alphas_cumprod[cast(ArrayConstructor, mx.array)(self.timesteps, dtype=mx.int32)]
|
|
44
|
+
previous = mx.concatenate([alphas_cumprod[:1], alphas[:-1]])
|
|
45
|
+
self.alphas = alphas.astype(mx.float32)
|
|
46
|
+
self.previous = previous.astype(mx.float32)
|
|
47
|
+
self.sqrt_one_minus = mx.sqrt(1 - alphas).astype(mx.float32)
|
|
48
|
+
# NumPy numerator / Torch denominator dispatches through Torch's
|
|
49
|
+
# reciprocal in the denominator dtype before promotion. This rounding
|
|
50
|
+
# is observable in LINet's fp16 schedule.
|
|
51
|
+
inverse_variance = mx.reciprocal(1 - alphas)
|
|
52
|
+
mx.eval(inverse_variance)
|
|
53
|
+
self.sigmas = eta * mx.sqrt(
|
|
54
|
+
(1 - self.previous)
|
|
55
|
+
* inverse_variance.astype(mx.float32)
|
|
56
|
+
* (1 - self.alphas / self.previous)
|
|
57
|
+
)
|
|
58
|
+
mx.eval(self.alphas, self.previous, self.sqrt_one_minus, self.sigmas)
|
|
59
|
+
|
|
60
|
+
def __call__(
|
|
61
|
+
self,
|
|
62
|
+
denoiser: Callable[[Array, Array, Array | None], Array],
|
|
63
|
+
condition: Array | None,
|
|
64
|
+
conditioning_image: Array,
|
|
65
|
+
*,
|
|
66
|
+
initial_noise: Array | None = None,
|
|
67
|
+
step_noise: tuple[Array, ...] | None = None,
|
|
68
|
+
mask: Array | None = None,
|
|
69
|
+
) -> Array:
|
|
70
|
+
"""Run the reverse process and return the final sample.
|
|
71
|
+
|
|
72
|
+
Without ``mask`` (LMNet), ``conditioning_image`` of shape ``(B, C, D, H, W)``
|
|
73
|
+
is concatenated to the noise on every step and the sample has shape
|
|
74
|
+
``(B, 1, D, H, W)``. With ``mask`` (LINet), ``conditioning_image`` is the
|
|
75
|
+
clean latent ``(B, C, D, H, W)`` and ``mask`` is ``(B, 1, D, H, W)`` with 1
|
|
76
|
+
marking voxels to generate; voxels where ``mask`` is 0 are reset to
|
|
77
|
+
``conditioning_image`` on every step and in the result, and the sample has
|
|
78
|
+
the shape of ``conditioning_image``.
|
|
79
|
+
|
|
80
|
+
Args:
|
|
81
|
+
denoiser: Network called as ``denoiser(input, timestep, condition)`` with
|
|
82
|
+
``timestep`` an int32 array of shape ``(B,)``. Its output is the
|
|
83
|
+
noise prediction; in the masked case only the first ``C`` channels
|
|
84
|
+
are used and the network's extra mask channel is ignored.
|
|
85
|
+
condition: Cross-attention context ``(B, N, D_ctx)``, or ``None``.
|
|
86
|
+
conditioning_image: Conditioning volume or latent, see above.
|
|
87
|
+
initial_noise: Starting noise with the sample's shape; drawn from MLX's
|
|
88
|
+
RNG when omitted. Mainly for reference comparisons.
|
|
89
|
+
step_noise: One noise array per actual timestep, in the order the steps
|
|
90
|
+
run (largest timestep first); drawn from MLX's RNG when omitted.
|
|
91
|
+
mask: Region to generate, or ``None`` for unmasked sampling.
|
|
92
|
+
|
|
93
|
+
Returns:
|
|
94
|
+
The final sample. The lazy graph is evaluated after every step so it
|
|
95
|
+
does not grow across steps.
|
|
96
|
+
|
|
97
|
+
Raises:
|
|
98
|
+
ValueError: If ``initial_noise`` has the wrong shape or ``step_noise``
|
|
99
|
+
does not have one entry per timestep.
|
|
100
|
+
"""
|
|
101
|
+
mx = self.mx
|
|
102
|
+
shape = (
|
|
103
|
+
(conditioning_image.shape[0], 1, *conditioning_image.shape[2:])
|
|
104
|
+
if mask is None
|
|
105
|
+
else conditioning_image.shape
|
|
106
|
+
)
|
|
107
|
+
x = mx.random.normal(shape) if initial_noise is None else initial_noise
|
|
108
|
+
if tuple(x.shape) != tuple(shape):
|
|
109
|
+
raise ValueError(f"DDIM initial noise shape {x.shape} differs from {shape}")
|
|
110
|
+
if step_noise is not None and len(step_noise) != len(self.timesteps):
|
|
111
|
+
raise ValueError("DDIM needs one noise tensor per actual source timestep")
|
|
112
|
+
for iteration, index in enumerate(reversed(range(len(self.timesteps)))):
|
|
113
|
+
timestep = mx.full((x.shape[0],), self.timesteps[index], dtype=mx.int32)
|
|
114
|
+
if mask is not None:
|
|
115
|
+
x = x * mask + (1 - mask) * conditioning_image
|
|
116
|
+
network_input = mx.concatenate([x, mask], axis=1)
|
|
117
|
+
else:
|
|
118
|
+
network_input = mx.concatenate([x, conditioning_image], axis=1)
|
|
119
|
+
eps = denoiser(network_input, timestep, condition)
|
|
120
|
+
if mask is not None:
|
|
121
|
+
# Source predicts a fifth mask channel but retains the supplied
|
|
122
|
+
# mask: p_sample_ddim's mask_pred is deliberately unused.
|
|
123
|
+
eps = eps[:, : conditioning_image.shape[1]]
|
|
124
|
+
alpha, previous, sigma = self.alphas[index], self.previous[index], self.sigmas[index]
|
|
125
|
+
predicted = (x - self.sqrt_one_minus[index] * eps) / mx.sqrt(alpha)
|
|
126
|
+
direction = mx.sqrt(1 - previous - sigma**2) * eps
|
|
127
|
+
noise = mx.random.normal(shape) if step_noise is None else step_noise[iteration]
|
|
128
|
+
x = mx.sqrt(previous) * predicted + direction + sigma * noise
|
|
129
|
+
mx.eval(x)
|
|
130
|
+
if mask is not None:
|
|
131
|
+
x = x * mask + (1 - mask) * conditioning_image
|
|
132
|
+
return x
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
"""Checkpoint conversion without asset staging or runtime artifact loading.
|
|
2
|
+
|
|
3
|
+
Torch deserialization is lazy and requires the optional ``conversion`` extra.
|
|
4
|
+
Converted arrays retain source keys, NCDHW layouts, and float32 storage.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import TYPE_CHECKING, cast
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
from medmlx_core.conversion import load_torch_checkpoint
|
|
15
|
+
|
|
16
|
+
from lesiondiffusion_mlx._arrays import FloatArray, HostArray
|
|
17
|
+
|
|
18
|
+
if TYPE_CHECKING:
|
|
19
|
+
from torch import Tensor
|
|
20
|
+
|
|
21
|
+
type StateTensor = HostArray | Tensor
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def denoiser_weights[T: StateTensor](state: Mapping[str, T]) -> dict[str, T]:
|
|
25
|
+
"""Select the EMA parameters used by upstream's inference ``ema_scope``.
|
|
26
|
+
|
|
27
|
+
Each ``model.<name>`` entry (with any ``diffusion_model.`` prefix removed) is
|
|
28
|
+
replaced by ``model_ema.<name>``, where upstream's ``LitEma`` stores the key
|
|
29
|
+
with its dots removed. The returned keys are the source parameter names.
|
|
30
|
+
|
|
31
|
+
Raises:
|
|
32
|
+
ValueError: If an EMA entry is missing or differs in shape from its raw
|
|
33
|
+
parameter, or if the state has no denoiser parameters.
|
|
34
|
+
"""
|
|
35
|
+
result: dict[str, T] = {}
|
|
36
|
+
for key, value in state.items():
|
|
37
|
+
if not key.startswith("model."):
|
|
38
|
+
continue
|
|
39
|
+
name = key.removeprefix("model.").removeprefix("diffusion_model.")
|
|
40
|
+
ema_name = "model_ema." + name.replace(".", "")
|
|
41
|
+
if ema_name not in state:
|
|
42
|
+
raise ValueError(f"Released LesionDiffusion checkpoint lacks EMA {ema_name}")
|
|
43
|
+
if value.shape != state[ema_name].shape:
|
|
44
|
+
raise ValueError(f"EMA shape differs for {name}")
|
|
45
|
+
result[name] = state[ema_name]
|
|
46
|
+
if not result:
|
|
47
|
+
raise ValueError("LesionDiffusion checkpoint has no denoiser parameters")
|
|
48
|
+
return result
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def convert_state_dict(
|
|
52
|
+
state: Mapping[str, StateTensor], *, stage: str
|
|
53
|
+
) -> tuple[dict[str, FloatArray], dict[str, FloatArray]]:
|
|
54
|
+
"""Return model and text arrays for ``lmnet``, ``linet``, or ``vq``.
|
|
55
|
+
|
|
56
|
+
Accept NumPy arrays or source Torch tensors. Release pinning, tokenizer
|
|
57
|
+
copying, NPZ writing, and conversion metadata belong to the caller.
|
|
58
|
+
|
|
59
|
+
Returns:
|
|
60
|
+
``(arrays, text)``, both float32 NumPy arrays keyed by source name and in
|
|
61
|
+
source (NCDHW) layout. For ``lmnet`` and ``linet``, ``arrays`` holds the EMA
|
|
62
|
+
denoiser weights plus ``alphas_cumprod`` and ``text`` holds the BiomedCLIP
|
|
63
|
+
parameters under ``cond_stage_model.model.`` (prefix removed). For ``vq``,
|
|
64
|
+
``arrays`` holds the encoder, decoder, quantizer and conv layers and ``text``
|
|
65
|
+
is empty.
|
|
66
|
+
|
|
67
|
+
Raises:
|
|
68
|
+
ValueError: For an unknown stage, missing EMA or text parameters, or
|
|
69
|
+
non-finite values.
|
|
70
|
+
KeyError: If a denoiser stage has no ``alphas_cumprod``.
|
|
71
|
+
"""
|
|
72
|
+
if stage not in {"lmnet", "linet", "vq"}:
|
|
73
|
+
raise ValueError(f"Unknown LesionDiffusion stage: {stage}")
|
|
74
|
+
if stage == "vq":
|
|
75
|
+
selected = {
|
|
76
|
+
key: value
|
|
77
|
+
for key, value in state.items()
|
|
78
|
+
if key.startswith(
|
|
79
|
+
("encoder.", "decoder.", "quant_conv.", "post_quant_conv.", "quantize.")
|
|
80
|
+
)
|
|
81
|
+
}
|
|
82
|
+
else:
|
|
83
|
+
selected = denoiser_weights(state)
|
|
84
|
+
selected["alphas_cumprod"] = state["alphas_cumprod"]
|
|
85
|
+
arrays = {
|
|
86
|
+
key: np.asarray(tensor, dtype=np.float32)
|
|
87
|
+
if isinstance(tensor, np.ndarray)
|
|
88
|
+
else cast(FloatArray, tensor.detach().float().numpy())
|
|
89
|
+
for key, tensor in selected.items()
|
|
90
|
+
}
|
|
91
|
+
filename = {"lmnet": "LMNet.ckpt", "linet": "LINet.ckpt", "vq": "VQ.ckpt"}[stage]
|
|
92
|
+
if any(not np.all(np.isfinite(value)) for value in arrays.values()):
|
|
93
|
+
raise ValueError(f"Non-finite weights in {filename}")
|
|
94
|
+
text: dict[str, FloatArray] = {}
|
|
95
|
+
if stage != "vq":
|
|
96
|
+
text_prefix = "cond_stage_model.model."
|
|
97
|
+
text = {
|
|
98
|
+
key.removeprefix(text_prefix): np.asarray(tensor, dtype=np.float32)
|
|
99
|
+
if isinstance(tensor, np.ndarray)
|
|
100
|
+
else cast(FloatArray, tensor.detach().float().numpy())
|
|
101
|
+
for key, tensor in state.items()
|
|
102
|
+
if key.startswith(text_prefix)
|
|
103
|
+
}
|
|
104
|
+
if not text or any(not np.all(np.isfinite(value)) for value in text.values()):
|
|
105
|
+
raise ValueError(f"Missing or non-finite text parameters in {filename}")
|
|
106
|
+
return arrays, text
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def convert_checkpoint(
|
|
110
|
+
path: Path, *, stage: str
|
|
111
|
+
) -> tuple[dict[str, FloatArray], dict[str, FloatArray]]:
|
|
112
|
+
"""Deserialize one explicit source checkpoint using the conversion extra.
|
|
113
|
+
|
|
114
|
+
Loads with ``weights_only=True`` and converts its ``state_dict``; see
|
|
115
|
+
``convert_state_dict`` for the result and errors.
|
|
116
|
+
"""
|
|
117
|
+
checkpoint = load_torch_checkpoint(path, weights_only=True)
|
|
118
|
+
return convert_state_dict(
|
|
119
|
+
cast(Mapping[str, StateTensor], checkpoint["state_dict"]), stage=stage
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def convert_checkpoints(checkpoints: Mapping[str, Path]) -> dict[str, dict[str, FloatArray]]:
|
|
124
|
+
"""Convert explicit stage paths; return NPZ-ready mappings without staging files.
|
|
125
|
+
|
|
126
|
+
Keys of the result are ``lmnet``, ``linet`` and ``vq`` plus ``lmnet_text`` and
|
|
127
|
+
``linet_text`` for the stages that carry a text encoder.
|
|
128
|
+
"""
|
|
129
|
+
result: dict[str, dict[str, FloatArray]] = {}
|
|
130
|
+
for stage, path in checkpoints.items():
|
|
131
|
+
arrays, text = convert_checkpoint(path, stage=stage)
|
|
132
|
+
result[stage] = arrays
|
|
133
|
+
if stage != "vq":
|
|
134
|
+
result[stage + "_text"] = text
|
|
135
|
+
return result
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def write_conversion(
|
|
139
|
+
*,
|
|
140
|
+
checkpoints: Mapping[str, Path],
|
|
141
|
+
tokenizer_path: Path,
|
|
142
|
+
output_dir: Path,
|
|
143
|
+
overwrite: bool = False,
|
|
144
|
+
) -> Path:
|
|
145
|
+
"""Write a self-contained conversion for the MedMLX runner.
|
|
146
|
+
|
|
147
|
+
The architecture is pinned to ``reference.json``. The output directory gets one
|
|
148
|
+
float32 NPZ per stage (``lmnet``, ``linet``, ``vq``, ``lmnet_text``,
|
|
149
|
+
``linet_text``), a copy of the tokenizer and ``conversion.json``, which records
|
|
150
|
+
the schema, upstream revision, layout and the EMA selection that ``run`` checks.
|
|
151
|
+
|
|
152
|
+
Args:
|
|
153
|
+
checkpoints: Upstream ``.ckpt`` paths keyed by exactly ``lmnet``, ``linet``
|
|
154
|
+
and ``vq``.
|
|
155
|
+
tokenizer_path: BiomedCLIP ``tokenizer.json`` copied alongside the weights.
|
|
156
|
+
output_dir: Destination directory, created if needed.
|
|
157
|
+
overwrite: Replace existing output files instead of refusing.
|
|
158
|
+
|
|
159
|
+
Returns:
|
|
160
|
+
The resolved output directory.
|
|
161
|
+
|
|
162
|
+
Raises:
|
|
163
|
+
InvalidInputError: If the checkpoint keys are wrong, an input is missing, an
|
|
164
|
+
output would overwrite an input or an existing file (without
|
|
165
|
+
``overwrite``).
|
|
166
|
+
ValueError: If a checkpoint lacks EMA or text parameters or has non-finite
|
|
167
|
+
values; nothing is written in that case.
|
|
168
|
+
"""
|
|
169
|
+
import json
|
|
170
|
+
|
|
171
|
+
from medmlx_core.errors import InvalidInputError
|
|
172
|
+
|
|
173
|
+
from lesiondiffusion_mlx.medmlx import (
|
|
174
|
+
CONVERSION_FILE,
|
|
175
|
+
CONVERSION_SCHEMA,
|
|
176
|
+
CONVERSION_VERSION,
|
|
177
|
+
CONVERTED_FILES,
|
|
178
|
+
SOURCE_REF,
|
|
179
|
+
TOKENIZER_FILE,
|
|
180
|
+
)
|
|
181
|
+
|
|
182
|
+
if set(checkpoints) != {"lmnet", "linet", "vq"}:
|
|
183
|
+
raise InvalidInputError("Conversion requires explicit lmnet, linet and vq checkpoints")
|
|
184
|
+
output = output_dir.resolve()
|
|
185
|
+
destinations = [output / name for name in (*CONVERTED_FILES, TOKENIZER_FILE, CONVERSION_FILE)]
|
|
186
|
+
sources = [path.resolve() for path in (*checkpoints.values(), tokenizer_path)]
|
|
187
|
+
if set(destinations) & set(sources):
|
|
188
|
+
raise InvalidInputError("Conversion output must not overwrite an input file")
|
|
189
|
+
if not overwrite and any(path.exists() for path in destinations):
|
|
190
|
+
raise InvalidInputError("Conversion output exists; pass overwrite=True to replace it")
|
|
191
|
+
for path in sources:
|
|
192
|
+
if not path.is_file():
|
|
193
|
+
raise InvalidInputError(f"Missing conversion input: {path}")
|
|
194
|
+
# Parse tokenizer before writing any output, including all model conversions.
|
|
195
|
+
from tokenizers import Tokenizer
|
|
196
|
+
|
|
197
|
+
Tokenizer.from_file(str(tokenizer_path))
|
|
198
|
+
stages = convert_checkpoints(checkpoints)
|
|
199
|
+
output.mkdir(parents=True, exist_ok=True)
|
|
200
|
+
for stage, arrays in stages.items():
|
|
201
|
+
# ``allow_pickle`` became a real savez parameter only in NumPy 2.2 (previously
|
|
202
|
+
# ``**kwds`` swallowed it as a spurious saved array named "allow_pickle"); the
|
|
203
|
+
# numpy floor in pyproject.toml is pinned to 2.2 for this reason.
|
|
204
|
+
np.savez(output / (stage + ".npz"), allow_pickle=False, **arrays)
|
|
205
|
+
(output / TOKENIZER_FILE).write_bytes(tokenizer_path.read_bytes())
|
|
206
|
+
metadata = {
|
|
207
|
+
"schema_version": CONVERSION_SCHEMA,
|
|
208
|
+
"converter_version": CONVERSION_VERSION,
|
|
209
|
+
"source_ref": SOURCE_REF,
|
|
210
|
+
"layout": "source_torch_ncdhw",
|
|
211
|
+
"stored_dtype": "float32",
|
|
212
|
+
"inference_dtype": {"lmnet": "float32", "linet": "float16", "vq": "float16"},
|
|
213
|
+
"ema": True,
|
|
214
|
+
}
|
|
215
|
+
(output / CONVERSION_FILE).write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8")
|
|
216
|
+
return output
|
|
File without changes
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": 1,
|
|
3
|
+
"repository": "HengruiTianSJTU/LesionDiffusion",
|
|
4
|
+
"revision": "880736e2b8a4d9becdb6594da7f8af6db19d04d1",
|
|
5
|
+
"text_repository": "mlfoundations/open_clip",
|
|
6
|
+
"text_revision": "3ff1faf10b60be27252be7f6c84ce7c8c5e14ec8",
|
|
7
|
+
"files": [
|
|
8
|
+
"pipeline/preprocess.py",
|
|
9
|
+
"open_clip/hf_model.py",
|
|
10
|
+
"open_clip/hf_configs.py",
|
|
11
|
+
"pipeline/inference/sample_legacy.py",
|
|
12
|
+
"pipeline/inference/models.py",
|
|
13
|
+
"pipeline/inference/utils.py",
|
|
14
|
+
"pipeline/ldm/util.py",
|
|
15
|
+
"pipeline/ldm/models/autoencoder.py",
|
|
16
|
+
"pipeline/ldm/modules/attention.py",
|
|
17
|
+
"pipeline/ldm/modules/ema.py",
|
|
18
|
+
"pipeline/ldm/data/data_process_func.py",
|
|
19
|
+
"pipeline/ldm/data/Torchio_contrast_dataloader.py",
|
|
20
|
+
"pipeline/ldm/modules/encoders/biomedclip.py",
|
|
21
|
+
"pipeline/ldm/modules/diffusionmodules/util.py",
|
|
22
|
+
"pipeline/ldm/modules/diffusionmodules/model.py",
|
|
23
|
+
"pipeline/ldm/modules/diffusionmodules/openaimodel.py",
|
|
24
|
+
"pipeline/ldm/modules/distributions/distributions.py",
|
|
25
|
+
"pipeline/ldm/models/diffusion/ddim.py",
|
|
26
|
+
"pipeline/ldm/models/diffusion/ddpm.py",
|
|
27
|
+
"pipeline/main.py",
|
|
28
|
+
"pipeline/test.py",
|
|
29
|
+
"pipeline/configs/clip/open_clip_config.json",
|
|
30
|
+
"pipeline/configs/diffusion/maskdiffusion_test.yaml",
|
|
31
|
+
"pipeline/configs/diffusion/lesiondiffusion_test.yaml",
|
|
32
|
+
"pipeline/configs/autoencoder/lesiondiffusion_vq.yaml",
|
|
33
|
+
"pipeline/organ_type.json"
|
|
34
|
+
],
|
|
35
|
+
"organ_labels": {
|
|
36
|
+
"1": "spleen",
|
|
37
|
+
"2": "kidney right",
|
|
38
|
+
"3": "kidney left",
|
|
39
|
+
"4": "gallbladder",
|
|
40
|
+
"5": "liver",
|
|
41
|
+
"6": "stomach",
|
|
42
|
+
"7": "aorta",
|
|
43
|
+
"8": "inferior vena cava",
|
|
44
|
+
"9": "portal vein and splenic vein",
|
|
45
|
+
"10": "pancreas",
|
|
46
|
+
"11": "adrenal gland right",
|
|
47
|
+
"12": "adrenal gland left",
|
|
48
|
+
"13": "lung upper lobe left",
|
|
49
|
+
"14": "lung lower lobe left",
|
|
50
|
+
"15": "lung upper lobe right",
|
|
51
|
+
"16": "lung middle lobe right",
|
|
52
|
+
"17": "lung lower lobe right",
|
|
53
|
+
"18": "vertebrae L5",
|
|
54
|
+
"19": "vertebrae L4",
|
|
55
|
+
"20": "vertebrae L3",
|
|
56
|
+
"21": "vertebrae L2",
|
|
57
|
+
"22": "vertebrae L1",
|
|
58
|
+
"23": "vertebrae T12",
|
|
59
|
+
"24": "vertebrae T11",
|
|
60
|
+
"25": "vertebrae T10",
|
|
61
|
+
"26": "vertebrae T9",
|
|
62
|
+
"27": "vertebrae T8",
|
|
63
|
+
"28": "vertebrae T7",
|
|
64
|
+
"29": "vertebrae T6",
|
|
65
|
+
"30": "vertebrae T5",
|
|
66
|
+
"31": "vertebrae T4",
|
|
67
|
+
"32": "vertebrae T3",
|
|
68
|
+
"33": "vertebrae T2",
|
|
69
|
+
"34": "vertebrae T1",
|
|
70
|
+
"35": "vertebrae C7",
|
|
71
|
+
"36": "vertebrae C6",
|
|
72
|
+
"37": "vertebrae C5",
|
|
73
|
+
"38": "vertebrae C4",
|
|
74
|
+
"39": "vertebrae C3",
|
|
75
|
+
"40": "vertebrae C2",
|
|
76
|
+
"41": "vertebrae C1",
|
|
77
|
+
"42": "esophagus",
|
|
78
|
+
"43": "trachea",
|
|
79
|
+
"44": "heart myocardium",
|
|
80
|
+
"45": "heart atrium left",
|
|
81
|
+
"46": "heart ventricle left",
|
|
82
|
+
"47": "heart atrium right",
|
|
83
|
+
"48": "heart ventricle right",
|
|
84
|
+
"49": "pulmonary artery",
|
|
85
|
+
"50": "brain",
|
|
86
|
+
"51": "iliac artery left",
|
|
87
|
+
"52": "iliac artery right",
|
|
88
|
+
"53": "iliac vena left",
|
|
89
|
+
"54": "iliac vena right",
|
|
90
|
+
"55": "small bowel",
|
|
91
|
+
"56": "duodenum",
|
|
92
|
+
"57": "colon",
|
|
93
|
+
"58": "rib left 1",
|
|
94
|
+
"59": "rib left 2",
|
|
95
|
+
"60": "rib left 3",
|
|
96
|
+
"61": "rib left 4",
|
|
97
|
+
"62": "rib left 5",
|
|
98
|
+
"63": "rib left 6",
|
|
99
|
+
"64": "rib left 7",
|
|
100
|
+
"65": "rib left 8",
|
|
101
|
+
"66": "rib left 9",
|
|
102
|
+
"67": "rib left 10",
|
|
103
|
+
"68": "rib left 11",
|
|
104
|
+
"69": "rib left 12",
|
|
105
|
+
"70": "rib right 1",
|
|
106
|
+
"71": "rib right 2",
|
|
107
|
+
"72": "rib right 3",
|
|
108
|
+
"73": "rib right 4",
|
|
109
|
+
"74": "rib right 5",
|
|
110
|
+
"75": "rib right 6",
|
|
111
|
+
"76": "rib right 7",
|
|
112
|
+
"77": "rib right 8",
|
|
113
|
+
"78": "rib right 9",
|
|
114
|
+
"79": "rib right 10",
|
|
115
|
+
"80": "rib right 11",
|
|
116
|
+
"81": "rib right 12",
|
|
117
|
+
"82": "humerus left",
|
|
118
|
+
"83": "humerus right",
|
|
119
|
+
"84": "scapula left",
|
|
120
|
+
"85": "scapula right",
|
|
121
|
+
"86": "clavicula left",
|
|
122
|
+
"87": "clavicula right",
|
|
123
|
+
"88": "femur left",
|
|
124
|
+
"89": "femur right",
|
|
125
|
+
"90": "hip left",
|
|
126
|
+
"91": "hip right",
|
|
127
|
+
"92": "sacrum",
|
|
128
|
+
"93": "face",
|
|
129
|
+
"94": "gluteus maximus left",
|
|
130
|
+
"95": "gluteus maximus right",
|
|
131
|
+
"96": "gluteus medius left",
|
|
132
|
+
"97": "gluteus medius right",
|
|
133
|
+
"98": "gluteus minimus left",
|
|
134
|
+
"99": "gluteus minimus right",
|
|
135
|
+
"100": "autochthon left",
|
|
136
|
+
"101": "autochthon right",
|
|
137
|
+
"102": "iliopsoas left",
|
|
138
|
+
"103": "iliopsoas right",
|
|
139
|
+
"104": "urinary bladder"
|
|
140
|
+
}
|
|
141
|
+
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: lesiondiffusion-mlx
|
|
3
|
+
Version: 0.2.1
|
|
4
|
+
Summary: Native MLX implementation of LesionDiffusion
|
|
5
|
+
Author: Joseph Sandoval, Alfredo Sandoval Jr.
|
|
6
|
+
License-Expression: Apache-2.0 AND MIT AND BSD-3-Clause
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
License-File: THIRD_PARTY_NOTICES.md
|
|
9
|
+
Classifier: Operating System :: MacOS :: MacOS X
|
|
10
|
+
Requires-Python: <3.15,>=3.13
|
|
11
|
+
Requires-Dist: medmlx-core==0.2.3
|
|
12
|
+
Requires-Dist: nibabel<6,>=5.3
|
|
13
|
+
Requires-Dist: numpy<3,>=2.2
|
|
14
|
+
Requires-Dist: safetensors<1,>=0.6
|
|
15
|
+
Requires-Dist: scipy<2,>=1.15
|
|
16
|
+
Requires-Dist: tokenizers<1,>=0.21
|
|
17
|
+
Provides-Extra: conversion
|
|
18
|
+
Requires-Dist: torch<3,>=2.13; extra == 'conversion'
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: einops<0.9,>=0.8.1; extra == 'dev'
|
|
21
|
+
Requires-Dist: hypothesis<7,>=6.100; extra == 'dev'
|
|
22
|
+
Requires-Dist: pillow<14,>=13; extra == 'dev'
|
|
23
|
+
Requires-Dist: pyright==1.1.414; extra == 'dev'
|
|
24
|
+
Requires-Dist: pytest<10,>=8.4; extra == 'dev'
|
|
25
|
+
Requires-Dist: ruff<1,>=0.13; extra == 'dev'
|
|
26
|
+
Requires-Dist: scipy-stubs<1.19,>=1.18.1.1; extra == 'dev'
|
|
27
|
+
Requires-Dist: torch<2.14,>=2.13; extra == 'dev'
|
|
28
|
+
Provides-Extra: reference
|
|
29
|
+
Requires-Dist: einops==0.8.2; extra == 'reference'
|
|
30
|
+
Requires-Dist: omegaconf==2.3.1; extra == 'reference'
|
|
31
|
+
Requires-Dist: transformers==5.19.0; extra == 'reference'
|
|
32
|
+
Description-Content-Type: text/markdown
|
|
33
|
+
|
|
34
|
+
# lesiondiffusion-mlx
|
|
35
|
+
|
|
36
|
+
Add synthetic lesions to CT with native MLX, with controls for organ, size, placement and phenotype.
|
|
37
|
+
|
|
38
|
+
[Docs](docs/technical.md) · [MedMLX](https://github.com/MedMLX)
|
|
39
|
+
|
|
40
|
+

|
|
41
|
+
|
|
42
|
+
*Liver CT before and after lesion generation; the contour marks the generated lesion.*
|
|
43
|
+
|
|
44
|
+
| Input | Output |
|
|
45
|
+
| :--- | :--- |
|
|
46
|
+
| CT + aligned binary organ mask | Synthetic CT and lesion mask |
|
|
47
|
+
|
|
48
|
+
**Apple Silicon · Metal · Python 3.13 / 3.14** · [uv](https://docs.astral.sh/uv/)
|
|
49
|
+
|
|
50
|
+
## Quick start
|
|
51
|
+
|
|
52
|
+
```bash
|
|
53
|
+
uv venv --python 3.13
|
|
54
|
+
source .venv/bin/activate
|
|
55
|
+
repo="https://github.com/MedMLX/lesiondiffusion-mlx"
|
|
56
|
+
uv pip install \
|
|
57
|
+
"lesiondiffusion-mlx[conversion] @ git+$repo" \
|
|
58
|
+
huggingface-hub
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
Get `LMNet.ckpt`, `LINet.ckpt` and `VQ.ckpt` from the [upstream weights folder](https://drive.google.com/drive/folders/1n6_eAWhsHBFBYtDxPtp_FXp_tNHhOCHi). Convert once, then generate:
|
|
62
|
+
|
|
63
|
+
```python
|
|
64
|
+
from pathlib import Path
|
|
65
|
+
|
|
66
|
+
from huggingface_hub import hf_hub_download
|
|
67
|
+
from lesiondiffusion_mlx import write_conversion
|
|
68
|
+
|
|
69
|
+
tokenizer = hf_hub_download("microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224", "tokenizer.json")
|
|
70
|
+
write_conversion(
|
|
71
|
+
checkpoints={"lmnet": Path("LMNet.ckpt"), "linet": Path("LINet.ckpt"), "vq": Path("VQ.ckpt")},
|
|
72
|
+
tokenizer_path=Path(tokenizer),
|
|
73
|
+
output_dir=Path("lesiondiffusion-weights"),
|
|
74
|
+
)
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from lesiondiffusion_mlx.medmlx import run
|
|
79
|
+
|
|
80
|
+
result = run(
|
|
81
|
+
image_path="ct_1mm_ras.nii.gz",
|
|
82
|
+
organ_mask_path="liver_mask.nii.gz",
|
|
83
|
+
weights_path="lesiondiffusion-weights",
|
|
84
|
+
output_dir="out",
|
|
85
|
+
organ="liver",
|
|
86
|
+
lesion_type="tumor",
|
|
87
|
+
bbox_size_mm=[30, 30, 20],
|
|
88
|
+
placement="inside_parenchyma",
|
|
89
|
+
)
|
|
90
|
+
print(result["outputs"]["synthetic_image"])
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Inputs must support the documented crop sizes. Outputs use a RAS 1 mm grid; voxels outside the lesion can change. Synthetic output is not ground truth.
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
[Apache-2.0](LICENSE). See [third-party notices](THIRD_PARTY_NOTICES.md). Model weights retain their publisher terms.
|
|
98
|
+
|
|
99
|
+
For research and education. Outputs require human review.
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
lesiondiffusion_mlx/__init__.py,sha256=UeGgZA4aYNnDN9ehDIdnD_DGBrWBN90SS-2vUp9vjq8,1888
|
|
2
|
+
lesiondiffusion_mlx/_arrays.py,sha256=80ywKgaNBlkOadJiEzvNPlFOd54yked_4nHFTJU7Rdo,617
|
|
3
|
+
lesiondiffusion_mlx/_nifti.py,sha256=VctealMh5XsLDAThw3nsYNPOEUS7zYSrfJHOuunFUxY,2594
|
|
4
|
+
lesiondiffusion_mlx/_schedules.py,sha256=kNOBm6o29lVWHjb44TCgdv0iXiNzZcQdShKcFq-42jQ,1866
|
|
5
|
+
lesiondiffusion_mlx/_source_half.py,sha256=B6LA7xO-PPA9-O1imqa_cmkNi-BpcZFmDuxMCOVsOV8,20912
|
|
6
|
+
lesiondiffusion_mlx/_spatial.py,sha256=sVtRgbWUg16YJhUAs8tWUqNmYr3sstzTENpT-JXzAsk,2015
|
|
7
|
+
lesiondiffusion_mlx/_typing.py,sha256=rf2pV_VR4VeJK25UfVf1mcNzXQtNzJ2T7NwH9OrTJfI,3677
|
|
8
|
+
lesiondiffusion_mlx/medmlx.py,sha256=C_nBE-6ZiwU4hb5OplRDNfPCwcb97ZS2IHK7Fer87cE,73568
|
|
9
|
+
lesiondiffusion_mlx/mlx_model.py,sha256=k0pd4I31Oa_HbN0Yi5H90caoBxOpbTpBBgjIKVIIA1s,20282
|
|
10
|
+
lesiondiffusion_mlx/mlx_pipeline.py,sha256=0RaG3wLv-ulpnBtWhNCORxZa1O53gT56X4QdpyLGxUY,5637
|
|
11
|
+
lesiondiffusion_mlx/mlx_sampling.py,sha256=M3muk0AWKJ_ek7eK9uhrogmM28rcr-eRM662lIIuHy4,6650
|
|
12
|
+
lesiondiffusion_mlx/mlx_weights.py,sha256=Rel5WjwreY1Wo8a3WKHRRXKYxRctN6L7MGjBYV0S4Rc,8946
|
|
13
|
+
lesiondiffusion_mlx/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
14
|
+
lesiondiffusion_mlx/reference.json,sha256=tGAHnA3XwV4KrQQwZRyRzjKqyJ5Fx6jDvIEnZymRVd0,4210
|
|
15
|
+
lesiondiffusion_mlx-0.2.1.dist-info/METADATA,sha256=Sutps01ZjmvxK-jLu0REurZGHoeHI0EKPmtBeV98xXw,3476
|
|
16
|
+
lesiondiffusion_mlx-0.2.1.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
17
|
+
lesiondiffusion_mlx-0.2.1.dist-info/entry_points.txt,sha256=9i2dc9GW_EHGf1XvevGK0W1rrfrsb8hKLq7HZ70io54,67
|
|
18
|
+
lesiondiffusion_mlx-0.2.1.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
19
|
+
lesiondiffusion_mlx-0.2.1.dist-info/licenses/THIRD_PARTY_NOTICES.md,sha256=jsI1422mmHSWHxbHHU0p_JWF_sBfaga2XOKRlCEWMW0,5770
|
|
20
|
+
lesiondiffusion_mlx-0.2.1.dist-info/RECORD,,
|