diffct 2.4.0__py3-none-any.whl → 2.4.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- diffct/__init__.py +1 -1
- diffct/_solver_state.py +7 -1
- diffct/backends.py +165 -110
- diffct/kernels/cone_beam.py +27 -20
- diffct/kernels/fan_beam.py +23 -16
- diffct/kernels/geometry_vjp.py +45 -31
- diffct/operators.py +69 -32
- diffct/projectors.py +3 -2
- {diffct-2.4.0.dist-info → diffct-2.4.1.dist-info}/METADATA +7 -7
- {diffct-2.4.0.dist-info → diffct-2.4.1.dist-info}/RECORD +13 -13
- {diffct-2.4.0.dist-info → diffct-2.4.1.dist-info}/WHEEL +0 -0
- {diffct-2.4.0.dist-info → diffct-2.4.1.dist-info}/licenses/LICENSE +0 -0
- {diffct-2.4.0.dist-info → diffct-2.4.1.dist-info}/licenses/NOTICE +0 -0
diffct/__init__.py
CHANGED
diffct/_solver_state.py
CHANGED
|
@@ -211,6 +211,8 @@ def load_checkpoint(operator, directory, expected, iterations, elements, names):
|
|
|
211
211
|
try:
|
|
212
212
|
directory = Path(directory).resolve()
|
|
213
213
|
saved = json.loads((directory / "metadata.json").read_text(encoding="utf-8"))
|
|
214
|
+
if not isinstance(saved, dict):
|
|
215
|
+
raise ValueError("checkpoint metadata must be an object")
|
|
214
216
|
# JSON canonicalization also normalizes tuple-valued physical spacing.
|
|
215
217
|
canonical = json.loads(json.dumps(expected))
|
|
216
218
|
if any(saved.get(key) != value for key, value in canonical.items()):
|
|
@@ -225,12 +227,16 @@ def load_checkpoint(operator, directory, expected, iterations, elements, names):
|
|
|
225
227
|
if expected["algorithm"] in ("cgls", "cgls_into"):
|
|
226
228
|
finite(saved.get("gamma"), "checkpoint gamma")
|
|
227
229
|
manifest = saved.get("states", {})
|
|
230
|
+
if not isinstance(manifest, dict):
|
|
231
|
+
raise ValueError("checkpoint states must be an object")
|
|
228
232
|
if set(manifest) != set(names):
|
|
229
233
|
raise ValueError("checkpoint does not contain the complete solver state")
|
|
230
234
|
states = {}
|
|
231
235
|
for name in names:
|
|
232
|
-
store = NpyStore(directory / f"{name}.npy", mode="r")
|
|
233
236
|
specification = manifest[name]
|
|
237
|
+
if not isinstance(specification, dict):
|
|
238
|
+
raise ValueError("checkpoint state specification must be an object")
|
|
239
|
+
store = NpyStore(directory / f"{name}.npy", mode="r")
|
|
234
240
|
shape = operator.projection_shape if name in ("r", "q", "row") else operator.volume_shape
|
|
235
241
|
if name == "step":
|
|
236
242
|
shape = (1,)
|
diffct/backends.py
CHANGED
|
@@ -168,7 +168,8 @@ class ReferenceProjector:
|
|
|
168
168
|
self.spacing = voxel_spacing_tuple(owner.voxel_spacing, len(owner.global_volume_shape))
|
|
169
169
|
self.chunk_shape, self.view_chunk_size, self.detector_chunk_shape = self._work_chunks()
|
|
170
170
|
if not validate:
|
|
171
|
-
#
|
|
171
|
+
# Call-time owners validate frames separately and sample surfaces
|
|
172
|
+
# from the invocation's single geometry snapshot below.
|
|
172
173
|
return
|
|
173
174
|
with torch.enable_grad():
|
|
174
175
|
geometry = self._snapshot_geometry(synchronize=synchronize)
|
|
@@ -273,8 +274,7 @@ class ReferenceProjector:
|
|
|
273
274
|
def _validate_geometry(self, geometry, fallback, *, admission=False, synchronize=True):
|
|
274
275
|
"""Finish every local geometry check before any rank starts numerics."""
|
|
275
276
|
owner = self.owner
|
|
276
|
-
if not admission and
|
|
277
|
-
(owner.world_size == 1 and isinstance(owner.detector_surface, ParameterizedSurface))):
|
|
277
|
+
if not admission and owner.detector_surface is None:
|
|
278
278
|
return
|
|
279
279
|
devices = tuple(torch.device(device) for device in (owner._devices or (fallback,)))
|
|
280
280
|
error = None
|
|
@@ -294,14 +294,8 @@ class ReferenceProjector:
|
|
|
294
294
|
device = devices[batch % len(devices)]
|
|
295
295
|
origins, directions, _ = self._rays(
|
|
296
296
|
views, pixels, device, validate=True, geometry=staged[device],
|
|
297
|
-
admission=admission,
|
|
298
297
|
)
|
|
299
|
-
|
|
300
|
-
or not torch.isfinite(directions.to(dtype=owner.dtype)).all().item()):
|
|
301
|
-
raise ValueError("detector surface world coordinates must be finite")
|
|
302
|
-
if (owner.beam not in ("parallel", "parallel3d")
|
|
303
|
-
and torch.any(torch.all(directions == 0, dim=-1)).item()):
|
|
304
|
-
raise ValueError("source and detector surface pixels must differ")
|
|
298
|
+
self._check_rays(origins, directions)
|
|
305
299
|
except (TypeError, ValueError, RuntimeError) as failure:
|
|
306
300
|
error = failure
|
|
307
301
|
if synchronize:
|
|
@@ -309,10 +303,23 @@ class ReferenceProjector:
|
|
|
309
303
|
elif error is not None:
|
|
310
304
|
raise error
|
|
311
305
|
|
|
312
|
-
def
|
|
306
|
+
def _check_rays(self, origins, directions):
|
|
307
|
+
owner = self.owner
|
|
308
|
+
if (not torch.isfinite(origins.to(dtype=owner.dtype)).all().item()
|
|
309
|
+
or not torch.isfinite(directions.to(dtype=owner.dtype)).all().item()):
|
|
310
|
+
raise ValueError("detector surface world coordinates must be finite")
|
|
311
|
+
if (owner.beam not in ("parallel", "parallel3d")
|
|
312
|
+
and torch.any(torch.all(directions == 0, dim=-1)).item()):
|
|
313
|
+
raise ValueError("source and detector surface pixels must differ")
|
|
314
|
+
|
|
315
|
+
def _rays(self, views, pixels, device, validate=False, stats=None, geometry=None, *, admit=False):
|
|
316
|
+
"""Rays of one view/pixel batch.
|
|
317
|
+
|
|
318
|
+
``admit`` applies the admission checks of ``_validate_geometry`` to the
|
|
319
|
+
same sampled offsets before composing the numerical rays, so a
|
|
320
|
+
parameterized sampler runs once for both.
|
|
321
|
+
"""
|
|
313
322
|
owner = self.owner
|
|
314
|
-
# Admission compares original coordinates before FP32 rounding.
|
|
315
|
-
dtype = torch.float64 if validate and admission else owner.dtype
|
|
316
323
|
axes = [(torch.arange(pixel.start, pixel.stop, dtype=torch.float64, device="cpu")
|
|
317
324
|
+ .5 - size / 2) * pitch
|
|
318
325
|
for pixel, size, pitch in zip(pixels, owner.detector_shape, owner.detector_spacing)]
|
|
@@ -338,99 +345,138 @@ class ReferenceProjector:
|
|
|
338
345
|
offsets = offsets[views]
|
|
339
346
|
elif tuple(offsets.shape) not in (shared_shape, (len(ids), *shared_shape)):
|
|
340
347
|
raise ValueError("detector_surface output shape must match the selected view/pixel batch")
|
|
341
|
-
if validate:
|
|
348
|
+
if validate or admit:
|
|
342
349
|
if not torch.isfinite(offsets).all().item():
|
|
343
350
|
raise ValueError("detector_surface offsets must be finite")
|
|
344
351
|
if owner.beam not in ("cone", "parallel3d") and torch.any(offsets[..., 1] != 0).item():
|
|
345
352
|
raise ValueError("2D detector_surface middle offsets must be zero")
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
353
|
+
def compose(validate, stats):
|
|
354
|
+
check_range = (validate and owner.beam not in ("parallel", "parallel3d")
|
|
355
|
+
and owner.dtype == torch.float32 and surface is not None)
|
|
356
|
+
dtype = torch.float64 if check_range else owner.dtype
|
|
357
|
+
frames = (owner._learnable or owner._trajectory) if geometry is None else geometry[:frame_count]
|
|
358
|
+
native_surface = owner.backend == "numba" and surface is not None
|
|
359
|
+
native_legacy = native_surface and not isinstance(surface, ParameterizedSurface)
|
|
360
|
+
if native_surface and not native_legacy:
|
|
361
|
+
dtype = torch.float32
|
|
362
|
+
for component in (*frames, offsets):
|
|
363
|
+
dtype = torch.promote_types(dtype, component.dtype)
|
|
364
|
+
local_dtype = offsets.dtype if native_legacy else dtype
|
|
365
|
+
local = self._transfer(offsets.to(dtype=local_dtype), device, stats)
|
|
366
|
+
if local.ndim == u.ndim + 1:
|
|
367
|
+
local = local.unsqueeze(0)
|
|
368
|
+
dimensions = len(owner.global_volume_shape)
|
|
369
|
+
selected = tuple(self._transfer(value[views].to(
|
|
370
|
+
dtype=torch.promote_types(torch.float32, value.dtype) if native_legacy else dtype,
|
|
371
|
+
), device, stats) for value in frames)
|
|
372
|
+
|
|
373
|
+
def frame(value):
|
|
374
|
+
return value.reshape(len(ids), *([1] * u.ndim), dimensions)
|
|
375
|
+
|
|
376
|
+
def world_points(offsets, components):
|
|
377
|
+
axis_u = frame(components[2])
|
|
378
|
+
points = frame(components[1]) + offsets[..., 0, None] * axis_u
|
|
379
|
+
if owner.beam in ("cone", "parallel3d"):
|
|
380
|
+
axis_v = frame(components[3])
|
|
381
|
+
points = points + offsets[..., 1, None] * axis_v
|
|
382
|
+
normal_dtype = torch.promote_types(axis_u.dtype, axis_v.dtype)
|
|
383
|
+
normal = torch.linalg.cross(axis_u.to(dtype=normal_dtype),
|
|
384
|
+
axis_v.to(dtype=normal_dtype), dim=-1)
|
|
385
|
+
else:
|
|
386
|
+
normal = torch.stack((axis_u[..., 1], -axis_u[..., 0]), dim=-1)
|
|
387
|
+
return points + offsets[..., 2, None] * normal
|
|
388
|
+
|
|
389
|
+
if check_range and not native_surface:
|
|
390
|
+
# Certify ordinary batches without expanding raw FP64 world points.
|
|
391
|
+
# L1 norms bound Euclidean distances without squaring tiny bases.
|
|
392
|
+
pitches = local.new_tensor(self.spacing[::-1])
|
|
393
|
+
maximum = local.abs().amax(dim=tuple(range(1, local.ndim - 1)))
|
|
394
|
+
axis_u = selected[2]
|
|
395
|
+
if owner.beam == "cone":
|
|
396
|
+
basis = (axis_u, selected[3], torch.linalg.cross(axis_u, selected[3], dim=-1))
|
|
397
|
+
components = (0, 1, 2)
|
|
398
|
+
else:
|
|
399
|
+
basis = (axis_u, torch.stack((axis_u[:, 1], -axis_u[:, 0]), dim=-1))
|
|
400
|
+
components = (0, 2)
|
|
401
|
+
source_bound = torch.linalg.vector_norm(selected[0] / pitches, ord=1, dim=-1)
|
|
402
|
+
pixel_bound = torch.linalg.vector_norm(selected[1] / pitches, ord=1, dim=-1)
|
|
403
|
+
for component, axis in zip(components, basis):
|
|
404
|
+
pixel_bound = pixel_bound + maximum[:, component] * torch.linalg.vector_norm(
|
|
405
|
+
axis / pitches, ord=1, dim=-1,
|
|
406
|
+
)
|
|
407
|
+
# Cover rounding in the few frame/offset operations; nonfinite or
|
|
408
|
+
# boundary-adjacent bounds remain inconclusive and use exact points.
|
|
409
|
+
slack = 64 * torch.finfo(torch.float64).eps
|
|
410
|
+
source_bound = source_bound + slack * source_bound.clamp_min(1)
|
|
411
|
+
pixel_bound = pixel_bound + slack * pixel_bound.clamp_min(1)
|
|
412
|
+
certified = (torch.isfinite(source_bound) & torch.isfinite(pixel_bound)
|
|
413
|
+
& (torch.minimum(source_bound, pixel_bound) <= 1e6)
|
|
414
|
+
& (source_bound <= 1e15) & (pixel_bound <= 1e15))
|
|
415
|
+
if certified.all().item():
|
|
416
|
+
check_range = False
|
|
417
|
+
dtype = owner.dtype
|
|
418
|
+
selected = tuple(value.to(dtype=dtype) for value in selected)
|
|
419
|
+
local = local.to(dtype=dtype)
|
|
420
|
+
points = world_points(local, selected)
|
|
421
|
+
if check_range:
|
|
422
|
+
# Raw FP64 coordinates preserve the exact admission boundaries.
|
|
423
|
+
sources = frame(selected[0]).expand_as(points)
|
|
424
|
+
pitches = points.new_tensor(self.spacing[::-1])
|
|
425
|
+
source_distance = torch.linalg.vector_norm(sources / pitches, dim=-1)
|
|
426
|
+
pixel_distance = torch.linalg.vector_norm(points / pitches, dim=-1)
|
|
427
|
+
if torch.any(torch.minimum(source_distance, pixel_distance) > 1e6).item():
|
|
428
|
+
raise ValueError(
|
|
429
|
+
"the source or a detector pixel must lie within 1e6 voxels "
|
|
430
|
+
"of the volume center in every view"
|
|
431
|
+
)
|
|
432
|
+
if torch.any((source_distance > 1e15) | (pixel_distance > 1e15)).item():
|
|
433
|
+
raise ValueError(
|
|
434
|
+
"the source and detector surface pixels must lie within 1e15 voxels "
|
|
435
|
+
"of the volume center"
|
|
436
|
+
)
|
|
437
|
+
if native_surface:
|
|
438
|
+
# Native surface preprocessing composes before staging complete
|
|
439
|
+
# rays to FP32. Data transforms must preserve that operator.
|
|
440
|
+
selected = tuple(value.to(dtype=owner.dtype) for value in selected)
|
|
441
|
+
points = points.to(dtype=owner.dtype)
|
|
442
|
+
elif validate and dtype != owner.dtype:
|
|
443
|
+
# Casting the completed point misses cancellation/overflow during
|
|
444
|
+
# FP32 component arithmetic. Reuse the same sampled offsets.
|
|
445
|
+
selected = tuple(value.to(dtype=owner.dtype) for value in selected)
|
|
446
|
+
points = world_points(local.to(dtype=owner.dtype), selected)
|
|
447
|
+
if owner.beam in ("parallel", "parallel3d"):
|
|
448
|
+
origins, directions = points, frame(selected[0]).expand_as(points)
|
|
374
449
|
else:
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
points = world_points(local.to(dtype=owner.dtype), selected)
|
|
406
|
-
if owner.beam in ("parallel", "parallel3d"):
|
|
407
|
-
origins, directions = points, frame(selected[0]).expand_as(points)
|
|
408
|
-
else:
|
|
409
|
-
sources = frame(selected[0]).expand_as(points)
|
|
410
|
-
pitches = points.new_tensor(self.spacing[::-1])
|
|
411
|
-
source_distance = torch.linalg.vector_norm(sources / pitches, dim=-1)
|
|
412
|
-
pixel_distance = torch.linalg.vector_norm(points / pitches, dim=-1)
|
|
413
|
-
if validate:
|
|
414
|
-
if torch.any(torch.all(
|
|
415
|
-
points.to(dtype=owner.dtype) == sources.to(dtype=owner.dtype), dim=-1,
|
|
416
|
-
)).item():
|
|
417
|
-
raise ValueError("source and detector surface pixels must differ")
|
|
418
|
-
# Starting near the volume avoids subtracting its voxel planes
|
|
419
|
-
# from a distant endpoint; reversing a segment preserves [0, 1].
|
|
420
|
-
source_nearer = (source_distance <= pixel_distance).unsqueeze(-1)
|
|
421
|
-
origins = torch.where(source_nearer, sources, points)
|
|
422
|
-
difference = points - sources
|
|
423
|
-
directions = torch.where(source_nearer, difference, -difference)
|
|
424
|
-
|
|
425
|
-
pixel_ids = torch.meshgrid(*(torch.arange(pixel.start, pixel.stop, device=device)
|
|
426
|
-
for pixel in pixels), indexing="ij")
|
|
427
|
-
detector_ids = torch.zeros_like(pixel_ids[0])
|
|
428
|
-
for coordinate, size in zip(pixel_ids, owner.detector_shape):
|
|
429
|
-
detector_ids = detector_ids * size + coordinate
|
|
430
|
-
ray_ids = ((self._transfer(ids, device, stats) - owner.view_slice.start).reshape(
|
|
431
|
-
len(ids), *([1] * u.ndim),
|
|
432
|
-
) * math.prod(owner.detector_shape) + detector_ids).reshape(-1)
|
|
433
|
-
return origins.reshape(-1, dimensions), directions.reshape(-1, dimensions), ray_ids
|
|
450
|
+
sources = frame(selected[0]).expand_as(points)
|
|
451
|
+
pitches = points.new_tensor(self.spacing[::-1])
|
|
452
|
+
source_distance = torch.linalg.vector_norm(sources / pitches, dim=-1)
|
|
453
|
+
pixel_distance = torch.linalg.vector_norm(points / pitches, dim=-1)
|
|
454
|
+
if validate:
|
|
455
|
+
if torch.any(torch.all(
|
|
456
|
+
points.to(dtype=owner.dtype) == sources.to(dtype=owner.dtype), dim=-1,
|
|
457
|
+
)).item():
|
|
458
|
+
raise ValueError("source and detector surface pixels must differ")
|
|
459
|
+
# Starting near the volume avoids subtracting its voxel planes
|
|
460
|
+
# from a distant endpoint; reversing a segment preserves [0, 1].
|
|
461
|
+
source_nearer = (source_distance <= pixel_distance).unsqueeze(-1)
|
|
462
|
+
origins = torch.where(source_nearer, sources, points)
|
|
463
|
+
difference = points - sources
|
|
464
|
+
directions = torch.where(source_nearer, difference, -difference)
|
|
465
|
+
|
|
466
|
+
pixel_ids = torch.meshgrid(*(torch.arange(pixel.start, pixel.stop, device=device)
|
|
467
|
+
for pixel in pixels), indexing="ij")
|
|
468
|
+
detector_ids = torch.zeros_like(pixel_ids[0])
|
|
469
|
+
for coordinate, size in zip(pixel_ids, owner.detector_shape):
|
|
470
|
+
detector_ids = detector_ids * size + coordinate
|
|
471
|
+
ray_ids = ((self._transfer(ids, device, stats) - owner.view_slice.start).reshape(
|
|
472
|
+
len(ids), *([1] * u.ndim),
|
|
473
|
+
) * math.prod(owner.detector_shape) + detector_ids).reshape(-1)
|
|
474
|
+
return origins.reshape(-1, dimensions), directions.reshape(-1, dimensions), ray_ids
|
|
475
|
+
|
|
476
|
+
if admit:
|
|
477
|
+
with torch.no_grad():
|
|
478
|
+
self._check_rays(*compose(True, None)[:2])
|
|
479
|
+
return compose(validate, stats)
|
|
434
480
|
|
|
435
481
|
def _intervals(self, origins, directions, tile):
|
|
436
482
|
owner = self.owner
|
|
@@ -547,7 +593,7 @@ class ReferenceProjector:
|
|
|
547
593
|
return indices.flatten()
|
|
548
594
|
|
|
549
595
|
def _tensor_apply(self, tensor, is_project, reduce_cotangent=False, *, geometry=None,
|
|
550
|
-
reduce_output=True, _plan=None):
|
|
596
|
+
reduce_output=True, _plan=None, admit=False):
|
|
551
597
|
owner = self.owner
|
|
552
598
|
owner.last_execution_stats = None
|
|
553
599
|
start = time.perf_counter()
|
|
@@ -588,7 +634,7 @@ class ReferenceProjector:
|
|
|
588
634
|
slices[axis] = slice(first, min(first + step, leading[axis]))
|
|
589
635
|
pieces.append(self._tensor_apply(tensor[tuple(slices)], is_project, reduce_cotangent,
|
|
590
636
|
geometry=geometry, reduce_output=reduce_output,
|
|
591
|
-
_plan=plan))
|
|
637
|
+
_plan=plan, admit=admit))
|
|
592
638
|
part_stats = owner.last_execution_stats
|
|
593
639
|
for key in counters:
|
|
594
640
|
stats[key] += part_stats[key]
|
|
@@ -603,13 +649,14 @@ class ReferenceProjector:
|
|
|
603
649
|
# Empty ownership and empty leading dimensions retain a differentiable seam.
|
|
604
650
|
output = output + _zero_dependency(tensor).to(dtype=owner.dtype)
|
|
605
651
|
if is_project:
|
|
606
|
-
for tile, world in self._tiles(plan[0]):
|
|
652
|
+
for tile_index, (tile, world) in enumerate(self._tiles(plan[0])):
|
|
607
653
|
values = tensor[(..., *tile)]
|
|
608
654
|
if reduce_cotangent:
|
|
609
655
|
values = self._reduce(values, owner.view_group, owner.view_world_size, stats)
|
|
610
656
|
for batch, (views, pixels) in enumerate(self._batches(*plan[1:])):
|
|
611
657
|
device = self._device(batch, tensor.device, stats)
|
|
612
|
-
origins, directions, rays = self._rays(views, pixels, device, stats=stats, geometry=geometry
|
|
658
|
+
origins, directions, rays = self._rays(views, pixels, device, stats=stats, geometry=geometry,
|
|
659
|
+
admit=admit and tile_index == 0)
|
|
613
660
|
indices, lengths = self._intervals(origins, directions, world)
|
|
614
661
|
local = self._to(values, device, stats).reshape(*leading, math.prod(values.shape[-len(tile):]))
|
|
615
662
|
selected = torch.index_select(local, -1, indices.flatten()).reshape(*leading, *indices.shape)
|
|
@@ -632,7 +679,8 @@ class ReferenceProjector:
|
|
|
632
679
|
if reduce_cotangent:
|
|
633
680
|
values = self._reduce(values, owner.space_group, owner.space_world_size, stats)
|
|
634
681
|
values = self._to(values, device, stats).reshape(*leading, math.prod(values.shape[-len(shape):]))
|
|
635
|
-
origins, directions, _ = self._rays(views, pixels, device, stats=stats, geometry=geometry
|
|
682
|
+
origins, directions, _ = self._rays(views, pixels, device, stats=stats, geometry=geometry,
|
|
683
|
+
admit=admit)
|
|
636
684
|
for tile, world in self._tiles(plan[0]):
|
|
637
685
|
indices, lengths = self._intervals(origins, directions, world)
|
|
638
686
|
tile_shape = tuple(part.stop - part.start for part in tile)
|
|
@@ -657,10 +705,17 @@ class ReferenceProjector:
|
|
|
657
705
|
|
|
658
706
|
def _apply(self, tensor, is_project):
|
|
659
707
|
geometry = self._snapshot_geometry()
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
708
|
+
owner = self.owner
|
|
709
|
+
# A single process returns no output before every batch passes, so a
|
|
710
|
+
# parameterized surface is admitted in the numerical sampling pass and
|
|
711
|
+
# its sampler runs once per batch. Distributed calls finish all checks
|
|
712
|
+
# before any rank starts numerics.
|
|
713
|
+
admit = owner.world_size == 1 and isinstance(owner.detector_surface, ParameterizedSurface)
|
|
714
|
+
if not admit:
|
|
715
|
+
self._validate_geometry(geometry, tensor.device if isinstance(tensor, torch.Tensor) else torch.device("cpu"))
|
|
716
|
+
if owner.world_size > 1 and torch.is_grad_enabled():
|
|
717
|
+
return _DistributedApply.apply(tensor, self, is_project, False, owner._participation(), *geometry)
|
|
718
|
+
return self._tensor_apply(tensor, is_project, geometry=geometry, admit=admit)
|
|
664
719
|
|
|
665
720
|
def _geometry_grad(self, tensor, cotangent, is_project, reduce_cotangent, geometry):
|
|
666
721
|
owner = self.owner
|
diffct/kernels/cone_beam.py
CHANGED
|
@@ -120,6 +120,11 @@ def _cone_3d_forward_kernel(
|
|
|
120
120
|
det_y = d_detector_positions[iview, iu, iv, 1] / voxel_spacing
|
|
121
121
|
det_z = d_detector_positions[iview, iu, iv, 2] / voxel_spacing
|
|
122
122
|
|
|
123
|
+
# Preserve native endpoint composition, then clip in float64: when both
|
|
124
|
+
# endpoints are distant, float32 loses voxel widths before rebasing.
|
|
125
|
+
src_x, src_y, src_z = np.float64(src_x), np.float64(src_y), np.float64(src_z)
|
|
126
|
+
det_x, det_y, det_z = np.float64(det_x), np.float64(det_y), np.float64(det_z)
|
|
127
|
+
|
|
123
128
|
# === 3D RAY DIRECTION CALCULATION ===
|
|
124
129
|
# Ray direction vector from source to detector element in 3D space
|
|
125
130
|
dir_x, dir_y, dir_z = det_x - src_x, det_y - src_y, det_z - src_z
|
|
@@ -133,7 +138,7 @@ def _cone_3d_forward_kernel(
|
|
|
133
138
|
|
|
134
139
|
# === 3D RAY-VOLUME INTERSECTION CALCULATION ===
|
|
135
140
|
# Measure the ray parameter from the endpoint nearer the volume centre, so a
|
|
136
|
-
# distant source does not make the
|
|
141
|
+
# distant source does not make the parameters coarse. Only the
|
|
137
142
|
# source-to-detector segment is integrated.
|
|
138
143
|
if src_x * src_x + src_y * src_y + src_z * src_z <= det_x * det_x + det_y * det_y + det_z * det_z:
|
|
139
144
|
org_x = src_x
|
|
@@ -170,15 +175,14 @@ def _cone_3d_forward_kernel(
|
|
|
170
175
|
if t_min >= t_max: # No valid 3D intersection
|
|
171
176
|
d_sino[iview, iu, iv] = _ZERO; return
|
|
172
177
|
|
|
173
|
-
#
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
t_end = t_max - t_min
|
|
178
|
+
# Only setup needs float64; traverse in float32 from the volume entry.
|
|
179
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
180
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
181
|
+
ent_z = np.float32(org_z + t_min * dir_z)
|
|
182
|
+
ray_x = np.float32(dir_x)
|
|
183
|
+
ray_y = np.float32(dir_y)
|
|
184
|
+
ray_z = np.float32(dir_z)
|
|
185
|
+
t_end = np.float32(t_max - t_min)
|
|
182
186
|
|
|
183
187
|
# === 3D SIDDON METHOD TRAVERSAL INITIALIZATION ===
|
|
184
188
|
accum = _ZERO # Accumulated projection value
|
|
@@ -337,6 +341,10 @@ def _cone_3d_backward_kernel(
|
|
|
337
341
|
det_y = d_detector_positions[iview, iu, iv, 1] / voxel_spacing
|
|
338
342
|
det_z = d_detector_positions[iview, iu, iv, 2] / voxel_spacing
|
|
339
343
|
|
|
344
|
+
# Match forward setup while retaining native endpoint composition.
|
|
345
|
+
src_x, src_y, src_z = np.float64(src_x), np.float64(src_y), np.float64(src_z)
|
|
346
|
+
det_x, det_y, det_z = np.float64(det_x), np.float64(det_y), np.float64(det_z)
|
|
347
|
+
|
|
340
348
|
# === 3D RAY DIRECTION CALCULATION ===
|
|
341
349
|
# Ray direction vector from source to detector element in 3D space
|
|
342
350
|
dir_x, dir_y, dir_z = det_x - src_x, det_y - src_y, det_z - src_z
|
|
@@ -347,7 +355,7 @@ def _cone_3d_backward_kernel(
|
|
|
347
355
|
|
|
348
356
|
# === 3D RAY-VOLUME INTERSECTION CALCULATION ===
|
|
349
357
|
# Measure the ray parameter from the endpoint nearer the volume centre, so a
|
|
350
|
-
# distant source does not make the
|
|
358
|
+
# distant source does not make the parameters coarse. Only the
|
|
351
359
|
# source-to-detector segment is integrated.
|
|
352
360
|
if src_x * src_x + src_y * src_y + src_z * src_z <= det_x * det_x + det_y * det_y + det_z * det_z:
|
|
353
361
|
org_x = src_x
|
|
@@ -380,15 +388,14 @@ def _cone_3d_backward_kernel(
|
|
|
380
388
|
|
|
381
389
|
if t_min >= t_max: return
|
|
382
390
|
|
|
383
|
-
#
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
t_end = t_max - t_min
|
|
391
|
+
# Only setup needs float64; traverse in float32 from the volume entry.
|
|
392
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
393
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
394
|
+
ent_z = np.float32(org_z + t_min * dir_z)
|
|
395
|
+
ray_x = np.float32(dir_x)
|
|
396
|
+
ray_y = np.float32(dir_y)
|
|
397
|
+
ray_z = np.float32(dir_z)
|
|
398
|
+
t_end = np.float32(t_max - t_min)
|
|
392
399
|
|
|
393
400
|
# === 3D SIDDON METHOD TRAVERSAL INITIALIZATION ===
|
|
394
401
|
t = _ZERO
|
diffct/kernels/fan_beam.py
CHANGED
|
@@ -100,6 +100,11 @@ def _fan_2d_forward_kernel(
|
|
|
100
100
|
det_x = d_detector_positions[iang, idet, 0] / voxel_spacing
|
|
101
101
|
det_y = d_detector_positions[iang, idet, 1] / voxel_spacing
|
|
102
102
|
|
|
103
|
+
# Preserve native endpoint composition, then clip in float64: when both
|
|
104
|
+
# endpoints are distant, float32 loses voxel widths before rebasing.
|
|
105
|
+
src_x, src_y = np.float64(src_x), np.float64(src_y)
|
|
106
|
+
det_x, det_y = np.float64(det_x), np.float64(det_y)
|
|
107
|
+
|
|
103
108
|
# === RAY DIRECTION CALCULATION ===
|
|
104
109
|
# Ray direction vector from source to detector element
|
|
105
110
|
dir_x, dir_y = det_x - src_x, det_y - src_y
|
|
@@ -113,7 +118,7 @@ def _fan_2d_forward_kernel(
|
|
|
113
118
|
|
|
114
119
|
# === RAY-VOLUME INTERSECTION CALCULATION ===
|
|
115
120
|
# Measure the ray parameter from the endpoint nearer the volume centre, so a
|
|
116
|
-
# distant source does not make the
|
|
121
|
+
# distant source does not make the parameters coarse. Only the
|
|
117
122
|
# source-to-detector segment is integrated.
|
|
118
123
|
if src_x * src_x + src_y * src_y <= det_x * det_x + det_y * det_y:
|
|
119
124
|
org_x = src_x
|
|
@@ -138,13 +143,12 @@ def _fan_2d_forward_kernel(
|
|
|
138
143
|
if t_min >= t_max: # No valid intersection
|
|
139
144
|
d_sino[iang, idet] = _ZERO; return
|
|
140
145
|
|
|
141
|
-
#
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
t_end = t_max - t_min
|
|
146
|
+
# Only setup needs float64; traverse in float32 from the volume entry.
|
|
147
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
148
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
149
|
+
ray_x = np.float32(dir_x)
|
|
150
|
+
ray_y = np.float32(dir_y)
|
|
151
|
+
t_end = np.float32(t_max - t_min)
|
|
148
152
|
|
|
149
153
|
# === SIDDON METHOD TRAVERSAL (same algorithm as parallel beam) ===
|
|
150
154
|
accum = _ZERO # Accumulated projection value
|
|
@@ -270,6 +274,10 @@ def _fan_2d_backward_kernel(
|
|
|
270
274
|
det_x = d_detector_positions[iang, idet, 0] / voxel_spacing
|
|
271
275
|
det_y = d_detector_positions[iang, idet, 1] / voxel_spacing
|
|
272
276
|
|
|
277
|
+
# Match forward setup while retaining native endpoint composition.
|
|
278
|
+
src_x, src_y = np.float64(src_x), np.float64(src_y)
|
|
279
|
+
det_x, det_y = np.float64(det_x), np.float64(det_y)
|
|
280
|
+
|
|
273
281
|
# === RAY DIRECTION CALCULATION ===
|
|
274
282
|
# Ray direction vector from source to detector element
|
|
275
283
|
dir_x, dir_y = det_x - src_x, det_y - src_y
|
|
@@ -280,7 +288,7 @@ def _fan_2d_backward_kernel(
|
|
|
280
288
|
|
|
281
289
|
# === RAY-VOLUME INTERSECTION CALCULATION ===
|
|
282
290
|
# Measure the ray parameter from the endpoint nearer the volume centre, so a
|
|
283
|
-
# distant source does not make the
|
|
291
|
+
# distant source does not make the parameters coarse. Only the
|
|
284
292
|
# source-to-detector segment is integrated.
|
|
285
293
|
if src_x * src_x + src_y * src_y <= det_x * det_x + det_y * det_y:
|
|
286
294
|
org_x = src_x
|
|
@@ -302,13 +310,12 @@ def _fan_2d_backward_kernel(
|
|
|
302
310
|
|
|
303
311
|
if t_min >= t_max: return
|
|
304
312
|
|
|
305
|
-
#
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
t_end = t_max - t_min
|
|
313
|
+
# Only setup needs float64; traverse in float32 from the volume entry.
|
|
314
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
315
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
316
|
+
ray_x = np.float32(dir_x)
|
|
317
|
+
ray_y = np.float32(dir_y)
|
|
318
|
+
t_end = np.float32(t_max - t_min)
|
|
312
319
|
|
|
313
320
|
# === SIDDON METHOD TRAVERSAL INITIALIZATION ===
|
|
314
321
|
t = _ZERO
|
diffct/kernels/geometry_vjp.py
CHANGED
|
@@ -39,15 +39,15 @@ from ..constants import (
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
@cuda.jit(device=True)
|
|
42
|
-
def _face_terms(c,
|
|
43
|
-
"""Return
|
|
42
|
+
def _face_terms(c, local_t, entry_tau, entry_tau_minus_one, inv_length, d):
|
|
43
|
+
"""Return face derivatives using float32 distance from the volume entry.
|
|
44
44
|
|
|
45
|
-
|
|
46
|
-
the
|
|
47
|
-
direction component normal to the face.
|
|
45
|
+
The entry fractions are computed independently in float64 before traversal,
|
|
46
|
+
preserving the small fraction when just one endpoint is very distant.
|
|
48
47
|
"""
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
delta_tau = local_t * inv_length
|
|
49
|
+
tau_minus_one = entry_tau_minus_one + delta_tau
|
|
50
|
+
tau = entry_tau + delta_tau
|
|
51
51
|
return c * tau_minus_one / d, -c * tau / d
|
|
52
52
|
|
|
53
53
|
|
|
@@ -85,6 +85,10 @@ def _fan_2d_geometry_vjp_kernel(
|
|
|
85
85
|
det_x = d_detector_positions[iang, idet, 0] / voxel_spacing
|
|
86
86
|
det_y = d_detector_positions[iang, idet, 1] / voxel_spacing
|
|
87
87
|
|
|
88
|
+
# Match forward/adjoint setup without changing endpoint composition.
|
|
89
|
+
src_x, src_y = np.float64(src_x), np.float64(src_y)
|
|
90
|
+
det_x, det_y = np.float64(det_x), np.float64(det_y)
|
|
91
|
+
|
|
88
92
|
dir_x, dir_y = det_x - src_x, det_y - src_y
|
|
89
93
|
length = math.sqrt(dir_x * dir_x + dir_y * dir_y)
|
|
90
94
|
if length < _EPSILON:
|
|
@@ -129,11 +133,14 @@ def _fan_2d_geometry_vjp_kernel(
|
|
|
129
133
|
if t_min >= t_max:
|
|
130
134
|
return
|
|
131
135
|
|
|
132
|
-
ent_x = org_x + t_min * dir_x
|
|
133
|
-
ent_y = org_y + t_min * dir_y
|
|
134
|
-
ray_x = dir_x
|
|
135
|
-
ray_y = dir_y
|
|
136
|
-
t_end = t_max - t_min
|
|
136
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
137
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
138
|
+
ray_x = np.float32(dir_x)
|
|
139
|
+
ray_y = np.float32(dir_y)
|
|
140
|
+
t_end = np.float32(t_max - t_min)
|
|
141
|
+
entry_tau = np.float32((t_min - t_a0) / length)
|
|
142
|
+
entry_tau_minus_one = np.float32((t_min - t_b0) / length)
|
|
143
|
+
inv_length = np.float32(inv_len)
|
|
137
144
|
|
|
138
145
|
accum = _ZERO
|
|
139
146
|
t = _ZERO
|
|
@@ -167,11 +174,11 @@ def _fan_2d_geometry_vjp_kernel(
|
|
|
167
174
|
if pend_axis >= 0:
|
|
168
175
|
c = (-f) if first else (f_prev - f)
|
|
169
176
|
if pend_axis == 0:
|
|
170
|
-
da, db = _face_terms(c, pend_t
|
|
177
|
+
da, db = _face_terms(c, pend_t, entry_tau, entry_tau_minus_one, inv_length, ray_x)
|
|
171
178
|
ga_x += da
|
|
172
179
|
gb_x += db
|
|
173
180
|
else:
|
|
174
|
-
da, db = _face_terms(c, pend_t
|
|
181
|
+
da, db = _face_terms(c, pend_t, entry_tau, entry_tau_minus_one, inv_length, ray_y)
|
|
175
182
|
ga_y += da
|
|
176
183
|
gb_y += db
|
|
177
184
|
first = False
|
|
@@ -191,16 +198,16 @@ def _fan_2d_geometry_vjp_kernel(
|
|
|
191
198
|
iy += step_y
|
|
192
199
|
ty += dt_y
|
|
193
200
|
if ex_axis == 0:
|
|
194
|
-
da, db = _face_terms(f_prev,
|
|
201
|
+
da, db = _face_terms(f_prev, t_end, entry_tau, entry_tau_minus_one, inv_length, ray_x)
|
|
195
202
|
ga_x += da
|
|
196
203
|
gb_x += db
|
|
197
204
|
elif ex_axis == 1:
|
|
198
|
-
da, db = _face_terms(f_prev,
|
|
205
|
+
da, db = _face_terms(f_prev, t_end, entry_tau, entry_tau_minus_one, inv_length, ray_y)
|
|
199
206
|
ga_y += da
|
|
200
207
|
gb_y += db
|
|
201
208
|
|
|
202
209
|
# Length term: p = s * |D| * S with S = accum / |D| (voxel units).
|
|
203
|
-
S = accum
|
|
210
|
+
S = accum * inv_length
|
|
204
211
|
ga_x -= S * ray_x
|
|
205
212
|
ga_y -= S * ray_y
|
|
206
213
|
gb_x += S * ray_x
|
|
@@ -260,6 +267,10 @@ def _cone_3d_geometry_vjp_kernel(
|
|
|
260
267
|
det_y = d_detector_positions[iview, iu, iv, 1] / voxel_spacing
|
|
261
268
|
det_z = d_detector_positions[iview, iu, iv, 2] / voxel_spacing
|
|
262
269
|
|
|
270
|
+
# Match forward/adjoint setup without changing endpoint composition.
|
|
271
|
+
src_x, src_y, src_z = np.float64(src_x), np.float64(src_y), np.float64(src_z)
|
|
272
|
+
det_x, det_y, det_z = np.float64(det_x), np.float64(det_y), np.float64(det_z)
|
|
273
|
+
|
|
263
274
|
dir_x, dir_y, dir_z = det_x - src_x, det_y - src_y, det_z - src_z
|
|
264
275
|
length = math.sqrt(dir_x * dir_x + dir_y * dir_y + dir_z * dir_z)
|
|
265
276
|
if length < _EPSILON:
|
|
@@ -318,13 +329,16 @@ def _cone_3d_geometry_vjp_kernel(
|
|
|
318
329
|
if t_min >= t_max:
|
|
319
330
|
return
|
|
320
331
|
|
|
321
|
-
ent_x = org_x + t_min * dir_x
|
|
322
|
-
ent_y = org_y + t_min * dir_y
|
|
323
|
-
ent_z = org_z + t_min * dir_z
|
|
324
|
-
ray_x = dir_x
|
|
325
|
-
ray_y = dir_y
|
|
326
|
-
ray_z = dir_z
|
|
327
|
-
t_end = t_max - t_min
|
|
332
|
+
ent_x = np.float32(org_x + t_min * dir_x)
|
|
333
|
+
ent_y = np.float32(org_y + t_min * dir_y)
|
|
334
|
+
ent_z = np.float32(org_z + t_min * dir_z)
|
|
335
|
+
ray_x = np.float32(dir_x)
|
|
336
|
+
ray_y = np.float32(dir_y)
|
|
337
|
+
ray_z = np.float32(dir_z)
|
|
338
|
+
t_end = np.float32(t_max - t_min)
|
|
339
|
+
entry_tau = np.float32((t_min - t_a0) / length)
|
|
340
|
+
entry_tau_minus_one = np.float32((t_min - t_b0) / length)
|
|
341
|
+
inv_length = np.float32(inv_len)
|
|
328
342
|
|
|
329
343
|
accum = _ZERO
|
|
330
344
|
t = _ZERO
|
|
@@ -364,15 +378,15 @@ def _cone_3d_geometry_vjp_kernel(
|
|
|
364
378
|
if pend_axis >= 0:
|
|
365
379
|
c = (-f) if first else (f_prev - f)
|
|
366
380
|
if pend_axis == 0:
|
|
367
|
-
da, db = _face_terms(c, pend_t
|
|
381
|
+
da, db = _face_terms(c, pend_t, entry_tau, entry_tau_minus_one, inv_length, ray_x)
|
|
368
382
|
ga_x += da
|
|
369
383
|
gb_x += db
|
|
370
384
|
elif pend_axis == 1:
|
|
371
|
-
da, db = _face_terms(c, pend_t
|
|
385
|
+
da, db = _face_terms(c, pend_t, entry_tau, entry_tau_minus_one, inv_length, ray_y)
|
|
372
386
|
ga_y += da
|
|
373
387
|
gb_y += db
|
|
374
388
|
else:
|
|
375
|
-
da, db = _face_terms(c, pend_t
|
|
389
|
+
da, db = _face_terms(c, pend_t, entry_tau, entry_tau_minus_one, inv_length, ray_z)
|
|
376
390
|
ga_z += da
|
|
377
391
|
gb_z += db
|
|
378
392
|
first = False
|
|
@@ -398,19 +412,19 @@ def _cone_3d_geometry_vjp_kernel(
|
|
|
398
412
|
iz += step_z
|
|
399
413
|
tz += dt_z
|
|
400
414
|
if ex_axis == 0:
|
|
401
|
-
da, db = _face_terms(f_prev,
|
|
415
|
+
da, db = _face_terms(f_prev, t_end, entry_tau, entry_tau_minus_one, inv_length, ray_x)
|
|
402
416
|
ga_x += da
|
|
403
417
|
gb_x += db
|
|
404
418
|
elif ex_axis == 1:
|
|
405
|
-
da, db = _face_terms(f_prev,
|
|
419
|
+
da, db = _face_terms(f_prev, t_end, entry_tau, entry_tau_minus_one, inv_length, ray_y)
|
|
406
420
|
ga_y += da
|
|
407
421
|
gb_y += db
|
|
408
422
|
elif ex_axis == 2:
|
|
409
|
-
da, db = _face_terms(f_prev,
|
|
423
|
+
da, db = _face_terms(f_prev, t_end, entry_tau, entry_tau_minus_one, inv_length, ray_z)
|
|
410
424
|
ga_z += da
|
|
411
425
|
gb_z += db
|
|
412
426
|
|
|
413
|
-
S = accum
|
|
427
|
+
S = accum * inv_length
|
|
414
428
|
ga_x -= S * ray_x
|
|
415
429
|
ga_y -= S * ray_y
|
|
416
430
|
ga_z -= S * ray_z
|
diffct/operators.py
CHANGED
|
@@ -71,7 +71,7 @@ def _detector_spacing(value, beam):
|
|
|
71
71
|
return (pitch, pitch)
|
|
72
72
|
|
|
73
73
|
|
|
74
|
-
def _validate_trajectory(trajectory, beam, *, detector_surface=False):
|
|
74
|
+
def _validate_trajectory(trajectory, beam, *, detector_surface=False, clone=True):
|
|
75
75
|
dimensions = 3 if beam in ("cone", "parallel3d") else 2
|
|
76
76
|
count = 4 if dimensions == 3 else 3
|
|
77
77
|
if not isinstance(trajectory, (tuple, list)) or len(trajectory) != count:
|
|
@@ -93,8 +93,9 @@ def _validate_trajectory(trajectory, beam, *, detector_surface=False):
|
|
|
93
93
|
n_views = component.shape[0]
|
|
94
94
|
if not torch.isfinite(component).all().item():
|
|
95
95
|
raise ValueError("trajectory tensors must contain only finite values")
|
|
96
|
-
geometry.append(component.detach().clone())
|
|
96
|
+
geometry.append(component.detach().clone() if clone else component.detach())
|
|
97
97
|
|
|
98
|
+
device = next((value.device for value in geometry if value.is_cuda), geometry[0].device)
|
|
98
99
|
axes = (geometry[0], geometry[2]) if beam == "parallel" else (
|
|
99
100
|
(geometry[2], geometry[3]) if beam == "cone" else (
|
|
100
101
|
(geometry[0], geometry[2], geometry[3]) if beam == "parallel3d" else (geometry[2],)
|
|
@@ -106,11 +107,11 @@ def _validate_trajectory(trajectory, beam, *, detector_surface=False):
|
|
|
106
107
|
raise ValueError("trajectory direction axes must be unit vectors")
|
|
107
108
|
if beam in ("parallel", "cone", "parallel3d"):
|
|
108
109
|
first, second = (geometry[0], geometry[2]) if beam == "parallel" else (geometry[2], geometry[3])
|
|
109
|
-
dots = torch.sum(first * second.to(
|
|
110
|
+
dots = torch.sum(first.to(device) * second.to(device), dim=1)
|
|
110
111
|
if torch.any(torch.abs(dots) > 1e-4).item():
|
|
111
112
|
raise ValueError("trajectory direction axes must be orthogonal")
|
|
112
113
|
if beam == "parallel3d":
|
|
113
|
-
direction = geometry[0].
|
|
114
|
+
direction = geometry[0].to(device=device, dtype=torch.float64)
|
|
114
115
|
frame = torch.stack((direction, geometry[2].to(direction.device, torch.float64),
|
|
115
116
|
geometry[3].to(direction.device, torch.float64)), dim=-1)
|
|
116
117
|
# Numerical rank rejects a tangent frame at machine roundoff without
|
|
@@ -119,7 +120,7 @@ def _validate_trajectory(trajectory, beam, *, detector_surface=False):
|
|
|
119
120
|
raise ValueError("parallel3d rays must not be tangent to the detector plane")
|
|
120
121
|
if beam in ("fan", "cone") and not detector_surface:
|
|
121
122
|
# float64 avoids overflow of half-precision input and dtype mismatches in cross().
|
|
122
|
-
source = geometry[0].
|
|
123
|
+
source = geometry[0].to(device=device, dtype=torch.float64)
|
|
123
124
|
principal = geometry[1].to(source.device, torch.float64) - source
|
|
124
125
|
lengths = torch.linalg.vector_norm(principal, dim=1)
|
|
125
126
|
if torch.any(lengths == 0).item():
|
|
@@ -302,8 +303,8 @@ class Projector:
|
|
|
302
303
|
trajectory
|
|
303
304
|
tensor requires gradients, the projector clones the geometry. Otherwise it
|
|
304
305
|
keeps references, reads the current values at every call, and returns
|
|
305
|
-
gradients for those tensors. Live default frames are checked
|
|
306
|
-
|
|
306
|
+
gradients for those tensors. Live default frames and overrides are checked
|
|
307
|
+
on each call.
|
|
307
308
|
``project``, ``backproject``, ``__call__`` and their ``*_into`` variants
|
|
308
309
|
accept a keyword-only ``trajectory`` replacing all frames for that call.
|
|
309
310
|
It must retain the constructor's beam and view count; it is validated and
|
|
@@ -531,11 +532,14 @@ class Projector:
|
|
|
531
532
|
# Both FP32 backends set up each segment from the endpoint nearer
|
|
532
533
|
# the volume centre. Measure per-axis distances in voxel units.
|
|
533
534
|
spacing = voxel_spacing_tuple(self.voxel_spacing, len(self.global_volume_shape))
|
|
535
|
+
device = next((value.device for value in self._trajectory[:2] if value.is_cuda),
|
|
536
|
+
self._trajectory[0].device)
|
|
537
|
+
pitches = torch.tensor(spacing[::-1], device=device, dtype=torch.float64)
|
|
534
538
|
distances = torch.stack([
|
|
535
539
|
torch.linalg.vector_norm(
|
|
536
|
-
component.
|
|
540
|
+
component.to(device=device, dtype=torch.float64) / pitches,
|
|
537
541
|
dim=1,
|
|
538
|
-
)
|
|
542
|
+
)
|
|
539
543
|
for component in self._trajectory[:2]
|
|
540
544
|
])
|
|
541
545
|
if torch.any(distances.min(dim=0).values > 1e6).item():
|
|
@@ -655,14 +659,13 @@ class Projector:
|
|
|
655
659
|
*active_devices, *((input_device,) if is_cuda else ()),
|
|
656
660
|
*((output_device,) if output_device.type == "cuda" else ()), *collective_devices,
|
|
657
661
|
)))
|
|
658
|
-
parameterized = isinstance(self.detector_surface, ParameterizedSurface)
|
|
659
662
|
surface = self.detector_surface is not None or self.detector_chunk_shape is not None
|
|
660
663
|
volume_elements = math.prod(self.volume_shape)
|
|
661
664
|
projection_elements = math.prod(self.projection_shape)
|
|
662
665
|
|
|
663
666
|
full_allowed = (isinstance(tensor, torch.Tensor) and is_cuda and output_store is None
|
|
664
667
|
and self.volume_chunk_shape is None and self.view_chunk_size is None
|
|
665
|
-
and self.detector_chunk_shape is None and
|
|
668
|
+
and self.detector_chunk_shape is None and self.schedule == "auto"
|
|
666
669
|
and self.partition == "views")
|
|
667
670
|
|
|
668
671
|
def full_fits(budgets, allocation_limit, reservation_budget):
|
|
@@ -2042,14 +2045,15 @@ class Projector:
|
|
|
2042
2045
|
)
|
|
2043
2046
|
with torch.cuda.device(device):
|
|
2044
2047
|
if sampled:
|
|
2045
|
-
|
|
2048
|
+
# The full plan budgets the complete composition on this
|
|
2049
|
+
# device. Keep its graph here instead of replaying pixels.
|
|
2050
|
+
with torch.enable_grad():
|
|
2046
2051
|
pixels = tuple(slice(0, size) for size in self.detector_shape)
|
|
2047
2052
|
inputs, selections = self._surface_batch_inputs(geometry, geometry_spec, global_slice, pixels)
|
|
2048
2053
|
offsets = _sample_offsets(
|
|
2049
2054
|
self.beam, self.detector_shape, self.detector_spacing, inputs, geometry_spec,
|
|
2050
2055
|
global_slice, pixels, local_geometry=True,
|
|
2051
2056
|
)
|
|
2052
|
-
with torch.no_grad():
|
|
2053
2057
|
local_geometry = _sample_geometry(
|
|
2054
2058
|
self.beam, self.detector_shape, self.detector_spacing, (*inputs[:len(self._trajectory)], offsets),
|
|
2055
2059
|
_GeometrySpec("offsets"),
|
|
@@ -2071,13 +2075,13 @@ class Projector:
|
|
|
2071
2075
|
self.voxel_spacing,
|
|
2072
2076
|
)
|
|
2073
2077
|
if sampled:
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
native_legacy=geometry_spec.kind == "offsets",
|
|
2078
|
+
parts = torch.autograd.grad(
|
|
2079
|
+
local_geometry, inputs,
|
|
2080
|
+
tuple(weight.to(device=value.device, dtype=value.dtype)
|
|
2081
|
+
for value, weight in zip(local_geometry, parts)),
|
|
2082
|
+
allow_unused=True,
|
|
2080
2083
|
)
|
|
2084
|
+
del local_geometry
|
|
2081
2085
|
for index, (grad, part) in enumerate(zip(grads, parts)):
|
|
2082
2086
|
if part is not None:
|
|
2083
2087
|
if sampled:
|
|
@@ -2353,25 +2357,43 @@ class Projector:
|
|
|
2353
2357
|
self.last_execution_stats = None
|
|
2354
2358
|
self._snapshot_d2h_bytes = 0
|
|
2355
2359
|
owner, error, surface_metadata = self, None, None
|
|
2360
|
+
override = trajectory is not None
|
|
2361
|
+
if trajectory is None and self._learnable:
|
|
2362
|
+
trajectory = self._learnable
|
|
2356
2363
|
if trajectory is not None:
|
|
2357
2364
|
try:
|
|
2358
2365
|
frames, views = _validate_trajectory(
|
|
2359
2366
|
trajectory, self.beam, detector_surface=self.detector_surface is not None,
|
|
2367
|
+
clone=override,
|
|
2360
2368
|
)
|
|
2361
2369
|
if views != self._n_views:
|
|
2362
2370
|
raise ValueError("per-call trajectory must keep the constructor view count")
|
|
2363
2371
|
owner = copy(self)
|
|
2364
2372
|
owner._trajectory = frames
|
|
2365
|
-
owner._learnable = tuple(trajectory)
|
|
2366
|
-
|
|
2373
|
+
owner._learnable = (tuple(trajectory)
|
|
2374
|
+
if not override or any(value.requires_grad for value in trajectory) else ())
|
|
2375
|
+
owner._geometry_cache = {}
|
|
2376
|
+
if override:
|
|
2377
|
+
owner._schedule_cache = {}
|
|
2367
2378
|
surface = owner.detector_surface
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2379
|
+
# Default spatial ranks retain the constructor's collectively
|
|
2380
|
+
# agreed backward participation, even with locally fixed frames.
|
|
2381
|
+
if override:
|
|
2382
|
+
owner._geometry_trainable = bool(owner._learnable) or (
|
|
2383
|
+
isinstance(surface, ParameterizedSurface)
|
|
2384
|
+
and any(value.requires_grad for value in surface.parameters)
|
|
2385
|
+
)
|
|
2372
2386
|
owner._validate_geometry_range()
|
|
2373
2387
|
if owner.backend == "torch":
|
|
2374
|
-
|
|
2388
|
+
if override:
|
|
2389
|
+
owner._reference = ReferenceProjector(owner, synchronize=False)
|
|
2390
|
+
else:
|
|
2391
|
+
owner._reference = copy(self._reference)
|
|
2392
|
+
owner._reference.owner = owner
|
|
2393
|
+
if not override and surface is None:
|
|
2394
|
+
owner._reference._validate_geometry(
|
|
2395
|
+
frames, frames[0].device, admission=True, synchronize=False,
|
|
2396
|
+
)
|
|
2375
2397
|
else:
|
|
2376
2398
|
value = args[0]
|
|
2377
2399
|
input_device = torch.device(value.device) if hasattr(value, "device") else torch.device("cpu")
|
|
@@ -2380,12 +2402,13 @@ class Projector:
|
|
|
2380
2402
|
else torch.device("cuda", torch.cuda.current_device()),
|
|
2381
2403
|
)
|
|
2382
2404
|
if surface is not None:
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2405
|
+
if override:
|
|
2406
|
+
geometry, spec, trainable, surface_metadata = owner._surface_validation_inputs()
|
|
2407
|
+
owner._geometry_trainable = owner._geometry_trainable or trainable
|
|
2408
|
+
owner._validate_surface_geometry(geometry, spec, devices)
|
|
2409
|
+
del geometry
|
|
2387
2410
|
else:
|
|
2388
|
-
geometry =
|
|
2411
|
+
geometry = frames
|
|
2389
2412
|
pixels = owner.detector_chunk_shape or tuple(min(size, 64) for size in owner.detector_shape)
|
|
2390
2413
|
local_views = owner.view_slice.stop - owner.view_slice.start
|
|
2391
2414
|
for device_rank, device in enumerate(devices):
|
|
@@ -2422,9 +2445,18 @@ class Projector:
|
|
|
2422
2445
|
# the numerical operation runs inside a data transform.
|
|
2423
2446
|
with torch._C._DisableFuncTorch():
|
|
2424
2447
|
self._partition.preflight(
|
|
2425
|
-
error, (
|
|
2448
|
+
error, (override, owner._geometry_trainable, surface_metadata),
|
|
2426
2449
|
cache_key="operator request", revision=owner._agreement_revision(),
|
|
2427
2450
|
)
|
|
2451
|
+
if isinstance(owner.detector_surface, ParameterizedSurface):
|
|
2452
|
+
# Backward may replay the sampler after the caller replaces it.
|
|
2453
|
+
# Capture Python references here; execution owns tensor snapshots.
|
|
2454
|
+
if owner is self:
|
|
2455
|
+
owner = copy(self)
|
|
2456
|
+
owner.detector_surface = copy(owner.detector_surface)
|
|
2457
|
+
if owner._reference is not None:
|
|
2458
|
+
owner._reference = copy(owner._reference)
|
|
2459
|
+
owner._reference.owner = owner
|
|
2428
2460
|
result = getattr(owner, operation)(*args)
|
|
2429
2461
|
if owner is not self:
|
|
2430
2462
|
self.last_execution_stats = owner.last_execution_stats
|
|
@@ -2450,6 +2482,11 @@ class Projector:
|
|
|
2450
2482
|
return output
|
|
2451
2483
|
with torch.no_grad():
|
|
2452
2484
|
plan, geometry_spec, geometry = self._prepare_execution(source, is_project, sink)
|
|
2485
|
+
if (self.world_size == 1 and geometry_spec is not None
|
|
2486
|
+
and geometry_spec.kind == "sampled"):
|
|
2487
|
+
# Caller-owned outputs must stay intact if a later batch is invalid.
|
|
2488
|
+
# Tensor-returning calls already validate each batch before use.
|
|
2489
|
+
self._validate_surface_geometry(geometry, geometry_spec, plan.devices, plan)
|
|
2453
2490
|
self._run_store_streamed(source, sink, is_project, geometry if geometry else None, plan, geometry_spec)
|
|
2454
2491
|
return output
|
|
2455
2492
|
|
diffct/projectors.py
CHANGED
|
@@ -102,9 +102,10 @@ def _launch_ray_kernel(beam, native_volume, sinogram, geometry, detector_spacing
|
|
|
102
102
|
# rays share an xy path and the backprojection atomics coalesce.
|
|
103
103
|
rows = min(32, 1 << max(0, n_v - 1).bit_length())
|
|
104
104
|
grid, tpb = _grid_3d(n_v, n_u, n_views, tpb=(rows, 64 // rows, 2))
|
|
105
|
-
if max(grid[1], grid[2]) > 65535: # CUDA grid y/z limit: keep the default launch's range
|
|
106
|
-
grid, tpb = _grid_3d(n_v, n_u, n_views)
|
|
107
105
|
else:
|
|
106
|
+
# FP64 ray setup uses more registers; smaller blocks preserve occupancy.
|
|
107
|
+
grid, tpb = _grid_3d(n_v, n_u, n_views, tpb=(8, 4, 4))
|
|
108
|
+
if max(grid[1], grid[2]) > 65535: # Preserve the default launch's supported range.
|
|
108
109
|
grid, tpb = _grid_3d(n_v, n_u, n_views)
|
|
109
110
|
arguments = (
|
|
110
111
|
(d_sino, n_views, n_u, n_v, d_volume, W, H, D)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: diffct
|
|
3
|
-
Version: 2.4.
|
|
3
|
+
Version: 2.4.1
|
|
4
4
|
Summary: Differentiable CT projection and reconstruction: precision modes, streaming, distributed execution and geometry gradients
|
|
5
5
|
Project-URL: Homepage, https://github.com/sypsyp97/diffct
|
|
6
6
|
Project-URL: Documentation, https://sypsyp97.github.io/diffct/
|
|
@@ -177,18 +177,18 @@ volume slabs use `--partition space` under torchrun. Run
|
|
|
177
177
|
|
|
178
178
|
<p align="center">
|
|
179
179
|
<picture>
|
|
180
|
-
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/sypsyp97/diffct/main/docs/assets/scaling_dark.png?v=2.4.
|
|
181
|
-
<img src="https://raw.githubusercontent.com/sypsyp97/diffct/main/docs/assets/scaling_light.png?v=2.4.
|
|
180
|
+
<source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/sypsyp97/diffct/main/docs/assets/scaling_dark.png?v=2.4.1">
|
|
181
|
+
<img src="https://raw.githubusercontent.com/sypsyp97/diffct/main/docs/assets/scaling_light.png?v=2.4.1" width="100%" alt="Time per forward, adjoint and CGLS iteration on 1, 4 and 8 GPUs, 512^3 volume">
|
|
182
182
|
</picture>
|
|
183
183
|
</p>
|
|
184
184
|
|
|
185
185
|
| 512³ volume | 1 GPU | 4 GPUs, 1 node | 8 GPUs, 2 nodes |
|
|
186
186
|
|---|---:|---:|---:|
|
|
187
|
-
| Forward projection |
|
|
188
|
-
| Adjoint (backprojection) |
|
|
189
|
-
| CGLS iteration |
|
|
187
|
+
| Forward projection | 569.99 ms | 151.02 ms (3.8×) | 81.50 ms (7.0×) |
|
|
188
|
+
| Adjoint (backprojection) | 1061.36 ms | 278.75 ms (3.8×) | 220.34 ms (4.8×) |
|
|
189
|
+
| CGLS iteration | 1756.03 ms | 468.05 ms (3.8×) | 333.08 ms (5.3×) |
|
|
190
190
|
|
|
191
|
-
diffct 2.4.
|
|
191
|
+
diffct 2.4.1, FP32, A100 64 GB GPUs; circular cone beam, 360 views, 1024×768 detector, pitch 1.25. Default memory settings, without an imposed allocator cap.
|
|
192
192
|
|
|
193
193
|
Projection times are medians of seven samples after three warmups. CGLS uses three ten-iteration solves after a two-iteration warmup; the median includes initialization and is divided by ten. Timings use synchronized wall time and the slowest rank. Raw samples and configuration: [scaling.json](https://github.com/sypsyp97/diffct/blob/main/docs/assets/scaling.json).
|
|
194
194
|
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
diffct/__init__.py,sha256=
|
|
2
|
-
diffct/_solver_state.py,sha256=
|
|
1
|
+
diffct/__init__.py,sha256=yYLUQ2lVM28RH2J4YlSafnN7-QCgrf9CgtKrzdu4E5Y,2965
|
|
2
|
+
diffct/_solver_state.py,sha256=DKfrgrKcSpx3YetbF8hjA2yzMzJag-iHVpMZGup_dLw,16258
|
|
3
3
|
diffct/_solver_tv.py,sha256=9nE1DVRzdUlcfd-OQQ86GpntX6CcaVvsSVWK-L4uR_I,5783
|
|
4
4
|
diffct/analytical.py,sha256=u2PUp8GTNHoqwP2S4s0y9xNjiVBLfnYMULuuThcWmdU,33245
|
|
5
5
|
diffct/analytical_execution.py,sha256=ZuOmt45-TQg3wb2J0erJe2fXIq0ZQr-uXrrbYiAWg7E,25660
|
|
6
|
-
diffct/backends.py,sha256=
|
|
6
|
+
diffct/backends.py,sha256=VaALT_iE-aL4jAbzJ5xGvINfGLrHqqvuPozz0R8h048,47960
|
|
7
7
|
diffct/chunking.py,sha256=EzozqIuG8NH_vuBRPqaN5zL0rvRFp5ftwReAhZXBP44,11342
|
|
8
8
|
diffct/constants.py,sha256=oy3QNC6an43Ogdn1m81znz1-EMp9GeinVf2wkGYiXfE,3160
|
|
9
9
|
diffct/differentiable.py,sha256=cFirhqXRpUemL7y7Sl_0uqZKGJNj__ptQEcTwYNKzys,1879
|
|
10
10
|
diffct/distribution.py,sha256=iLC3BgAQ9lOkndbEOvNB_j_UaA2Ug9w1KiTUHkOzG8o,9982
|
|
11
11
|
diffct/geometry.py,sha256=iwWXQksIJGOPSuNWJ8tImWj6yPZCC71G3RBh2VDVylY,42581
|
|
12
|
-
diffct/operators.py,sha256=
|
|
12
|
+
diffct/operators.py,sha256=krbRZZ9vb6nuOkQHHfxEimdKlWPvv3XOyP4FnfiQxac,152188
|
|
13
13
|
diffct/pipeline.py,sha256=jp7AvDeQQOME3OPkO9XTcjWFLvOLrv8wczhAf1jcklM,10891
|
|
14
|
-
diffct/projectors.py,sha256=
|
|
14
|
+
diffct/projectors.py,sha256=0oAFSA5mycN59dayTAWK2W7o96MP8oLuVp2V9X-1388,45334
|
|
15
15
|
diffct/reconstruction.py,sha256=KdDrpxQ31ufyUlL_bkzvDzaTcrK4BB97tnJ04ZfVfos,29368
|
|
16
16
|
diffct/solvers.py,sha256=ZroVUlhHOE2IAXB0clw9hUCvomrFSgYdGRQWwnSFZxs,17297
|
|
17
17
|
diffct/storage.py,sha256=EdcFTzRtEAPRvAZhRPrSf8gRNF5o_JVbPLE8eKPhNdY,4153
|
|
@@ -21,13 +21,13 @@ diffct/surfaces.py,sha256=gFk-cRCec8PtWYoWS3iNlf5yBxsaR_0pWgZV9-rUSFs,7848
|
|
|
21
21
|
diffct/utils.py,sha256=cih3GyZ33gnOrTW7O8r2_aXIP2a-rFjOMuAPuvHeQ4I,14794
|
|
22
22
|
diffct/kernels/__init__.py,sha256=YNdsqAkexN25pI7QaHy4phAnQNICQ7tO8K8S1zt7TnI,1899
|
|
23
23
|
diffct/kernels/analytical_transpose.py,sha256=D4vr_ZFPrjFeQyhoDqTyijbkTY-g-sHJet52ho-cbkQ,9187
|
|
24
|
-
diffct/kernels/cone_beam.py,sha256=
|
|
25
|
-
diffct/kernels/fan_beam.py,sha256=
|
|
26
|
-
diffct/kernels/geometry_vjp.py,sha256=
|
|
24
|
+
diffct/kernels/cone_beam.py,sha256=jKlbqQPHhrng-pTbyMFx2b2l0pdQIMiaaB5hj6ayPQI,26753
|
|
25
|
+
diffct/kernels/fan_beam.py,sha256=NPha0Wl-_ayOdS6kAoXJngQffsUczp3rRnojXZ-F1y8,17556
|
|
26
|
+
diffct/kernels/geometry_vjp.py,sha256=YiPMdsuXINLu1oFhPcSbG1LW2P3WSfeJeEvHNLrmQzU,22008
|
|
27
27
|
diffct/kernels/parallel_beam.py,sha256=tCiSx9W8nO4mn8DVyRuK4E0Ry8q38g3pAUACab0UYVU,17143
|
|
28
28
|
diffct/kernels/surface_geometry.py,sha256=vPNzHtiuz-tvyK_lYH0JlXG8btRjEJlmTDUdB0WM550,5264
|
|
29
|
-
diffct-2.4.
|
|
30
|
-
diffct-2.4.
|
|
31
|
-
diffct-2.4.
|
|
32
|
-
diffct-2.4.
|
|
33
|
-
diffct-2.4.
|
|
29
|
+
diffct-2.4.1.dist-info/METADATA,sha256=6JD-c14bkSl9hu8pekWYMKsziuAP3uSP-TXx3v03SMI,17661
|
|
30
|
+
diffct-2.4.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
|
|
31
|
+
diffct-2.4.1.dist-info/licenses/LICENSE,sha256=a1fjzGs44LpDnV_LmA2Tzp8z-eJKoNVVdLOnh-F6J6E,11344
|
|
32
|
+
diffct-2.4.1.dist-info/licenses/NOTICE,sha256=du9I2sDF5IDfiY9vF64WWzMJh-MarpVXyLfWQmhbvmI,714
|
|
33
|
+
diffct-2.4.1.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|