pytme 0.3.1__tar.gz → 0.3.1.dev20250731__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/PKG-INFO +2 -4
  2. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/pyproject.toml +2 -3
  3. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/match_template.py +2 -2
  4. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/postprocess.py +16 -15
  5. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/preprocessor_gui.py +1 -0
  6. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/refine_matches.py +5 -7
  7. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_analyzer.py +2 -3
  8. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_extensions.py +0 -1
  9. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_orientations.py +0 -12
  10. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/aggregation.py +22 -12
  11. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/_jax_utils.py +60 -16
  12. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/cupy_backend.py +11 -11
  13. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/jax_backend.py +27 -9
  14. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/matching_backend.py +11 -0
  15. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/npfftw_backend.py +3 -0
  16. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/density.py +58 -1
  17. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/matching_data.py +24 -0
  18. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/matching_exhaustive.py +5 -2
  19. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/matching_scores.py +23 -0
  20. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/orientations.py +20 -7
  21. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/LICENSE +0 -0
  22. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/MANIFEST.in +0 -0
  23. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/README.md +0 -0
  24. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/pytme.egg-info/SOURCES.txt +0 -0
  25. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/__init__.py +0 -0
  26. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/estimate_memory_usage.py +0 -0
  27. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/eval.py +0 -0
  28. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/extract_candidates.py +0 -0
  29. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/match_template_filters.py +0 -0
  30. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/preprocess.py +0 -0
  31. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/scripts/pytme_runner.py +0 -0
  32. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/setup.cfg +0 -0
  33. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/setup.py +0 -0
  34. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/__init__.py +0 -0
  35. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/.DS_Store +0 -0
  36. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/.DS_Store +0 -0
  37. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/blob_width18.npy +0 -0
  38. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/edgegaussian_sigma3.npy +0 -0
  39. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/gaussian_sigma2.npy +0 -0
  40. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/hamming_width6.npy +0 -0
  41. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/kaiserb_width18.npy +0 -0
  42. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/localgaussian_sigma0510.npy +0 -0
  43. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/mean_size5.npy +0 -0
  44. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/ntree_sigma0510.npy +0 -0
  45. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Blurring/rank_rank3.npy +0 -0
  46. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Maps/.DS_Store +0 -0
  47. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Maps/emd_8621.mrc.gz +0 -0
  48. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/README.md +0 -0
  49. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Raw/.DS_Store +0 -0
  50. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Raw/em_map.map +0 -0
  51. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/.DS_Store +0 -0
  52. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/1pdj.cif +0 -0
  53. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/1pdj.pdb +0 -0
  54. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/5khe.cif +0 -0
  55. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/5khe.ent +0 -0
  56. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/5khe.pdb +0 -0
  57. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/data/Structures/5uz4.cif +0 -0
  58. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/preprocessing/__init__.py +0 -0
  59. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/preprocessing/test_compose.py +0 -0
  60. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/preprocessing/test_frequency_filters.py +0 -0
  61. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/preprocessing/test_preprocessor.py +0 -0
  62. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/preprocessing/test_utils.py +0 -0
  63. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_backends.py +0 -0
  64. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_density.py +0 -0
  65. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_cli.py +0 -0
  66. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_data.py +0 -0
  67. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_exhaustive.py +0 -0
  68. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_memory.py +0 -0
  69. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_optimization.py +0 -0
  70. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_matching_utils.py +0 -0
  71. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_parser.py +0 -0
  72. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_rotations.py +0 -0
  73. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tests/test_structure.py +0 -0
  74. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/__init__.py +0 -0
  75. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/__version__.py +0 -0
  76. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/__init__.py +0 -0
  77. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/_utils.py +0 -0
  78. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/base.py +0 -0
  79. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/peaks.py +0 -0
  80. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/analyzer/proxy.py +0 -0
  81. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/__init__.py +0 -0
  82. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/_cupy_utils.py +0 -0
  83. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/_numpyfftw_utils.py +0 -0
  84. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/mlx_backend.py +0 -0
  85. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/backends/pytorch_backend.py +0 -0
  86. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/cli.py +0 -0
  87. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/__init__.py +0 -0
  88. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48n309.npy +0 -0
  89. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48n527.npy +0 -0
  90. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48n9.npy +0 -0
  91. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u1.npy +0 -0
  92. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u1153.npy +0 -0
  93. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u1201.npy +0 -0
  94. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u1641.npy +0 -0
  95. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u181.npy +0 -0
  96. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u2219.npy +0 -0
  97. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u27.npy +0 -0
  98. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u2947.npy +0 -0
  99. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u3733.npy +0 -0
  100. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u4749.npy +0 -0
  101. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u5879.npy +0 -0
  102. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u7111.npy +0 -0
  103. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u815.npy +0 -0
  104. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u83.npy +0 -0
  105. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c48u8649.npy +0 -0
  106. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c600v.npy +0 -0
  107. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/c600vc.npy +0 -0
  108. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/metadata.yaml +0 -0
  109. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/quat_to_numpy.py +0 -0
  110. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/data/scattering_factors.pickle +0 -0
  111. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/external/bindings.cpp +0 -0
  112. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/__init__.py +0 -0
  113. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/_utils.py +0 -0
  114. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/bandpass.py +0 -0
  115. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/compose.py +0 -0
  116. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/ctf.py +0 -0
  117. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/reconstruction.py +0 -0
  118. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/wedge.py +0 -0
  119. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/filters/whitening.py +0 -0
  120. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/mask.py +0 -0
  121. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/matching_optimization.py +0 -0
  122. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/matching_utils.py +0 -0
  123. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/memory.py +0 -0
  124. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/parser.py +0 -0
  125. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/preprocessor.py +0 -0
  126. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/rotations.py +0 -0
  127. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/structure.py +0 -0
  128. {pytme-0.3.1 → pytme-0.3.1.dev20250731}/tme/types.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pytme
3
- Version: 0.3.1
3
+ Version: 0.3.1.dev20250731
4
4
  Summary: Python Template Matching Engine
5
5
  Author: Valentin Maurer
6
6
  Author-email: Valentin Maurer <valentin.maurer@embl-hamburg.de>
@@ -26,9 +26,7 @@ Requires-Dist: importlib_resources
26
26
  Requires-Dist: joblib
27
27
  Provides-Extra: cupy
28
28
  Requires-Dist: cupy-cuda12x>13.0.0; extra == "cupy"
29
- Provides-Extra: cupy-voltools
30
- Requires-Dist: cupy-cuda12x>13.0.0; extra == "cupy-voltools"
31
- Requires-Dist: voltools; extra == "cupy-voltools"
29
+ Requires-Dist: voltools; extra == "cupy"
32
30
  Provides-Extra: pytorch
33
31
  Requires-Dist: torch; extra == "pytorch"
34
32
  Requires-Dist: torchvision; extra == "pytorch"
@@ -7,7 +7,7 @@ name="pytme"
7
7
  authors = [
8
8
  { name = "Valentin Maurer", email = "valentin.maurer@embl-hamburg.de" },
9
9
  ]
10
- version="0.3.1"
10
+ version="0.3.1.dev20250731"
11
11
  description="Python Template Matching Engine"
12
12
  readme="README.md"
13
13
  requires-python = ">=3.11"
@@ -33,8 +33,7 @@ classifiers = [
33
33
  ]
34
34
 
35
35
  [project.optional-dependencies]
36
- cupy = ["cupy-cuda12x>13.0.0"]
37
- cupy_voltools = ["cupy-cuda12x>13.0.0", "voltools"]
36
+ cupy = ["cupy-cuda12x>13.0.0", "voltools"]
38
37
  pytorch = ["torch", "torchvision"]
39
38
  jax = ["jax[cuda12]", "jaxlib"]
40
39
  jax_cpu = ["jax", "jaxlib"]
@@ -359,8 +359,8 @@ def parse_args():
359
359
  "--invert-target-contrast",
360
360
  action="store_true",
361
361
  default=False,
362
- help="Invert the target's contrast for cases where templates to-be-matched have "
363
- "negative values, e.g. tomograms.",
362
+ help="Invert the target contrast. Useful for matching on tomograms if the "
363
+ "template has not been inverted.",
364
364
  )
365
365
  io_group.add_argument(
366
366
  "--scramble-phases",
@@ -188,7 +188,7 @@ def parse_args():
188
188
  )
189
189
  additional_group.add_argument(
190
190
  "--n-false-positives",
191
- type=int,
191
+ type=float,
192
192
  default=None,
193
193
  required=False,
194
194
  help="Number of accepted false-positives picks to determine minimum score.",
@@ -318,11 +318,7 @@ def normalize_input(foregrounds: Tuple[str], backgrounds: Tuple[str]) -> Tuple:
318
318
  data = load_matching_output(foreground)
319
319
  scores, _, rotations, rotation_mapping, *_ = data
320
320
 
321
- # We could normalize to unit sdev, but that might lead to unexpected
322
- # results for flat background distributions
323
- # scores -= scores.mean()
324
321
  indices = tuple(slice(0, x) for x in scores.shape)
325
-
326
322
  indices_update = scores > scores_out[indices]
327
323
  scores_out[indices][indices_update] = scores[indices_update]
328
324
 
@@ -369,9 +365,7 @@ def normalize_input(foregrounds: Tuple[str], backgrounds: Tuple[str]) -> Tuple:
369
365
  scores_norm = np.full(out_shape_norm, fill_value=0, dtype=np.float32)
370
366
  for background in backgrounds:
371
367
  data_norm = load_matching_output(background)
372
-
373
- scores = data_norm[0]
374
- # scores -= scores.mean()
368
+ scores, _, rotations, rotation_mapping, *_ = data_norm
375
369
 
376
370
  indices = tuple(slice(0, x) for x in scores.shape)
377
371
  indices_update = scores > scores_norm[indices]
@@ -381,9 +375,10 @@ def normalize_input(foregrounds: Tuple[str], backgrounds: Tuple[str]) -> Tuple:
381
375
  update = tuple(slice(0, int(x)) for x in np.minimum(out_shape, scores.shape))
382
376
  scores_out = np.full(out_shape, fill_value=0, dtype=np.float32)
383
377
  scores_out[update] = data[0][update] - scores_norm[update]
378
+ scores_out = np.fmax(scores_out, 0, out=scores_out)
379
+ scores_out[update] += scores_norm[update].mean()
384
380
 
385
381
  # scores_out[update] = np.divide(scores_out[update], 1 - scores_norm[update])
386
- scores_out = np.fmax(scores_out, 0, out=scores_out)
387
382
  data[0] = scores_out
388
383
 
389
384
  fg, bg = simple_stats(data[0]), simple_stats(scores_norm)
@@ -485,8 +480,11 @@ def main():
485
480
  if orientations is None:
486
481
  translations, rotations, scores, details = [], [], [], []
487
482
 
488
- # Data processed by normalize_input is guaranteed to have this shape
489
- scores, offset, rotation_array, rotation_mapping, meta = data
483
+ var = None
484
+ # Data processed by normalize_input is guaranteed to have this shape)
485
+ scores, _, rotation_array, rotation_mapping, *_ = data
486
+ if len(data) == 6:
487
+ scores, _, rotation_array, rotation_mapping, var, *_ = data
490
488
 
491
489
  cropped_shape = np.subtract(
492
490
  scores.shape, np.multiply(args.min_boundary_distance, 2)
@@ -509,13 +507,16 @@ def main():
509
507
  )
510
508
  args.n_false_positives = max(args.n_false_positives, 1)
511
509
  n_correlations = np.size(scores[cropped_slice]) * len(rotation_mapping)
510
+ std = np.std(scores[cropped_slice])
511
+ if var is not None:
512
+ std = np.asarray(np.sqrt(var)).reshape(())
513
+
512
514
  minimum_score = np.multiply(
513
515
  erfcinv(2 * args.n_false_positives / n_correlations),
514
- np.sqrt(2) * np.std(scores[cropped_slice]),
516
+ np.sqrt(2) * std,
515
517
  )
516
- print(f"Determined minimum score cutoff: {minimum_score}.")
517
- minimum_score = max(minimum_score, 0)
518
- args.min_score = minimum_score
518
+ print(f"Determined cutoff --min-score {minimum_score}.")
519
+ args.min_score = max(minimum_score, 0)
519
520
 
520
521
  args.batch_dims = None
521
522
  if hasattr(cli_args, "batch_dims"):
@@ -474,6 +474,7 @@ def membrane_mask(
474
474
  **kwargs,
475
475
  ) -> NDArray:
476
476
  return create_mask(
477
+ center=(center_x, center_y, center_z),
477
478
  mask_type="membrane",
478
479
  shape=template.shape,
479
480
  radius=radius,
@@ -10,11 +10,9 @@ import subprocess
10
10
  from sys import exit
11
11
  from os import unlink
12
12
  from time import time
13
- from os.path import join
14
13
  from typing import Tuple, List, Dict
15
14
 
16
15
  import numpy as np
17
- from scipy import optimize
18
16
  from sklearn.metrics import roc_auc_score
19
17
 
20
18
  from tme import Orientations, Density
@@ -66,7 +64,6 @@ def parse_args():
66
64
  matching_group.add_argument(
67
65
  "-i",
68
66
  "--template",
69
- dest="template",
70
67
  type=str,
71
68
  required=True,
72
69
  help="Path to a template in PDB/MMCIF or other supported formats (see target).",
@@ -102,7 +99,7 @@ def parse_args():
102
99
  )
103
100
  matching_group.add_argument(
104
101
  "-s",
105
- dest="score",
102
+ "--score",
106
103
  type=str,
107
104
  default="batchFLCSphericalMask",
108
105
  choices=list(MATCHING_EXHAUSTIVE_REGISTER.keys()),
@@ -197,6 +194,7 @@ def create_matching_argdict(args) -> Dict:
197
194
  "-n": args.cores,
198
195
  "--ctf-file": args.ctf_file,
199
196
  "--invert-target-contrast": args.invert_target_contrast,
197
+ "--backend" : args.backend,
200
198
  }
201
199
  return arg_dict
202
200
 
@@ -252,7 +250,7 @@ class DeepMatcher:
252
250
  if args.lowpass_range:
253
251
  self.filter_parameters["--lowpass"] = 0
254
252
  if args.highpass_range:
255
- self.filter_parameters["--highpass"] = 200
253
+ self.filter_parameters["--highpass"] = 0
256
254
 
257
255
  self.postprocess_args = create_postprocessing_argdict(args)
258
256
  self.log_file = f"{args.output_prefix}_optimization_log.txt"
@@ -309,14 +307,14 @@ class DeepMatcher:
309
307
 
310
308
  match_template = argdict_to_command(
311
309
  self.match_template_args,
312
- executable="match_template.py",
310
+ executable="match_template",
313
311
  )
314
312
  run_command(match_template)
315
313
 
316
314
  # Assume we get a new peak for each input in the same order
317
315
  postprocess = argdict_to_command(
318
316
  self.postprocess_args,
319
- executable="postprocess.py",
317
+ executable="postprocess",
320
318
  )
321
319
  run_command(postprocess)
322
320
 
@@ -165,7 +165,6 @@ class TestMaxScoreOverRotations:
165
165
  assert res[0].dtype == be._float_dtype
166
166
  assert res[1].size == self.data.ndim
167
167
  assert np.allclose(res[2].shape, self.data.shape)
168
- assert len(res) == 4
169
168
 
170
169
  @pytest.mark.parametrize("use_memmap", [False, True])
171
170
  @pytest.mark.parametrize("score_threshold", [0, 1e10, -1e10])
@@ -181,7 +180,7 @@ class TestMaxScoreOverRotations:
181
180
 
182
181
  data2 = self.data * 2
183
182
  score_analyzer(state, data2, rotation_matrix=self.rotation_matrix)
184
- scores, translation_offset, rotations, mapping = score_analyzer.result(state)
183
+ scores, offset, rotations, mapping, *_ = score_analyzer.result(state)
185
184
 
186
185
  assert np.all(scores >= score_threshold)
187
186
  max_scores = np.maximum(self.data, data2)
@@ -214,7 +213,7 @@ class TestMaxScoreOverRotations:
214
213
  ret = MaxScoreOverRotations.merge(
215
214
  results=states, use_memmap=use_memmap, score_threshold=score_threshold
216
215
  )
217
- scores, translation, rotations, mapping = ret
216
+ scores, translation, rotations, mapping, *_ = ret
218
217
  assert np.all(scores >= score_threshold)
219
218
  max_scores = np.maximum(self.data, data2)
220
219
  max_scores = np.maximum(max_scores, score_threshold)
@@ -53,7 +53,6 @@ class TestExtensions:
53
53
  @pytest.mark.parametrize("min_distance", [0, 5, 10])
54
54
  def test_find_candidate_indices(self, dimension, dtype, min_distance):
55
55
  coordinates = COORDINATES[dimension].astype(dtype)
56
- print(coordinates.shape)
57
56
 
58
57
  min_distance = np.array([min_distance]).astype(dtype)[0]
59
58
 
@@ -95,18 +95,6 @@ class TestDensity:
95
95
  self.orientations.rotations, orientations_new.rotations, atol=1e-3
96
96
  )
97
97
 
98
- @pytest.mark.parametrize("input_format", ("text", "star", "tbl"))
99
- @pytest.mark.parametrize("output_format", ("text", "star", "tbl"))
100
- def test_file_format_io(self, input_format: str, output_format: str):
101
- _, output_file = mkstemp(suffix=f".{input_format}")
102
- _, output_file2 = mkstemp(suffix=f".{output_format}")
103
-
104
- self.orientations.to_file(output_file)
105
- orientations_new = Orientations.from_file(output_file)
106
- orientations_new.to_file(output_file2)
107
-
108
- assert True
109
-
110
98
  @pytest.mark.parametrize("drop_oob", (True, False))
111
99
  @pytest.mark.parametrize("shape", (10, 40, 80))
112
100
  @pytest.mark.parametrize("odd", (True, False))
@@ -132,12 +132,14 @@ class MaxScoreOverRotations(AbstractAnalyzer):
132
132
  - scores : BackendArray of shape `self._shape` filled with `score_threshold`.
133
133
  - rotations : BackendArray of shape `self._shape` filled with -1.
134
134
  - rotation_mapping : dict, empty mapping from rotation bytes to indices.
135
+ - ssum : BackendArray, accumulator for sum of squared scores.
135
136
  """
136
137
  scores = be.full(
137
138
  shape=self._shape, dtype=be._float_dtype, fill_value=self._score_threshold
138
139
  )
139
140
  rotations = be.full(self._shape, dtype=be._int_dtype, fill_value=-1)
140
- return scores, rotations, {}
141
+ ssum = be.full((1), dtype=be._float_dtype, fill_value=0)
142
+ return scores, rotations, {}, ssum
141
143
 
142
144
  def __call__(
143
145
  self,
@@ -156,6 +158,7 @@ class MaxScoreOverRotations(AbstractAnalyzer):
156
158
  - scores : BackendArray, current maximum scores.
157
159
  - rotations : BackendArray, current rotation indices.
158
160
  - rotation_mapping : dict, mapping from rotation bytes to indices.
161
+ - ssum : BackendArray, accumulator for sum of squared scores.
159
162
  scores : BackendArray
160
163
  Array of new scores to update analyzer with.
161
164
  rotation_matrix : BackendArray
@@ -168,7 +171,7 @@ class MaxScoreOverRotations(AbstractAnalyzer):
168
171
  # be.tobytes behaviour caused overhead for certain GPU/CUDA combinations
169
172
  # If the analyzer is not shared and each rotation is unique, we can
170
173
  # use index to rotation mapping and invert prior to merging.
171
- prev_scores, rotations, rotation_mapping = state
174
+ prev_scores, rotations, rotation_mapping, ssum = state
172
175
 
173
176
  rotation_index = len(rotation_mapping)
174
177
  rotation_matrix = be.astype(rotation_matrix, be._float_dtype)
@@ -180,13 +183,14 @@ class MaxScoreOverRotations(AbstractAnalyzer):
180
183
  rotation = be.tobytes(rotation_matrix)
181
184
  rotation_index = rotation_mapping.setdefault(rotation, rotation_index)
182
185
 
186
+ ssum = be.add(ssum, be.ssum(scores), out=ssum)
183
187
  scores, rotations = be.max_score_over_rotations(
184
188
  scores=scores,
185
189
  max_scores=prev_scores,
186
190
  rotations=rotations,
187
191
  rotation_index=rotation_index,
188
192
  )
189
- return scores, rotations, rotation_mapping
193
+ return scores, rotations, rotation_mapping, ssum
190
194
 
191
195
  @staticmethod
192
196
  def _invert_rmap(rotation_mapping: dict) -> dict:
@@ -224,6 +228,7 @@ class MaxScoreOverRotations(AbstractAnalyzer):
224
228
  - scores : BackendArray, current maximum scores.
225
229
  - rotations : BackendArray, current rotation indices.
226
230
  - rotation_mapping : dict, mapping from rotation indices to matrices.
231
+ - ssum : BackendArray, accumulator for sum of squared scores.
227
232
  targetshape : Tuple[int], optional
228
233
  Shape of the target for convolution mode correction.
229
234
  templateshape : Tuple[int], optional
@@ -240,9 +245,9 @@ class MaxScoreOverRotations(AbstractAnalyzer):
240
245
  Returns
241
246
  -------
242
247
  tuple
243
- Final result tuple (scores, offset, rotations, rotation_mapping).
248
+ Final result tuple (scores, offset, rotations, rotation_mapping, ssum).
244
249
  """
245
- scores, rotations, rotation_mapping = state
250
+ scores, rotations, rotation_mapping, ssum = state
246
251
 
247
252
  # Apply postprocessing if parameters are provided
248
253
  if fourier_shift is not None:
@@ -269,11 +274,13 @@ class MaxScoreOverRotations(AbstractAnalyzer):
269
274
  if self._inversion_mapping:
270
275
  rotation_mapping = {be.tobytes(v): k for k, v in rotation_mapping.items()}
271
276
 
277
+ n_rotations = max(len(rotation_mapping), 1)
272
278
  return (
273
279
  scores,
274
280
  be.to_numpy_array(self._offset),
275
281
  rotations,
276
282
  self._invert_rmap(rotation_mapping),
283
+ be.to_numpy_array(ssum) / (scores.size * n_rotations),
277
284
  )
278
285
 
279
286
  def _harmonize_states(states: List[Tuple]):
@@ -287,18 +294,18 @@ class MaxScoreOverRotations(AbstractAnalyzer):
287
294
  if states[i] is None:
288
295
  continue
289
296
 
290
- scores, offset, rotations, rotation_mapping = states[i]
297
+ scores, offset, rotations, rotation_mapping, ssum = states[i]
291
298
  if out_shape is None:
292
299
  out_shape = np.zeros(scores.ndim, int)
293
300
  out_shape = np.maximum(out_shape, np.add(offset, scores.shape))
294
301
 
295
302
  new_param = {}
296
303
  for key, value in rotation_mapping.items():
297
- rotation_bytes = be.tobytes(value)
304
+ rotation_bytes = np.asarray(value).tobytes()
298
305
  new_param[rotation_bytes] = key
299
306
  if rotation_bytes not in new_rotation_mapping:
300
307
  new_rotation_mapping[rotation_bytes] = len(new_rotation_mapping)
301
- states[i] = (scores, offset, rotations, new_param)
308
+ states[i] = (scores, offset, rotations, new_param, ssum)
302
309
  out_shape = tuple(int(x) for x in out_shape)
303
310
  return new_rotation_mapping, out_shape, states
304
311
 
@@ -329,11 +336,10 @@ class MaxScoreOverRotations(AbstractAnalyzer):
329
336
  if len(results) == 1:
330
337
  ret = results[0]
331
338
  if use_memmap:
332
- scores, offset, rotations, rotation_mapping = ret
339
+ scores, offset, rotations, rotation_mapping, ssum = ret
333
340
  scores = array_to_memmap(scores)
334
341
  rotations = array_to_memmap(rotations)
335
- ret = (scores, offset, rotations, rotation_mapping)
336
-
342
+ ret = (scores, offset, rotations, rotation_mapping, ssum)
337
343
  return ret
338
344
 
339
345
  # Determine output array shape and create consistent rotation map
@@ -368,6 +374,7 @@ class MaxScoreOverRotations(AbstractAnalyzer):
368
374
  )
369
375
  rotations_out = np.full(out_shape, fill_value=-1, dtype=rotations_dtype)
370
376
 
377
+ total_ssum = 0
371
378
  for i in range(len(results)):
372
379
  if results[i] is None:
373
380
  continue
@@ -385,7 +392,9 @@ class MaxScoreOverRotations(AbstractAnalyzer):
385
392
  shape=out_shape,
386
393
  dtype=rotations_dtype,
387
394
  )
388
- scores, offset, rotations, rotation_mapping = results[i]
395
+ scores, offset, rotations, rotation_mapping, ssum = results[i]
396
+
397
+ total_ssum = np.add(total_ssum, ssum)
389
398
  stops = np.add(offset, scores.shape).astype(int)
390
399
  indices = tuple(slice(*pos) for pos in zip(offset, stops))
391
400
 
@@ -428,6 +437,7 @@ class MaxScoreOverRotations(AbstractAnalyzer):
428
437
  np.zeros(scores_out.ndim, dtype=int),
429
438
  rotations_out,
430
439
  cls._invert_rmap(master_rotation_mapping),
440
+ total_ssum / len(results),
431
441
  )
432
442
 
433
443
 
@@ -10,14 +10,14 @@ from typing import Tuple
10
10
  from functools import partial
11
11
 
12
12
  import jax.numpy as jnp
13
- from jax import pmap, lax, vmap
13
+ from jax import pmap, lax, vmap, jit
14
14
 
15
15
  from ..types import BackendArray
16
16
  from ..backends import backend as be
17
17
  from ..matching_utils import normalize_template as _normalize_template
18
18
 
19
19
 
20
- __all__ = ["scan"]
20
+ __all__ = ["scan", "setup_scan"]
21
21
 
22
22
 
23
23
  def _correlate(template: BackendArray, ft_target: BackendArray) -> BackendArray:
@@ -112,12 +112,56 @@ def _identity(arr: BackendArray, arr_filter: BackendArray) -> BackendArray:
112
112
  return arr
113
113
 
114
114
 
115
- @partial(
116
- pmap,
117
- in_axes=(0,) + (None,) * 6,
118
- static_broadcasted_argnums=[6, 7, 8, 9],
119
- axis_name="batch",
120
- )
115
+ def _mask_scores(arr, mask):
116
+ return arr.at[:].multiply(mask)
117
+
118
+
119
+ def _select_config(analyzer_kwargs, device_idx):
120
+ return analyzer_kwargs[device_idx]
121
+
122
+
123
+ def setup_scan(analyzer_kwargs, callback_class, fast_shape, rotate_mask):
124
+ """Create separate scan function with initialized analyzer for each device"""
125
+ device_scans = [
126
+ partial(
127
+ scan,
128
+ fast_shape=fast_shape,
129
+ rotate_mask=rotate_mask,
130
+ analyzer=callback_class(**device_config),
131
+ )
132
+ for device_config in analyzer_kwargs
133
+ ]
134
+
135
+ @partial(
136
+ pmap,
137
+ in_axes=(0,) + (None,) * 6,
138
+ axis_name="batch",
139
+ )
140
+ def scan_combined(
141
+ target,
142
+ template,
143
+ template_mask,
144
+ rotations,
145
+ template_filter,
146
+ target_filter,
147
+ score_mask,
148
+ ):
149
+ return lax.switch(
150
+ lax.axis_index("batch"),
151
+ device_scans,
152
+ target,
153
+ template,
154
+ template_mask,
155
+ rotations,
156
+ template_filter,
157
+ target_filter,
158
+ score_mask,
159
+ )
160
+
161
+ return scan_combined
162
+
163
+
164
+ @partial(jit, static_argnums=(7, 8, 9))
121
165
  def scan(
122
166
  target: BackendArray,
123
167
  template: BackendArray,
@@ -125,19 +169,13 @@ def scan(
125
169
  rotations: BackendArray,
126
170
  template_filter: BackendArray,
127
171
  target_filter: BackendArray,
172
+ score_mask: BackendArray,
128
173
  fast_shape: Tuple[int],
129
174
  rotate_mask: bool,
130
- analyzer_class: object,
131
- analyzer_kwargs: Tuple[Tuple],
175
+ analyzer: object,
132
176
  ) -> Tuple[BackendArray, BackendArray]:
133
177
  eps = jnp.finfo(template.dtype).resolution
134
178
 
135
- kwargs = lax.switch(
136
- lax.axis_index("batch"),
137
- [lambda: analyzer_kwargs[i] for i in range(len(analyzer_kwargs))],
138
- )
139
- analyzer = analyzer_class(**be._tuple_to_dict(kwargs))
140
-
141
179
  if hasattr(target_filter, "shape"):
142
180
  target = _apply_fourier_filter(target, target_filter)
143
181
 
@@ -159,6 +197,10 @@ def scan(
159
197
  if template_filter.shape != ():
160
198
  _template_filter_func = _apply_fourier_filter
161
199
 
200
+ _score_mask_func = _identity
201
+ if score_mask.shape != ():
202
+ _score_mask_func = _mask_scores
203
+
162
204
  def _sample_transform(ret, rotation_matrix):
163
205
  state, index = ret
164
206
  template_rot, template_mask_rot = be.rigid_transform(
@@ -185,6 +227,8 @@ def scan(
185
227
  n_observations=n_observations,
186
228
  eps=eps,
187
229
  )
230
+ scores = _score_mask_func(scores, score_mask)
231
+
188
232
  state = analyzer(state, scores, rotation_matrix, rotation_index=index)
189
233
  return (state, index + 1), None
190
234
 
@@ -81,6 +81,17 @@ class CupyBackend(NumpyFFTWBackend):
81
81
  """,
82
82
  "norm_scores",
83
83
  )
84
+
85
+ # Sum of square computation similar to the demeaned variance in pytom
86
+ self.ssum = cp.ReductionKernel(
87
+ f"{ftype} arr",
88
+ f"{ftype} ret",
89
+ "arr * arr",
90
+ "a + b",
91
+ "ret = a",
92
+ "0",
93
+ f"ssum_{ftype}",
94
+ )
84
95
  self.texture_available = find_spec("voltools") is not None
85
96
 
86
97
  def to_backend_array(self, arr: NDArray) -> CupyArray:
@@ -139,17 +150,6 @@ class CupyBackend(NumpyFFTWBackend):
139
150
  peaks = self._array_backend.array(self._array_backend.nonzero(max_filter)).T
140
151
  return peaks
141
152
 
142
- # The default methods in Cupy were oddly slow
143
- def var(self, a, *args, **kwargs):
144
- out = a - self._array_backend.mean(a, *args, **kwargs)
145
- self._array_backend.square(out, out)
146
- out = self._array_backend.mean(out, *args, **kwargs)
147
- return out
148
-
149
- def std(self, a, *args, **kwargs):
150
- out = self.var(a, *args, **kwargs)
151
- return self._array_backend.sqrt(out)
152
-
153
153
  def _get_texture(self, arr: CupyArray, order: int = 3, prefilter: bool = False):
154
154
  key = id(arr)
155
155
  if key in TEXTURE_CACHE:
@@ -197,6 +197,13 @@ class JaxBackend(NumpyFFTWBackend):
197
197
  def _tuple_to_dict(self, data: Tuple) -> Dict:
198
198
  return {x[0]: self._from_hashable(*x[1]) for x in data}
199
199
 
200
+ def _unbatch(self, data, target_ndim, index):
201
+ if not isinstance(data, type(self.zeros(1))):
202
+ return data
203
+ elif data.ndim <= target_ndim:
204
+ return data
205
+ return data[index]
206
+
200
207
  def scan(
201
208
  self,
202
209
  matching_data: type,
@@ -211,12 +218,14 @@ class JaxBackend(NumpyFFTWBackend):
211
218
  Emulates output of :py:meth:`tme.matching_exhaustive.scan` using
212
219
  :py:class:`tme.analyzer.MaxScoreOverRotations`.
213
220
  """
214
- from ._jax_utils import scan as scan_inner
221
+ from ._jax_utils import setup_scan
222
+ from ..analyzer import MaxScoreOverRotations
215
223
 
216
224
  pad_target = True if len(splits) > 1 else False
217
225
  convolution_mode = "valid" if pad_target else "same"
218
226
  target_pad = matching_data.target_padding(pad_target=pad_target)
219
227
 
228
+ score_mask = self.full((1,), fill_value=1, dtype=bool)
220
229
  target_shape = tuple(
221
230
  (x.stop - x.start + p) for x, p in zip(splits[0][0], target_pad)
222
231
  )
@@ -270,8 +279,10 @@ class JaxBackend(NumpyFFTWBackend):
270
279
  cur_args = analyzer_args.copy()
271
280
  cur_args["offset"] = translation_offset
272
281
  cur_args.update(callback_class_args)
282
+ analyzer_kwargs.append(cur_args)
273
283
 
274
- analyzer_kwargs.append(self._dict_to_tuple(cur_args))
284
+ if pad_target:
285
+ score_mask = base._score_mask(fast_shape, shift)
275
286
 
276
287
  _target = self.astype(base._target, self._float_dtype)
277
288
  translation_offsets.append(translation_offset)
@@ -298,7 +309,13 @@ class JaxBackend(NumpyFFTWBackend):
298
309
  create_filter, create_template_filter, create_target_filter = (False,) * 3
299
310
  base, targets = None, self._array_backend.stack(targets)
300
311
 
301
- analyzer_kwargs = tuple(analyzer_kwargs)
312
+ scan_inner = setup_scan(
313
+ analyzer_kwargs=analyzer_kwargs,
314
+ callback_class=callback_class,
315
+ fast_shape=fast_shape,
316
+ rotate_mask=rotate_mask,
317
+ )
318
+
302
319
  states = scan_inner(
303
320
  self.astype(targets, self._float_dtype),
304
321
  self.astype(matching_data.template, self._float_dtype),
@@ -306,17 +323,18 @@ class JaxBackend(NumpyFFTWBackend):
306
323
  matching_data.rotations,
307
324
  template_filter,
308
325
  target_filter,
309
- fast_shape,
310
- rotate_mask,
311
- callback_class,
312
- analyzer_kwargs,
326
+ score_mask,
313
327
  )
314
328
 
329
+ ndim = targets.ndim - 1
315
330
  for index in range(targets.shape[0]):
316
- kwargs = self._tuple_to_dict(analyzer_kwargs[index])
331
+ kwargs = analyzer_kwargs[index]
317
332
  analyzer = callback_class(**kwargs)
333
+ state = [self._unbatch(x, ndim, index) for x in states]
334
+
335
+ if isinstance(analyzer, MaxScoreOverRotations):
336
+ state[2] = rotation_mapping
318
337
 
319
- state = (states[0][index], states[1][index], rotation_mapping)
320
338
  ret.append(analyzer.result(state, **kwargs))
321
339
  return ret
322
340
 
@@ -863,6 +863,17 @@ class MatchingBackend(ABC):
863
863
  Indices of ``k`` largest elements in ``arr``.
864
864
  """
865
865
 
866
+ @abstractmethod
867
+ def ssum(self, arr, *args, **kwargs) -> BackendArray:
868
+ """
869
+ Compute the sum of squares of ``arr``.
870
+
871
+ Returns
872
+ -------
873
+ BackendArray
874
+ Sum of squares with shape ().
875
+ """
876
+
866
877
  def indices(self, *args, **kwargs) -> BackendArray:
867
878
  """
868
879
  Creates an array representing the index grid of an input.
@@ -201,6 +201,9 @@ class NumpyFFTWBackend(_NumpyWrapper, MatchingBackend):
201
201
  sorted_indices = self.unravel_index(indices=sorted_indices, shape=arr.shape)
202
202
  return sorted_indices
203
203
 
204
+ def ssum(self, arr, *args, **kwargs):
205
+ return self.sum(self.square(arr), *args, **kwargs)
206
+
204
207
  def indices(self, *args, **kwargs) -> NDArray:
205
208
  return self._array_backend.indices(*args, **kwargs)
206
209