torchsx 0.2.2__tar.gz → 0.2.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (136) hide show
  1. torchsx-0.2.4/PKG-INFO +239 -0
  2. torchsx-0.2.4/README.md +212 -0
  3. {torchsx-0.2.2 → torchsx-0.2.4}/pyproject.toml +6 -6
  4. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_amortized_training.py +14 -3
  5. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_average_last.py +2 -2
  6. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_cli_pipeline.py +172 -42
  7. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_cli_production.py +91 -11
  8. torchsx-0.2.4/tests/test_completeness.py +84 -0
  9. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_fold_transformer.py +100 -1
  10. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_grid_tooling.py +10 -0
  11. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_large_batch_diagnostics.py +0 -2
  12. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_phase_classes.py +3 -2
  13. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_posterior_prior.py +8 -14
  14. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_prepared_store.py +53 -0
  15. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_tether.py +11 -5
  16. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/__init__.py +1 -1
  17. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/_pipeline.py +195 -131
  18. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/merge_nn.py +3 -3
  19. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/train.py +61 -9
  20. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/times.py +7 -2
  21. torchsx-0.2.4/torchsx/merging/hkl/stats.py +108 -0
  22. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/prepare_store.py +11 -1
  23. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scale/amortized.py +11 -3
  24. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/store.py +27 -0
  25. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/fold_transformer.py +79 -11
  26. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/posterior_target.py +10 -13
  27. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/train_amortized.py +76 -23
  28. torchsx-0.2.4/torchsx.egg-info/PKG-INFO +239 -0
  29. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx.egg-info/SOURCES.txt +1 -0
  30. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx.egg-info/requires.txt +2 -2
  31. torchsx-0.2.2/PKG-INFO +0 -224
  32. torchsx-0.2.2/README.md +0 -197
  33. torchsx-0.2.2/torchsx/merging/hkl/stats.py +0 -78
  34. torchsx-0.2.2/torchsx.egg-info/PKG-INFO +0 -224
  35. {torchsx-0.2.2 → torchsx-0.2.4}/LICENSE +0 -0
  36. {torchsx-0.2.2 → torchsx-0.2.4}/setup.cfg +0 -0
  37. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_amortized_scale.py +0 -0
  38. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_asym_scatter.py +0 -0
  39. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_bg_mean_offset.py +0 -0
  40. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_core.py +0 -0
  41. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_duckdb_load.py +0 -0
  42. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_ewald_offset.py +0 -0
  43. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_filter.py +0 -0
  44. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_geometry.py +0 -0
  45. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_geometry_reciprocal.py +0 -0
  46. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_hkl.py +0 -0
  47. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_io.py +0 -0
  48. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_lazy_imports.py +0 -0
  49. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_levels.py +0 -0
  50. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_measurement_sources.py +0 -0
  51. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_merge_stats.py +0 -0
  52. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_merging_layers.py +0 -0
  53. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_metadata.py +0 -0
  54. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_native_index.py +0 -0
  55. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_native_parse.py +0 -0
  56. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_observation_filter.py +0 -0
  57. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_peer_features.py +0 -0
  58. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_pooling.py +0 -0
  59. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_posterior_nodes.py +0 -0
  60. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_posterior_smooth_sigma.py +0 -0
  61. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_pseudo_translation.py +0 -0
  62. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_reciprocal_basis.py +0 -0
  63. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_scatter_mixture.py +0 -0
  64. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_split_gaussian_mean.py +0 -0
  65. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_split_log_grid.py +0 -0
  66. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_split_mixture.py +0 -0
  67. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_stateless_merge.py +0 -0
  68. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_stats.py +0 -0
  69. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_stream.py +0 -0
  70. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_structured_prior.py +0 -0
  71. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_train_schedule.py +0 -0
  72. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_wilson_surface.py +0 -0
  73. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_writeback.py +0 -0
  74. {torchsx-0.2.2 → torchsx-0.2.4}/tests/test_writeback_native.py +0 -0
  75. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/_categorical.py +0 -0
  76. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/_core.py +0 -0
  77. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/__init__.py +0 -0
  78. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/merge_stats.py +0 -0
  79. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/cli/stats.py +0 -0
  80. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/geometry.py +0 -0
  81. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/__init__.py +0 -0
  82. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_crystfel.py +0 -0
  83. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_duckdb.py +0 -0
  84. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_metadata.py +0 -0
  85. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_native.py +0 -0
  86. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_parser.py +0 -0
  87. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_safetensors.py +0 -0
  88. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/_writeback.py +0 -0
  89. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/io/sources.py +0 -0
  90. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/levels.py +0 -0
  91. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/measurement.py +0 -0
  92. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merge_stats.py +0 -0
  93. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/__init__.py +0 -0
  94. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/config.py +0 -0
  95. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/hkl/__init__.py +0 -0
  96. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/hkl/filter.py +0 -0
  97. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/hkl/hkl.py +0 -0
  98. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/hkl/noise.py +0 -0
  99. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/likelihood/__init__.py +0 -0
  100. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/likelihood/base.py +0 -0
  101. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/likelihood/detector_noise.py +0 -0
  102. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/likelihood/poisson.py +0 -0
  103. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/observations.py +0 -0
  104. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/priors/__init__.py +0 -0
  105. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/priors/modulation.py +0 -0
  106. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/priors/reciprocal_basis.py +0 -0
  107. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/__init__.py +0 -0
  108. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/_posterior_nodes_triton.py +0 -0
  109. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/_split_log_grid_triton.py +0 -0
  110. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/_split_mixture_triton.py +0 -0
  111. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/base.py +0 -0
  112. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/quadrature/split_log_grid.py +0 -0
  113. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scale/__init__.py +0 -0
  114. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scale/pseudo_translation.py +0 -0
  115. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scatter/__init__.py +0 -0
  116. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scatter/base.py +0 -0
  117. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scatter/gengauss.py +0 -0
  118. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scatter/mixture.py +0 -0
  119. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/scatter/predicted.py +0 -0
  120. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/tether/__init__.py +0 -0
  121. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/tether/batch.py +0 -0
  122. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/tether/kmean.py +0 -0
  123. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/tether/terms.py +0 -0
  124. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/merging/tether/tether.py +0 -0
  125. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/__init__.py +0 -0
  126. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/heads.py +0 -0
  127. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/merge_target.py +0 -0
  128. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/metrics.py +0 -0
  129. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/objective.py +0 -0
  130. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/obs_features.py +0 -0
  131. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/nn/pooling.py +0 -0
  132. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/stats.py +0 -0
  133. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx/stream.py +0 -0
  134. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx.egg-info/dependency_links.txt +0 -0
  135. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx.egg-info/entry_points.txt +0 -0
  136. {torchsx-0.2.2 → torchsx-0.2.4}/torchsx.egg-info/top_level.txt +0 -0
torchsx-0.2.4/PKG-INFO ADDED
@@ -0,0 +1,239 @@
1
+ Metadata-Version: 2.4
2
+ Name: torchsx
3
+ Version: 0.2.4
4
+ Summary: A PyTorch-based scaling and merging library for serial crystallography
5
+ Author: Hans Peter Seidel
6
+ License-Expression: MIT
7
+ Project-URL: Repository, https://github.com/HatPdotS/TorchSX
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: Programming Language :: Python :: 3 :: Only
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: Topic :: Scientific/Engineering :: Physics
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown
14
+ License-File: LICENSE
15
+ Requires-Dist: torchref<0.9,>=0.8.1
16
+ Requires-Dist: torch<2.15,>=2.7
17
+ Requires-Dist: numpy>=2
18
+ Requires-Dist: safetensors>=0.4
19
+ Requires-Dist: matplotlib>=3.7
20
+ Requires-Dist: duckdb>=1.0
21
+ Requires-Dist: torchsx-native==0.1.*
22
+ Provides-Extra: dev
23
+ Requires-Dist: pytest>=6.0; extra == "dev"
24
+ Requires-Dist: black>=21.5; extra == "dev"
25
+ Requires-Dist: isort>=5.9; extra == "dev"
26
+ Dynamic: license-file
27
+
28
+ <p align="center">
29
+ <picture>
30
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/torchsx-banner-dark.png">
31
+ <img src="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/torchsx-banner-light.png" alt="TorchSX" width="100%">
32
+ </picture>
33
+ </p>
34
+
35
+ <h1 align="center">A Py<ins>Torch</ins>-based scaling and merging library for serial crystallography (<ins>SX</ins>)</h1>
36
+
37
+ <div align="center">
38
+
39
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
40
+ [![PyTorch](https://img.shields.io/badge/PyTorch-2.7+-ee4c2c.svg)](https://pytorch.org/)
41
+ [![TorchRef](https://img.shields.io/badge/TorchRef-0.8-f97316.svg)](https://github.com/HatPdotS/TorchRef)
42
+ [![CUDA](https://img.shields.io/badge/CUDA-supported-76b900.svg)](https://developer.nvidia.com/cuda-zone)
43
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
44
+
45
+ </div>
46
+
47
+ TorchSX merges serial crystallography data with an error model learned for
48
+ every single observation instead of a few parameters per crystal. On the
49
+ time-resolved tubulin data below, its difference maps from 3,875 images per
50
+ state are as good as Partialator's from 31,000.
51
+
52
+ <p align="center">
53
+ <img src="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/readme-tubulin-overview.webp" alt="TorchSX overview: a fold transformer predicts a scale and a scatter law for every observation; tubulin azo-combretastatin difference maps from TorchSX and Partialator against the number of images per state" width="100%">
54
+ </p>
55
+
56
+ **Top:** a fold transformer reads one crystal's observations together:
57
+ geometry, I, σ and I/Ī for most of them, geometry only for the fold whose
58
+ observations it is predicting, and the crystal's cell, energy and resolution.
59
+ For every observation it predicts a scale K and a scatter law, a mixture of
60
+ Gaussians in log intensity whose components cover pooled, partial, outlier
61
+ and sharp observations. **Bottom:** light-minus-dark difference maps of
62
+ azo-combretastatin unbinding from tubulin, 1 ns after light activation
63
+ (Wranik et al.<sup>[1](#references)</sup>), against the number of images per
64
+ state in the merge. The correlation with the noise-free map is estimated as
65
+ √CC, where CC is the correlation between the maps of two independent merges,
66
+ A and B, of disjoint images; open symbols are the full merge (A + B). Shaded: TorchSX with
67
+ 3,875 images per state and Partialator with 31,000 (8×), the closest pair in
68
+ that correlation.
69
+
70
+ ## What it does
71
+
72
+ - **Learns an error model per observation.** One network per dataset predicts,
73
+ for every observation, a scale K (partiality, pulse intensity, crystal
74
+ volume) and how far the measurement may plausibly fall from it.
75
+ - **Never uses an observation to predict itself.** Each crystal's observations
76
+ are split into five folds. The network reads the whole crystal once per
77
+ fold, with that fold's measurements hidden, and predicts the hidden ones
78
+ from their geometry and the rest of the crystal. On the way it corrects the
79
+ crystal's orientation and photon energy.
80
+ - **Merges by posterior, not by averaging.** Each reflection's intensity is
81
+ integrated out under a Wilson prior. `F-obs` and `SIGF-obs` are the mean and
82
+ standard deviation of the posterior of |F| given all its observations.
83
+ - **Cannot see light or dark.** The scale network is never given a state,
84
+ delay or timepoint, so it cannot absorb a difference signal. One network
85
+ serves all states, and each state is merged on its own.
86
+ - **Holds the merge at the sum of the spots.** A tether keeps the mean scale
87
+ free of trends in resolution and direction, so partiality stays in K and the
88
+ fall-off of the integrated intensities stays in the merged data.
89
+ - **Handles pseudo-translation.** Translational NCS, lattice translocation
90
+ defects and stacking faults are found in the data and modelled in the prior
91
+ and the scale model.
92
+
93
+ TorchSX reads integrated CrystFEL `.stream` files and probixi `.duckdb` runs
94
+ and writes merged MTZs. DIALS input is in progress. TorchSX is under active
95
+ development: if something behaves unexpectedly, please open an issue on GitHub.
96
+
97
+ The output works with any refinement or map program. Refinement and difference
98
+ maps in this repo were done with [TorchRef](https://github.com/HatPdotS/TorchRef)<sup>[2](#references)</sup>.
99
+
100
+ ## Install
101
+
102
+ ```bash
103
+ pip install torchsx
104
+ ```
105
+
106
+ Python ≥ 3.10. This brings TorchRef 0.8, torch, triton, gemmi and
107
+ `torchsx-native`, the compiled `.stream` reader for your platform.
108
+
109
+ - **GPU.** TorchSX is meant to run on a GPU: its core algorithms are Triton
110
+ kernels, and CPU runtime, or whether it runs on a CPU at all, has not been
111
+ investigated.
112
+ - **GPU memory** should stay below 10 GB. It scales with the redundancy
113
+ requested for training (default 100) and the number of unique reflections,
114
+ not with the amount of data.
115
+ - **Host memory** does not scale with the amount of data either. The inputs
116
+ are prepared block by block into an on-disk store of 56 bytes per
117
+ observation, and training and merging read it one batch at a time, the next
118
+ batch while the current one is computed.
119
+
120
+ From a checkout, build the native package first (needs Rust, see
121
+ [native/README.md](https://github.com/HatPdotS/TorchSX/blob/master/native/README.md)):
122
+ `pip install ./native`, then `pip install -e .`.
123
+
124
+ ## Quick start
125
+
126
+ ```bash
127
+ torchsx.train indexed.stream --spacegroup 'P 63' --dmin 2.0 -o run/
128
+ ```
129
+
130
+ This trains the network (400 updates by default) and merges the data with it.
131
+ `run/` then contains:
132
+
133
+ ```
134
+ run/
135
+ ├── merged.mtz merged amplitudes from all crystals
136
+ ├── merged_half1.mtz the same from one half of the crystals
137
+ ├── merged_half2.mtz ... and from the other half
138
+ ├── merged.metrics.json in-run precision: CC1/2 and R_split between the two halves
139
+ ├── best.pt the network at the lowest validation loss; it made the merges
140
+ ├── last.pt the network at the end of training
141
+ ├── prepared/ the prepared observations on disk, reused by later runs on the same input
142
+ ├── run.json the resolved settings
143
+ ├── training_summary.json updates, best step, run time; time and GPU memory by phase
144
+ └── metrics.jsonl training log: one line per update and per validation
145
+ ```
146
+
147
+ Each MTZ holds:
148
+
149
+ | Column | MTZ type | Content |
150
+ |---|---|---|
151
+ | `H`, `K`, `L` | H | Miller index; Friedel mates merged |
152
+ | `F-obs` | F | posterior mean of the amplitude \|F\| |
153
+ | `SIGF-obs` | Q | posterior standard deviation of the amplitude |
154
+ | `R-free-flags` | I | refinement free set: 0 = free (5 %), 1 = work |
155
+
156
+ There are no intensity columns. `F-obs` is already the posterior estimate of
157
+ the amplitude, so it must not go through French–Wilson again; refinement
158
+ programs use it as it is. With `--no-merge-friedel` the mates stay apart and the
159
+ files carry `F-obs(+)`, `SIGF-obs(+)`, `F-obs(-)` and `SIGF-obs(-)`.
160
+ `--free-from earlier.mtz` takes the free set from an existing file.
161
+
162
+ The two halves split the crystals, not the training: both are merged with the
163
+ same network, so the CC1/2 and R_split in `merged.metrics.json` measure the
164
+ in-run precision, not the agreement of independent merges. For independent
165
+ halves, train two runs on disjoint sets of patterns.
166
+
167
+ ### Light and dark
168
+
169
+ ```bash
170
+ torchsx.train --split dark=dark.stream --split light=light.stream \
171
+ --spacegroup 'P 63' --dmin 2.0 -o run/
172
+ ```
173
+
174
+ One network is trained on both states, and each state is merged on its own:
175
+ `merged_dark.mtz` and `merged_light.mtz`, each with its two halves. A state
176
+ can come from several streams (repeat its name), and streams with a CrystFEL
177
+ custom-split file work too (`--split-file`); patterns the file assigns to no
178
+ state still train the network. See
179
+ [States](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md#states).
180
+
181
+ ## The model
182
+
183
+ The signal that reached the detector aperture of observation *j* of
184
+ reflection *h* is
185
+
186
+ S_j = K_j · I_h · exp(ε_j)
187
+
188
+ where I_h is the reflection's true intensity, a latent variable with a Wilson
189
+ prior; K_j is the predicted scale of this observation; and ε_j is a random
190
+ multiplicative scatter, with E[exp(ε_j)] = 1, whose distribution is also
191
+ predicted for this observation: a mixture of four Gaussians in log space, each
192
+ of which may be wider below its centre than above it, as the scatter of
193
+ partial observations is. The detector then records Poisson counts of S_j on
194
+ top of the background.
195
+
196
+ The network is trained on random batches of whole crystals, sized to a given
197
+ multiplicity (`--redundancy`, default 100), by maximising the marginal
198
+ likelihood of their observations with every I_h integrated out. No point
199
+ estimate of the intensity enters the loss, so the quantity that is scored is
200
+ the quantity that is merged. Like a weather forecaster scored on its
201
+ forecasts, the network is rewarded for confidence where the data agree and
202
+ penalised for it where they do not; a wider scatter costs likelihood, so
203
+ agreement cannot be bought by widening it. The integrals over I_h and ε_j run
204
+ in Triton kernels that never hold the full integrand in memory.
205
+
206
+ The [command-line reference](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md#model)
207
+ describes the model, the tether and the pseudo-translation search in more
208
+ detail.
209
+
210
+ ## Other commands
211
+
212
+ - `torchsx.merge run/best.pt more.stream -o merged.mtz` merges new or additional
213
+ data with a trained network.
214
+ - `torchsx.merge-stats A/merged.mtz B/merged.mtz -o stats/` compares merges:
215
+ completeness, CC1/2, R_split and, for light/dark pairs, the information in
216
+ ΔF ([docs/merge_stats.md](https://github.com/HatPdotS/TorchSX/blob/master/docs/merge_stats.md)).
217
+ `torchsx.merge-stats run/` finds the merges of each state in a run directory
218
+ and writes `run/merge_stats/`.
219
+ - `torchsx.observation-stats indexed.stream --spacegroup 'P 63' -o figures/`
220
+ plots the raw observations: intensity distributions, skewness, examples
221
+ ([docs/stats.md](https://github.com/HatPdotS/TorchSX/blob/master/docs/stats.md)).
222
+
223
+ All options, gain and measurement settings, caching and reproducibility are in
224
+ the [command-line reference](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md).
225
+
226
+ In Python, `torchsx.SFXStream.read('indexed.stream', spacegroup='P 63')` reads a
227
+ stream with its geometry and measurement settings.
228
+
229
+ ## References
230
+
231
+ 1. Wranik, M. et al. Watching the release of a photopharmacological drug from
232
+ tubulin using time-resolved serial crystallography. *Nat. Commun.* **14**,
233
+ 903 (2023). [doi:10.1038/s41467-023-36481-5](https://doi.org/10.1038/s41467-023-36481-5)
234
+ 2. Seidel, H.-P., Standfuss, J. & Weinert, T. TorchRef: an open-source PyTorch
235
+ framework for crystallographic refinement. *J. Appl. Cryst.*, accepted (2026).
236
+
237
+ ## License
238
+
239
+ MIT. See [LICENSE](https://github.com/HatPdotS/TorchSX/blob/master/LICENSE).
@@ -0,0 +1,212 @@
1
+ <p align="center">
2
+ <picture>
3
+ <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/torchsx-banner-dark.png">
4
+ <img src="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/torchsx-banner-light.png" alt="TorchSX" width="100%">
5
+ </picture>
6
+ </p>
7
+
8
+ <h1 align="center">A Py<ins>Torch</ins>-based scaling and merging library for serial crystallography (<ins>SX</ins>)</h1>
9
+
10
+ <div align="center">
11
+
12
+ [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
13
+ [![PyTorch](https://img.shields.io/badge/PyTorch-2.7+-ee4c2c.svg)](https://pytorch.org/)
14
+ [![TorchRef](https://img.shields.io/badge/TorchRef-0.8-f97316.svg)](https://github.com/HatPdotS/TorchRef)
15
+ [![CUDA](https://img.shields.io/badge/CUDA-supported-76b900.svg)](https://developer.nvidia.com/cuda-zone)
16
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
17
+
18
+ </div>
19
+
20
+ TorchSX merges serial crystallography data with an error model learned for
21
+ every single observation instead of a few parameters per crystal. On the
22
+ time-resolved tubulin data below, its difference maps from 3,875 images per
23
+ state are as good as Partialator's from 31,000.
24
+
25
+ <p align="center">
26
+ <img src="https://raw.githubusercontent.com/HatPdotS/TorchSX/master/assets/readme-tubulin-overview.webp" alt="TorchSX overview: a fold transformer predicts a scale and a scatter law for every observation; tubulin azo-combretastatin difference maps from TorchSX and Partialator against the number of images per state" width="100%">
27
+ </p>
28
+
29
+ **Top:** a fold transformer reads one crystal's observations together:
30
+ geometry, I, σ and I/Ī for most of them, geometry only for the fold whose
31
+ observations it is predicting, and the crystal's cell, energy and resolution.
32
+ For every observation it predicts a scale K and a scatter law, a mixture of
33
+ Gaussians in log intensity whose components cover pooled, partial, outlier
34
+ and sharp observations. **Bottom:** light-minus-dark difference maps of
35
+ azo-combretastatin unbinding from tubulin, 1 ns after light activation
36
+ (Wranik et al.<sup>[1](#references)</sup>), against the number of images per
37
+ state in the merge. The correlation with the noise-free map is estimated as
38
+ √CC, where CC is the correlation between the maps of two independent merges,
39
+ A and B, of disjoint images; open symbols are the full merge (A + B). Shaded: TorchSX with
40
+ 3,875 images per state and Partialator with 31,000 (8×), the closest pair in
41
+ that correlation.
42
+
43
+ ## What it does
44
+
45
+ - **Learns an error model per observation.** One network per dataset predicts,
46
+ for every observation, a scale K (partiality, pulse intensity, crystal
47
+ volume) and how far the measurement may plausibly fall from it.
48
+ - **Never uses an observation to predict itself.** Each crystal's observations
49
+ are split into five folds. The network reads the whole crystal once per
50
+ fold, with that fold's measurements hidden, and predicts the hidden ones
51
+ from their geometry and the rest of the crystal. On the way it corrects the
52
+ crystal's orientation and photon energy.
53
+ - **Merges by posterior, not by averaging.** Each reflection's intensity is
54
+ integrated out under a Wilson prior. `F-obs` and `SIGF-obs` are the mean and
55
+ standard deviation of the posterior of |F| given all its observations.
56
+ - **Cannot see light or dark.** The scale network is never given a state,
57
+ delay or timepoint, so it cannot absorb a difference signal. One network
58
+ serves all states, and each state is merged on its own.
59
+ - **Holds the merge at the sum of the spots.** A tether keeps the mean scale
60
+ free of trends in resolution and direction, so partiality stays in K and the
61
+ fall-off of the integrated intensities stays in the merged data.
62
+ - **Handles pseudo-translation.** Translational NCS, lattice translocation
63
+ defects and stacking faults are found in the data and modelled in the prior
64
+ and the scale model.
65
+
66
+ TorchSX reads integrated CrystFEL `.stream` files and probixi `.duckdb` runs
67
+ and writes merged MTZs. DIALS input is in progress. TorchSX is under active
68
+ development: if something behaves unexpectedly, please open an issue on GitHub.
69
+
70
+ The output works with any refinement or map program. Refinement and difference
71
+ maps in this repo were done with [TorchRef](https://github.com/HatPdotS/TorchRef)<sup>[2](#references)</sup>.
72
+
73
+ ## Install
74
+
75
+ ```bash
76
+ pip install torchsx
77
+ ```
78
+
79
+ Python ≥ 3.10. This brings TorchRef 0.8, torch, triton, gemmi and
80
+ `torchsx-native`, the compiled `.stream` reader for your platform.
81
+
82
+ - **GPU.** TorchSX is meant to run on a GPU: its core algorithms are Triton
83
+ kernels, and CPU runtime, or whether it runs on a CPU at all, has not been
84
+ investigated.
85
+ - **GPU memory** should stay below 10 GB. It scales with the redundancy
86
+ requested for training (default 100) and the number of unique reflections,
87
+ not with the amount of data.
88
+ - **Host memory** does not scale with the amount of data either. The inputs
89
+ are prepared block by block into an on-disk store of 56 bytes per
90
+ observation, and training and merging read it one batch at a time, the next
91
+ batch while the current one is computed.
92
+
93
+ From a checkout, build the native package first (needs Rust, see
94
+ [native/README.md](https://github.com/HatPdotS/TorchSX/blob/master/native/README.md)):
95
+ `pip install ./native`, then `pip install -e .`.
96
+
97
+ ## Quick start
98
+
99
+ ```bash
100
+ torchsx.train indexed.stream --spacegroup 'P 63' --dmin 2.0 -o run/
101
+ ```
102
+
103
+ This trains the network (400 updates by default) and merges the data with it.
104
+ `run/` then contains:
105
+
106
+ ```
107
+ run/
108
+ ├── merged.mtz merged amplitudes from all crystals
109
+ ├── merged_half1.mtz the same from one half of the crystals
110
+ ├── merged_half2.mtz ... and from the other half
111
+ ├── merged.metrics.json in-run precision: CC1/2 and R_split between the two halves
112
+ ├── best.pt the network at the lowest validation loss; it made the merges
113
+ ├── last.pt the network at the end of training
114
+ ├── prepared/ the prepared observations on disk, reused by later runs on the same input
115
+ ├── run.json the resolved settings
116
+ ├── training_summary.json updates, best step, run time; time and GPU memory by phase
117
+ └── metrics.jsonl training log: one line per update and per validation
118
+ ```
119
+
120
+ Each MTZ holds:
121
+
122
+ | Column | MTZ type | Content |
123
+ |---|---|---|
124
+ | `H`, `K`, `L` | H | Miller index; Friedel mates merged |
125
+ | `F-obs` | F | posterior mean of the amplitude \|F\| |
126
+ | `SIGF-obs` | Q | posterior standard deviation of the amplitude |
127
+ | `R-free-flags` | I | refinement free set: 0 = free (5 %), 1 = work |
128
+
129
+ There are no intensity columns. `F-obs` is already the posterior estimate of
130
+ the amplitude, so it must not go through French–Wilson again; refinement
131
+ programs use it as it is. With `--no-merge-friedel` the mates stay apart and the
132
+ files carry `F-obs(+)`, `SIGF-obs(+)`, `F-obs(-)` and `SIGF-obs(-)`.
133
+ `--free-from earlier.mtz` takes the free set from an existing file.
134
+
135
+ The two halves split the crystals, not the training: both are merged with the
136
+ same network, so the CC1/2 and R_split in `merged.metrics.json` measure the
137
+ in-run precision, not the agreement of independent merges. For independent
138
+ halves, train two runs on disjoint sets of patterns.
139
+
140
+ ### Light and dark
141
+
142
+ ```bash
143
+ torchsx.train --split dark=dark.stream --split light=light.stream \
144
+ --spacegroup 'P 63' --dmin 2.0 -o run/
145
+ ```
146
+
147
+ One network is trained on both states, and each state is merged on its own:
148
+ `merged_dark.mtz` and `merged_light.mtz`, each with its two halves. A state
149
+ can come from several streams (repeat its name), and streams with a CrystFEL
150
+ custom-split file work too (`--split-file`); patterns the file assigns to no
151
+ state still train the network. See
152
+ [States](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md#states).
153
+
154
+ ## The model
155
+
156
+ The signal that reached the detector aperture of observation *j* of
157
+ reflection *h* is
158
+
159
+ S_j = K_j · I_h · exp(ε_j)
160
+
161
+ where I_h is the reflection's true intensity, a latent variable with a Wilson
162
+ prior; K_j is the predicted scale of this observation; and ε_j is a random
163
+ multiplicative scatter, with E[exp(ε_j)] = 1, whose distribution is also
164
+ predicted for this observation: a mixture of four Gaussians in log space, each
165
+ of which may be wider below its centre than above it, as the scatter of
166
+ partial observations is. The detector then records Poisson counts of S_j on
167
+ top of the background.
168
+
169
+ The network is trained on random batches of whole crystals, sized to a given
170
+ multiplicity (`--redundancy`, default 100), by maximising the marginal
171
+ likelihood of their observations with every I_h integrated out. No point
172
+ estimate of the intensity enters the loss, so the quantity that is scored is
173
+ the quantity that is merged. Like a weather forecaster scored on its
174
+ forecasts, the network is rewarded for confidence where the data agree and
175
+ penalised for it where they do not; a wider scatter costs likelihood, so
176
+ agreement cannot be bought by widening it. The integrals over I_h and ε_j run
177
+ in Triton kernels that never hold the full integrand in memory.
178
+
179
+ The [command-line reference](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md#model)
180
+ describes the model, the tether and the pseudo-translation search in more
181
+ detail.
182
+
183
+ ## Other commands
184
+
185
+ - `torchsx.merge run/best.pt more.stream -o merged.mtz` merges new or additional
186
+ data with a trained network.
187
+ - `torchsx.merge-stats A/merged.mtz B/merged.mtz -o stats/` compares merges:
188
+ completeness, CC1/2, R_split and, for light/dark pairs, the information in
189
+ ΔF ([docs/merge_stats.md](https://github.com/HatPdotS/TorchSX/blob/master/docs/merge_stats.md)).
190
+ `torchsx.merge-stats run/` finds the merges of each state in a run directory
191
+ and writes `run/merge_stats/`.
192
+ - `torchsx.observation-stats indexed.stream --spacegroup 'P 63' -o figures/`
193
+ plots the raw observations: intensity distributions, skewness, examples
194
+ ([docs/stats.md](https://github.com/HatPdotS/TorchSX/blob/master/docs/stats.md)).
195
+
196
+ All options, gain and measurement settings, caching and reproducibility are in
197
+ the [command-line reference](https://github.com/HatPdotS/TorchSX/blob/master/docs/cli.md).
198
+
199
+ In Python, `torchsx.SFXStream.read('indexed.stream', spacegroup='P 63')` reads a
200
+ stream with its geometry and measurement settings.
201
+
202
+ ## References
203
+
204
+ 1. Wranik, M. et al. Watching the release of a photopharmacological drug from
205
+ tubulin using time-resolved serial crystallography. *Nat. Commun.* **14**,
206
+ 903 (2023). [doi:10.1038/s41467-023-36481-5](https://doi.org/10.1038/s41467-023-36481-5)
207
+ 2. Seidel, H.-P., Standfuss, J. & Weinert, T. TorchRef: an open-source PyTorch
208
+ framework for crystallographic refinement. *J. Appl. Cryst.*, accepted (2026).
209
+
210
+ ## License
211
+
212
+ MIT. See [LICENSE](https://github.com/HatPdotS/TorchSX/blob/master/LICENSE).
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "torchsx"
7
- version = "0.2.2"
7
+ version = "0.2.4"
8
8
  description = "A PyTorch-based scaling and merging library for serial crystallography"
9
9
  readme = "README.md"
10
10
  license = "MIT"
@@ -20,11 +20,11 @@ classifiers = [
20
20
  dependencies = [
21
21
  # Brings torch, triton and gemmi. TorchRef's minor releases change its API,
22
22
  # so the bound follows the minor version torchsx is tested against.
23
- "torchref>=0.7.0,<0.8",
24
- # The compiled fold encoder is version-sensitive: torch 2.13's inductor fails
25
- # on its backward pass (CantSplit) and 2.11 fails too. Pinned to 2.9, the
26
- # minor version it is known to train with.
27
- "torch==2.9.*",
23
+ "torchref>=0.8.1,<0.9",
24
+ # The fold encoder compiles with flex attention and dynamic shapes from 2.7
25
+ # on (compile_options sets inductor per version); where torch cannot compile
26
+ # it, it runs uncompiled.
27
+ "torch>=2.7,<2.15",
28
28
  "numpy>=2", # the Python writer's float32 arithmetic (NEP 50)
29
29
  "safetensors>=0.4", # the .sfx working format
30
30
  "matplotlib>=3.7", # the figures of torchsx.observation-stats and torchsx.merge-stats
@@ -292,8 +292,18 @@ def test_per_condition_targets_keep_the_difference(tmp_path):
292
292
  tr.close()
293
293
 
294
294
 
295
- @pytest.mark.parametrize("prior", ["wilson", "none"])
296
- def test_evaluation_changes_no_state(tmp_path, prior):
295
+ def test_flat_prior_is_rejected(tmp_path):
296
+ """The flat prior was removed (#24): its posterior runs off the top of the grid."""
297
+ prep, params, *_ = _synthetic(n_hkl=12, n_xtal=24, per=12)
298
+ scale = AmortizedScale.from_prep(prep, d_model=8, context_folds=2)
299
+ with pytest.raises(ValueError, match="removed"):
300
+ AmortizedTrainer(
301
+ prep, scale, MarginalNLL(SplitLogGrid(n_points=96), Poisson()), params,
302
+ config=TrainConfig(device="cpu", outdir=str(tmp_path), posterior_prior="none"),
303
+ )
304
+
305
+
306
+ def test_evaluation_changes_no_state(tmp_path):
297
307
  """Scoring held-out data must not feed its intensities into later training."""
298
308
  prep, params, *_ = _synthetic(n_hkl=12, n_xtal=24, per=12, n_cond=2)
299
309
  scale = AmortizedScale.from_prep(
@@ -303,7 +313,7 @@ def test_evaluation_changes_no_state(tmp_path, prior):
303
313
  prep, scale, MarginalNLL(SplitLogGrid(n_points=96), Poisson()), params,
304
314
  config=TrainConfig(
305
315
  max_obs_per_batch=24, heldout_frac=0.25, device="cpu", outdir=str(tmp_path),
306
- posterior_prior=prior, posterior_k_coarse=5, posterior_m_fine=17,
316
+ posterior_prior="wilson", posterior_k_coarse=5, posterior_m_fine=17,
307
317
  ),
308
318
  )
309
319
 
@@ -454,6 +464,7 @@ def test_deferred_encoder_backward_is_the_one_pass_gradient(tmp_path):
454
464
  torch.testing.assert_close(grads[1][n], grads[0][n], rtol=1e-5, atol=1e-7, msg=n)
455
465
 
456
466
 
467
+ @pytest.mark.compile
457
468
  @pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled fold path runs on CUDA only")
458
469
  def test_fold_predictions_without_autograd_run_flex_and_match_the_masked_kernel():
459
470
  """Without autograd the fold path runs each block under a checkpoint with autograd
@@ -115,8 +115,8 @@ def test_without_averaging_the_latest_update_stays(tmp_path):
115
115
  def _recorded_run(tr, oscillating, steps=6):
116
116
  after, step = [], tr.step
117
117
 
118
- def recorded(rows):
119
- rec = step(rows)
118
+ def recorded(rows, **kwargs):
119
+ rec = step(rows, **kwargs)
120
120
  after.append(_params(tr))
121
121
  return rec
122
122