midas-integrate-v2 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (127) hide show
  1. midas_integrate_v2-0.1.0/PKG-INFO +271 -0
  2. midas_integrate_v2-0.1.0/README.md +236 -0
  3. midas_integrate_v2-0.1.0/midas_integrate_v2/__init__.py +216 -0
  4. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/__init__.py +59 -0
  5. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/build.py +121 -0
  6. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/differentiable_gain.py +219 -0
  7. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/differentiable_mask.py +147 -0
  8. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/hard.py +205 -0
  9. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/mask.py +66 -0
  10. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/polygon.py +438 -0
  11. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/soft.py +194 -0
  12. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/subpixel.py +311 -0
  13. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/trans_opt.py +69 -0
  14. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/variance.py +372 -0
  15. midas_integrate_v2-0.1.0/midas_integrate_v2/binning/write_v1.py +262 -0
  16. midas_integrate_v2-0.1.0/midas_integrate_v2/bootstrap.py +241 -0
  17. midas_integrate_v2-0.1.0/midas_integrate_v2/cli.py +579 -0
  18. midas_integrate_v2-0.1.0/midas_integrate_v2/compat/__init__.py +13 -0
  19. midas_integrate_v2-0.1.0/midas_integrate_v2/compat/from_v1.py +191 -0
  20. midas_integrate_v2-0.1.0/midas_integrate_v2/compat/pyfai.py +84 -0
  21. midas_integrate_v2-0.1.0/midas_integrate_v2/compat/to_v1.py +83 -0
  22. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/__init__.py +33 -0
  23. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/absorption.py +127 -0
  24. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/background.py +169 -0
  25. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/binning_q.py +41 -0
  26. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/compton.py +127 -0
  27. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/delta_r_k.py +101 -0
  28. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/integrated.py +154 -0
  29. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/intensity.py +212 -0
  30. midas_integrate_v2-0.1.0/midas_integrate_v2/corrections/spline.py +126 -0
  31. midas_integrate_v2-0.1.0/midas_integrate_v2/dac/__init__.py +18 -0
  32. midas_integrate_v2-0.1.0/midas_integrate_v2/dac/gasket_mask.py +167 -0
  33. midas_integrate_v2-0.1.0/midas_integrate_v2/diagnostics.py +90 -0
  34. midas_integrate_v2-0.1.0/midas_integrate_v2/diff/__init__.py +11 -0
  35. midas_integrate_v2-0.1.0/midas_integrate_v2/diff/soft_bin.py +129 -0
  36. midas_integrate_v2-0.1.0/midas_integrate_v2/forward/__init__.py +3 -0
  37. midas_integrate_v2-0.1.0/midas_integrate_v2/forward/pixels.py +74 -0
  38. midas_integrate_v2-0.1.0/midas_integrate_v2/grazing/__init__.py +9 -0
  39. midas_integrate_v2-0.1.0/midas_integrate_v2/grazing/gisaxs.py +97 -0
  40. midas_integrate_v2-0.1.0/midas_integrate_v2/gui/__init__.py +10 -0
  41. midas_integrate_v2-0.1.0/midas_integrate_v2/gui/ring_picker.py +78 -0
  42. midas_integrate_v2-0.1.0/midas_integrate_v2/inelastic/__init__.py +4 -0
  43. midas_integrate_v2-0.1.0/midas_integrate_v2/inelastic/regroup.py +63 -0
  44. midas_integrate_v2-0.1.0/midas_integrate_v2/io/__init__.py +23 -0
  45. midas_integrate_v2-0.1.0/midas_integrate_v2/io/aps_dm.py +115 -0
  46. midas_integrate_v2-0.1.0/midas_integrate_v2/io/milk.py +220 -0
  47. midas_integrate_v2-0.1.0/midas_integrate_v2/io/mtex.py +120 -0
  48. midas_integrate_v2-0.1.0/midas_integrate_v2/io/multimodal.py +59 -0
  49. midas_integrate_v2-0.1.0/midas_integrate_v2/io/writers.py +474 -0
  50. midas_integrate_v2-0.1.0/midas_integrate_v2/kernels/__init__.py +8 -0
  51. midas_integrate_v2-0.1.0/midas_integrate_v2/kernels/integrate.py +140 -0
  52. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/__init__.py +38 -0
  53. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/bayesian.py +50 -0
  54. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/geometry.py +122 -0
  55. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/multi.py +204 -0
  56. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/profile.py +73 -0
  57. midas_integrate_v2-0.1.0/midas_integrate_v2/losses/quasi_2d.py +190 -0
  58. midas_integrate_v2-0.1.0/midas_integrate_v2/pdf.py +305 -0
  59. midas_integrate_v2-0.1.0/midas_integrate_v2/pipelines/__init__.py +10 -0
  60. midas_integrate_v2-0.1.0/midas_integrate_v2/pipelines/drift.py +135 -0
  61. midas_integrate_v2-0.1.0/midas_integrate_v2/pipelines/energy_sweep.py +112 -0
  62. midas_integrate_v2-0.1.0/midas_integrate_v2/ring_detect.py +191 -0
  63. midas_integrate_v2-0.1.0/midas_integrate_v2/spec.py +236 -0
  64. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/__init__.py +52 -0
  65. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/frame_source.py +467 -0
  66. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/integrate_stream.py +138 -0
  67. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/multi_detector.py +142 -0
  68. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/normalize.py +150 -0
  69. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/outlier.py +485 -0
  70. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/quality.py +115 -0
  71. midas_integrate_v2-0.1.0/midas_integrate_v2/streaming/triggers.py +71 -0
  72. midas_integrate_v2-0.1.0/midas_integrate_v2/texture/__init__.py +13 -0
  73. midas_integrate_v2-0.1.0/midas_integrate_v2/texture/pole_figure.py +132 -0
  74. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/PKG-INFO +271 -0
  75. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/SOURCES.txt +125 -0
  76. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/dependency_links.txt +1 -0
  77. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/entry_points.txt +6 -0
  78. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/requires.txt +19 -0
  79. midas_integrate_v2-0.1.0/midas_integrate_v2.egg-info/top_level.txt +1 -0
  80. midas_integrate_v2-0.1.0/pyproject.toml +73 -0
  81. midas_integrate_v2-0.1.0/setup.cfg +4 -0
  82. midas_integrate_v2-0.1.0/tests/test_absorption_cylindrical.py +46 -0
  83. midas_integrate_v2-0.1.0/tests/test_azimuthal_sigma_clip.py +247 -0
  84. midas_integrate_v2-0.1.0/tests/test_compton.py +54 -0
  85. midas_integrate_v2-0.1.0/tests/test_dac_gasket.py +94 -0
  86. midas_integrate_v2-0.1.0/tests/test_diagnostics.py +65 -0
  87. midas_integrate_v2-0.1.0/tests/test_empty_subtraction.py +134 -0
  88. midas_integrate_v2-0.1.0/tests/test_followup_losses.py +282 -0
  89. midas_integrate_v2-0.1.0/tests/test_followup_native_geometry.py +162 -0
  90. midas_integrate_v2-0.1.0/tests/test_followup_real_data_ceo2.py +219 -0
  91. midas_integrate_v2-0.1.0/tests/test_followup_real_data_varex.py +114 -0
  92. midas_integrate_v2-0.1.0/tests/test_frame_source_edf.py +38 -0
  93. midas_integrate_v2-0.1.0/tests/test_frame_source_ge.py +53 -0
  94. midas_integrate_v2-0.1.0/tests/test_fxye_sigma_audit.py +144 -0
  95. midas_integrate_v2-0.1.0/tests/test_hex_lattice.py +226 -0
  96. midas_integrate_v2-0.1.0/tests/test_io_milk.py +77 -0
  97. midas_integrate_v2-0.1.0/tests/test_io_mtex.py +51 -0
  98. midas_integrate_v2-0.1.0/tests/test_multi_detector.py +89 -0
  99. midas_integrate_v2-0.1.0/tests/test_normalize_nexus.py +86 -0
  100. midas_integrate_v2-0.1.0/tests/test_outlier_spatial.py +63 -0
  101. midas_integrate_v2-0.1.0/tests/test_parallax.py +105 -0
  102. midas_integrate_v2-0.1.0/tests/test_pdf.py +132 -0
  103. midas_integrate_v2-0.1.0/tests/test_pdf_pdfgetx3_roundtrip.py +96 -0
  104. midas_integrate_v2-0.1.0/tests/test_phase0_spec_compat.py +155 -0
  105. midas_integrate_v2-0.1.0/tests/test_phase1_parity.py +161 -0
  106. midas_integrate_v2-0.1.0/tests/test_phase2_autograd.py +173 -0
  107. midas_integrate_v2-0.1.0/tests/test_phase3_corrections.py +238 -0
  108. midas_integrate_v2-0.1.0/tests/test_phase4_corrections.py +169 -0
  109. midas_integrate_v2-0.1.0/tests/test_phase5_modules.py +131 -0
  110. midas_integrate_v2-0.1.0/tests/test_pipelines.py +102 -0
  111. midas_integrate_v2-0.1.0/tests/test_quality_flags.py +77 -0
  112. midas_integrate_v2-0.1.0/tests/test_texture_pole_figure.py +58 -0
  113. midas_integrate_v2-0.1.0/tests/test_v03_hard_bin.py +216 -0
  114. midas_integrate_v2-0.1.0/tests/test_v03_multi_image.py +281 -0
  115. midas_integrate_v2-0.1.0/tests/test_v03_trans_opt.py +255 -0
  116. midas_integrate_v2-0.1.0/tests/test_v04_cli.py +251 -0
  117. midas_integrate_v2-0.1.0/tests/test_v04_subpixel.py +214 -0
  118. midas_integrate_v2-0.1.0/tests/test_v04_write_map_bin.py +224 -0
  119. midas_integrate_v2-0.1.0/tests/test_v05_polygon.py +235 -0
  120. midas_integrate_v2-0.1.0/tests/test_v06_polygon_parallel.py +131 -0
  121. midas_integrate_v2-0.1.0/tests/test_v07_complete.py +498 -0
  122. midas_integrate_v2-0.1.0/tests/test_v081_pyfai_convention.py +173 -0
  123. midas_integrate_v2-0.1.0/tests/test_v082_no_approximations.py +263 -0
  124. midas_integrate_v2-0.1.0/tests/test_v08_diff_mask.py +242 -0
  125. midas_integrate_v2-0.1.0/tests/test_v08_ring_detect.py +159 -0
  126. midas_integrate_v2-0.1.0/tests/test_v08_streaming.py +427 -0
  127. midas_integrate_v2-0.1.0/tests/test_variance_robustness.py +239 -0
@@ -0,0 +1,271 @@
1
+ Metadata-Version: 2.4
2
+ Name: midas-integrate-v2
3
+ Version: 0.1.0
4
+ Summary: Differentiable, autodiff-aware radial integration for area X-ray detectors. Joint refinement counterpart to midas-integrate.
5
+ Author-email: Hemant Sharma <hsharma@anl.gov>
6
+ License-Expression: BSD-3-Clause
7
+ Project-URL: Homepage, https://github.com/marinerhemant/MIDAS
8
+ Project-URL: Issues, https://github.com/marinerhemant/MIDAS/issues
9
+ Keywords: MIDAS,X-ray,diffraction,radial integration,azimuthal integration,differentiable,PyTorch,HEDM,calibration
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.9
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Topic :: Scientific/Engineering :: Physics
19
+ Requires-Python: >=3.9
20
+ Description-Content-Type: text/markdown
21
+ Requires-Dist: numpy>=1.22
22
+ Requires-Dist: torch>=2.1
23
+ Requires-Dist: midas-integrate>=0.4.0
24
+ Requires-Dist: midas-calibrate-v2>=0.1.0
25
+ Provides-Extra: dev
26
+ Requires-Dist: pytest>=7.0; extra == "dev"
27
+ Requires-Dist: pytest-cov>=4.0; extra == "dev"
28
+ Provides-Extra: edf
29
+ Requires-Dist: fabio>=0.14; extra == "edf"
30
+ Provides-Extra: gui
31
+ Requires-Dist: streamlit>=1.32; extra == "gui"
32
+ Provides-Extra: aps-dm
33
+ Requires-Dist: requests; extra == "aps-dm"
34
+ Provides-Extra: milk
35
+
36
+ # midas-integrate-v2
37
+
38
+ Differentiable, autograd-aware radial integration. Companion to
39
+ [`midas-integrate`](../midas_integrate/README.md) — v2 sits *alongside*
40
+ v1, not in place of it.
41
+
42
+ ```bash
43
+ pip install midas-integrate-v2
44
+ ```
45
+
46
+ ## When to use which
47
+
48
+ | | `midas-integrate` (v1) | `midas-integrate-v2` |
49
+ |---|---|---|
50
+ | Production batch integration of detector frames | ✅ Use this | overkill |
51
+ | Refining geometry / corrections jointly with integrated profile | needs sidecar machinery | ✅ Use this |
52
+ | Joint refinement with `midas-calibrate-v2` | export → re-import | ✅ Native loop |
53
+ | Stage-4 thin-plate spline as a refinable layer | baked binary lookup | ✅ `nn.Module` |
54
+ | Per-ring `δr_k` (F2 fix) inside the radial map | sidecar JSON for downstream | ✅ Native, refinable |
55
+ | Bit-identical hot path | ✅ CSR kernel | ✅ same CSR kernel |
56
+ | Exact polygon-arc-arc bin overlap (no subpixel, no smooth-kernel approx) | ✅ numba kernel | ✅ pure-numpy/torch kernel |
57
+ | Hand-holding student notebooks | — | ✅ 5 notebooks, self-contained |
58
+
59
+ v2 reuses v1's CSR sparse-matmul integration kernel for the forward pass
60
+ (so the hard-binning path stays bit-identical), and adds a parallel
61
+ soft-binning forward path that's differentiable end-to-end.
62
+
63
+ ## Architecture
64
+
65
+ ```
66
+ midas_integrate_v2/
67
+ spec.py # IntegrationSpec — v2-native (iso_R*, a*/phi*) torch tensors
68
+ forward/ # pixel_to_REta from a spec (re-exports calibrate_v2)
69
+ binning/ # build_map (bridges to v1 numba) + MapCache
70
+ kernels/ # hard-bin integrate + profile_1d (v1 parity)
71
+ diff/ # soft-bin integrate (linear interp; differentiable)
72
+ corrections/ # δr_k, RBF spline, polarization, solid-angle, Q-bins
73
+ compat/ # v1 IntegrationParams ⇄ v2 IntegrationSpec
74
+ ```
75
+
76
+ ## Quickstart
77
+
78
+ ### 1. Bit-identical to v1, with a v2-native parameter dataclass
79
+
80
+ ```python
81
+ from midas_integrate.params import parse_params
82
+ from midas_integrate_v2 import (
83
+ spec_from_v1_params, build_geometry, integrate, profile_1d,
84
+ )
85
+
86
+ p = parse_params("paramstest.txt")
87
+ spec = spec_from_v1_params(p) # v2-native
88
+ geom = build_geometry(spec, dtype=torch.float64) # CSR + cached map
89
+ int2d = integrate(image, geom, mode="floor") # bit-identical to v1
90
+ prof = profile_1d(int2d, geom)
91
+ ```
92
+
93
+ ### 2. Joint refinement of geometry against an integrated-profile loss
94
+
95
+ ```python
96
+ from midas_integrate_v2 import (
97
+ spec_from_v1_params, integrate_with_corrections,
98
+ PolarizationCorrection, SolidAngleCorrection, PerRingOffsets,
99
+ RBFResidualCorrection,
100
+ EtaUniformityLoss, ProfileMSELoss, GaussianPriorLoss,
101
+ )
102
+
103
+ spec = spec_from_v1_params(p, requires_grad=True)
104
+ pol = PolarizationCorrection(pol_fraction=0.99, refinable=False)
105
+ sa = SolidAngleCorrection()
106
+ delta_rk = PerRingOffsets(n_rings=12) # F2 fix, refinable
107
+ spline = RBFResidualCorrection(centres, weights) # Stage-4 spline
108
+
109
+ opt = torch.optim.Adam([
110
+ spec.Lsd, spec.BC_y, spec.BC_z, spec.ty, spec.tz,
111
+ *delta_rk.parameters(),
112
+ *spline.parameters(),
113
+ ], lr=1e-3)
114
+
115
+ eta_loss = EtaUniformityLoss(intensity_floor=1.0)
116
+ prior = GaussianPriorLoss({"Lsd": (Lsd_seed, 100.0)})
117
+
118
+ for _ in range(200):
119
+ opt.zero_grad()
120
+ int2d = integrate_with_corrections(
121
+ image, spec,
122
+ residual=spline, per_ring_offsets=delta_rk,
123
+ ring_R_centres_px=ring_centres,
124
+ polarization=pol, solid_angle=sa,
125
+ )
126
+ loss = eta_loss(int2d) + 0.01 * prior(spec)
127
+ loss.backward()
128
+ opt.step()
129
+ ```
130
+
131
+ ### 2b. "Build once, integrate many" pure-torch path
132
+
133
+ ```python
134
+ from midas_integrate_v2 import (
135
+ spec_from_v1_paramstest, SoftBinGeometry,
136
+ integrate_soft, integrate_soft_batch,
137
+ )
138
+
139
+ spec = spec_from_v1_paramstest("paramstest.txt", requires_grad=False)
140
+ geom = SoftBinGeometry.from_spec(spec) # precompute once
141
+ profiles = integrate_soft_batch(images_3d, geom) # (N, n_eta, n_r)
142
+ ```
143
+
144
+ No numba in the call path — useful when you've already imported torch
145
+ and want to avoid the OpenMP runtime conflict with v1's numba mapper.
146
+
147
+ ### 3. Hand off back to v1 for batch integration
148
+
149
+ ```python
150
+ from midas_integrate_v2 import v1_params_from_spec
151
+ from midas_integrate.detector_mapper import build_and_write_map
152
+
153
+ p_v1 = v1_params_from_spec(spec) # tensor → scalar
154
+ build_and_write_map(p_v1, output_dir="run/") # v1 CLI then takes over
155
+ ```
156
+
157
+ ## Design choices
158
+
159
+ - **Implicit gradient strategy**: hard-binning forward keeps bit-parity
160
+ with v1; gradient flows through a parallel soft-binning kernel
161
+ (linear interpolation in R and η). Bin assignments are not
162
+ themselves differentiated — the upstream `(R, η) = pixel_to_REta(...)`
163
+ is, which is the slope you want for refinement.
164
+ - **Map cache**: hashes the same fields as v1's `compute_param_hash` so
165
+ v1 and v2 share `Map.bin` caches and never diverge.
166
+ - **`nn.Module` corrections**: `δr_k`, the Stage-4 RBF spline, and the
167
+ polarization/solid-angle factors are all torch modules with
168
+ `requires_grad`-controllable parameters. Mix and match as the
169
+ optimisation problem demands.
170
+ - **`IntegrationSpec`** uses v2 distortion names (`iso_R2, a1, phi1, …`);
171
+ the `from_v1`/`to_v1` adapters round-trip the chaotic legacy `p0..p14`
172
+ naming losslessly.
173
+
174
+ ## What's in v0.1.0 (the first release)
175
+
176
+ Tested end-to-end against the v1 production pipeline on real Pilatus +
177
+ Varex Aero CeO₂ data; **242 tests + 11 student notebooks**, all green.
178
+
179
+ ### Math correctness
180
+
181
+ - **Exact polygon-area pixel-bin overlap kernel** (Green's theorem on
182
+ circular-arc + radial-segment intersections). No subpixel
183
+ approximation, no smooth-kernel blur — the differentiator vs pyFAI /
184
+ dxchange / DPDAK / nika.
185
+ - **Exact tilt-aware solid-angle correction** (`Lsd² · (n̂·r) / |r|³`).
186
+ Bit-identical to v1 at fp64 on any detector pose.
187
+ - **Exact thin-plate-spline kernel** (`r² log r` with the analytic
188
+ `r=0` limit handled cleanly).
189
+ - **Polarisation correction**: standard `1 − PF · sin²(2θ) · cos²(η − plane)`.
190
+ - **Parallax correction**: `R + parallax · sin(2θ) / px` (matches v1).
191
+ - **Q ↔ R bin edge conversion**: `2θ = 2 arcsin(λ/2d)`, `R = (Lsd/px) tan(2θ)`.
192
+ - **BC ↔ PONI 0.5 px convention** for pyFAI interop pinned in
193
+ `compat.pyfai`; the `make_pyfai_integrator(spec)` helper makes it
194
+ impossible to drop the half-pixel shift.
195
+
196
+ ### Five binning kernels (each clearly labeled)
197
+
198
+ | Kernel | Math | Differentiable in geometry? | Use for |
199
+ |---|---|---|---|
200
+ | `PolygonBinGeometry` | **Exact** polygon-arc-arc | No | Production batch + calibration accuracy |
201
+ | `HardBinGeometry` | Hard floor (one sample per pixel) | No | Max throughput, fixed geometry |
202
+ | `SubpixelBinGeometry` | K×K oversampling of hard | No | Mid-fidelity fast path |
203
+ | `SoftBinGeometry` / `integrate_diff` | Linear-interp soft binning | **Yes** | Refinement / autograd |
204
+ | `MapCache` (wraps v1) | Same as v1 | No | v1-cache interop |
205
+
206
+ ### Differentiable refinement
207
+
208
+ - Autograd through every refinable parameter (Lsd, BC, tilts,
209
+ Parallax, wavelength, all 15 distortion coefficients).
210
+ - All 4 v2 corrections as `nn.Module`s with refinable parameters:
211
+ per-ring δr_k, Stage-4 thin-plate spline, polarisation, solid-angle.
212
+ - 9 loss families: profile MSE / weighted, η-uniformity, peak-position,
213
+ Gaussian prior, multi-image, batched-spec, η-slice, wedge, ring-masked.
214
+
215
+ ### Differentiable bad-pixel mask (`LearnableMask`)
216
+
217
+ The MIDAS differentiator no other azimuthal integrator has. Per-pixel
218
+ inclusion weight is a learnable parameter; train jointly with the
219
+ calibration loss + a sparsity prior, and bad pixels (hot, dead,
220
+ cosmic-ray-prone) get auto-zeroed while good pixels stay at weight ≈ 1.
221
+ Notebook 10 walks through the demo with planted hot pixels.
222
+
223
+ ### Production-deployable pipeline
224
+
225
+ - **Streaming**: `TIFFGlobSource` / `HDF5FrameSource` / `ZarrFrameSource`
226
+ iterators; `FrameNormalizer` (monitor / exposure / transmission);
227
+ `reject_cosmic_rays` (per-pixel temporal sigma-clip);
228
+ `integrate_stream` (out-of-core, memory constant in N-frames).
229
+ - **Variance propagation**: every binning kernel has an
230
+ `integrate_*_with_variance` variant returning `(mean, σ)` per bin.
231
+ Default Poisson; user-supplied variance images supported.
232
+ - **Output writers** with embedded provenance metadata (package
233
+ version, geometry hash, mask fraction, source file names): CSV,
234
+ XYE (Rietveld), FXYE (GSAS), DAT (PDF), 2D-CSV, HDF5.
235
+ - **Per-pixel masks** in every binning kernel — applied at *build time*,
236
+ so masked pixels never enter the integration.
237
+ - **3 CLI scripts**: `midas-integrate-v2` (single frame),
238
+ `midas-integrate-v2-batch` (sweep mode), `midas-integrate-v2-write-map`
239
+ (emit v1-format Map.bin / nMap.bin without numba).
240
+
241
+ ### Pedagogical material (11 notebooks, ~3.5 hrs end-to-end)
242
+
243
+ 01 First Diffraction Pattern → 02 Geometry Intuition → 03 Joint
244
+ Refinement → 04 Multi-Distance Calibration → 05 calibrate-v2 ↔
245
+ integrate-v2 Handoff → 06 Custom Losses → 07 Bayesian UQ → 08 PDF
246
+ Analysis → 09 Production Workflow → 10 Differentiable Mask → 11
247
+ Sweep-mode Batch Processing.
248
+
249
+ Each notebook is self-contained, executes end-to-end, and includes
250
+ "try it yourself" exercises.
251
+
252
+ ### Ecosystem
253
+
254
+ - **pyFAI migration guide** (`docs/MIGRATING_FROM_PYFAI.md`).
255
+ - **Performance benchmark** script (`bench/bench_integrate.py`).
256
+ - **Bootstrap helpers** (`estimate_BC_from_image`,
257
+ `estimate_initial_spec`) for users without a starting paramstest.
258
+ - **Ring auto-detect** (`detect_rings`, `suggest_material`) with built-in
259
+ CeO₂ / LaB₆ / Si / Cr₂O₃ d-spacings (Cr₂O₃ uses JCPDS 38-1479).
260
+
261
+ ## Roadmap
262
+
263
+ - **v0.2** — Polygon kernel GPU port (vectorise the scalar Python
264
+ loops onto torch+CUDA, keeping the math exact). Right answer for
265
+ sub-pixel `RBinSize` builds where the trivial fast path doesn't fire.
266
+ - **v0.3** — Multi-GPU integrate; NeXus-strict HDF5 output;
267
+ integration with the wider HEDM pipeline (peak fitting, indexing).
268
+
269
+ ## License
270
+
271
+ BSD-3-Clause.
@@ -0,0 +1,236 @@
1
+ # midas-integrate-v2
2
+
3
+ Differentiable, autograd-aware radial integration. Companion to
4
+ [`midas-integrate`](../midas_integrate/README.md) — v2 sits *alongside*
5
+ v1, not in place of it.
6
+
7
+ ```bash
8
+ pip install midas-integrate-v2
9
+ ```
10
+
11
+ ## When to use which
12
+
13
+ | | `midas-integrate` (v1) | `midas-integrate-v2` |
14
+ |---|---|---|
15
+ | Production batch integration of detector frames | ✅ Use this | overkill |
16
+ | Refining geometry / corrections jointly with integrated profile | needs sidecar machinery | ✅ Use this |
17
+ | Joint refinement with `midas-calibrate-v2` | export → re-import | ✅ Native loop |
18
+ | Stage-4 thin-plate spline as a refinable layer | baked binary lookup | ✅ `nn.Module` |
19
+ | Per-ring `δr_k` (F2 fix) inside the radial map | sidecar JSON for downstream | ✅ Native, refinable |
20
+ | Bit-identical hot path | ✅ CSR kernel | ✅ same CSR kernel |
21
+ | Exact polygon-arc-arc bin overlap (no subpixel, no smooth-kernel approx) | ✅ numba kernel | ✅ pure-numpy/torch kernel |
22
+ | Hand-holding student notebooks | — | ✅ 5 notebooks, self-contained |
23
+
24
+ v2 reuses v1's CSR sparse-matmul integration kernel for the forward pass
25
+ (so the hard-binning path stays bit-identical), and adds a parallel
26
+ soft-binning forward path that's differentiable end-to-end.
27
+
28
+ ## Architecture
29
+
30
+ ```
31
+ midas_integrate_v2/
32
+ spec.py # IntegrationSpec — v2-native (iso_R*, a*/phi*) torch tensors
33
+ forward/ # pixel_to_REta from a spec (re-exports calibrate_v2)
34
+ binning/ # build_map (bridges to v1 numba) + MapCache
35
+ kernels/ # hard-bin integrate + profile_1d (v1 parity)
36
+ diff/ # soft-bin integrate (linear interp; differentiable)
37
+ corrections/ # δr_k, RBF spline, polarization, solid-angle, Q-bins
38
+ compat/ # v1 IntegrationParams ⇄ v2 IntegrationSpec
39
+ ```
40
+
41
+ ## Quickstart
42
+
43
+ ### 1. Bit-identical to v1, with a v2-native parameter dataclass
44
+
45
+ ```python
46
+ from midas_integrate.params import parse_params
47
+ from midas_integrate_v2 import (
48
+ spec_from_v1_params, build_geometry, integrate, profile_1d,
49
+ )
50
+
51
+ p = parse_params("paramstest.txt")
52
+ spec = spec_from_v1_params(p) # v2-native
53
+ geom = build_geometry(spec, dtype=torch.float64) # CSR + cached map
54
+ int2d = integrate(image, geom, mode="floor") # bit-identical to v1
55
+ prof = profile_1d(int2d, geom)
56
+ ```
57
+
58
+ ### 2. Joint refinement of geometry against an integrated-profile loss
59
+
60
+ ```python
61
+ from midas_integrate_v2 import (
62
+ spec_from_v1_params, integrate_with_corrections,
63
+ PolarizationCorrection, SolidAngleCorrection, PerRingOffsets,
64
+ RBFResidualCorrection,
65
+ EtaUniformityLoss, ProfileMSELoss, GaussianPriorLoss,
66
+ )
67
+
68
+ spec = spec_from_v1_params(p, requires_grad=True)
69
+ pol = PolarizationCorrection(pol_fraction=0.99, refinable=False)
70
+ sa = SolidAngleCorrection()
71
+ delta_rk = PerRingOffsets(n_rings=12) # F2 fix, refinable
72
+ spline = RBFResidualCorrection(centres, weights) # Stage-4 spline
73
+
74
+ opt = torch.optim.Adam([
75
+ spec.Lsd, spec.BC_y, spec.BC_z, spec.ty, spec.tz,
76
+ *delta_rk.parameters(),
77
+ *spline.parameters(),
78
+ ], lr=1e-3)
79
+
80
+ eta_loss = EtaUniformityLoss(intensity_floor=1.0)
81
+ prior = GaussianPriorLoss({"Lsd": (Lsd_seed, 100.0)})
82
+
83
+ for _ in range(200):
84
+ opt.zero_grad()
85
+ int2d = integrate_with_corrections(
86
+ image, spec,
87
+ residual=spline, per_ring_offsets=delta_rk,
88
+ ring_R_centres_px=ring_centres,
89
+ polarization=pol, solid_angle=sa,
90
+ )
91
+ loss = eta_loss(int2d) + 0.01 * prior(spec)
92
+ loss.backward()
93
+ opt.step()
94
+ ```
95
+
96
+ ### 2b. "Build once, integrate many" pure-torch path
97
+
98
+ ```python
99
+ from midas_integrate_v2 import (
100
+ spec_from_v1_paramstest, SoftBinGeometry,
101
+ integrate_soft, integrate_soft_batch,
102
+ )
103
+
104
+ spec = spec_from_v1_paramstest("paramstest.txt", requires_grad=False)
105
+ geom = SoftBinGeometry.from_spec(spec) # precompute once
106
+ profiles = integrate_soft_batch(images_3d, geom) # (N, n_eta, n_r)
107
+ ```
108
+
109
+ No numba in the call path — useful when you've already imported torch
110
+ and want to avoid the OpenMP runtime conflict with v1's numba mapper.
111
+
112
+ ### 3. Hand off back to v1 for batch integration
113
+
114
+ ```python
115
+ from midas_integrate_v2 import v1_params_from_spec
116
+ from midas_integrate.detector_mapper import build_and_write_map
117
+
118
+ p_v1 = v1_params_from_spec(spec) # tensor → scalar
119
+ build_and_write_map(p_v1, output_dir="run/") # v1 CLI then takes over
120
+ ```
121
+
122
+ ## Design choices
123
+
124
+ - **Implicit gradient strategy**: hard-binning forward keeps bit-parity
125
+ with v1; gradient flows through a parallel soft-binning kernel
126
+ (linear interpolation in R and η). Bin assignments are not
127
+ themselves differentiated — the upstream `(R, η) = pixel_to_REta(...)`
128
+ is, which is the slope you want for refinement.
129
+ - **Map cache**: hashes the same fields as v1's `compute_param_hash` so
130
+ v1 and v2 share `Map.bin` caches and never diverge.
131
+ - **`nn.Module` corrections**: `δr_k`, the Stage-4 RBF spline, and the
132
+ polarization/solid-angle factors are all torch modules with
133
+ `requires_grad`-controllable parameters. Mix and match as the
134
+ optimisation problem demands.
135
+ - **`IntegrationSpec`** uses v2 distortion names (`iso_R2, a1, phi1, …`);
136
+ the `from_v1`/`to_v1` adapters round-trip the chaotic legacy `p0..p14`
137
+ naming losslessly.
138
+
139
+ ## What's in v0.1.0 (the first release)
140
+
141
+ Tested end-to-end against the v1 production pipeline on real Pilatus +
142
+ Varex Aero CeO₂ data; **242 tests + 11 student notebooks**, all green.
143
+
144
+ ### Math correctness
145
+
146
+ - **Exact polygon-area pixel-bin overlap kernel** (Green's theorem on
147
+ circular-arc + radial-segment intersections). No subpixel
148
+ approximation, no smooth-kernel blur — the differentiator vs pyFAI /
149
+ dxchange / DPDAK / nika.
150
+ - **Exact tilt-aware solid-angle correction** (`Lsd² · (n̂·r) / |r|³`).
151
+ Bit-identical to v1 at fp64 on any detector pose.
152
+ - **Exact thin-plate-spline kernel** (`r² log r` with the analytic
153
+ `r=0` limit handled cleanly).
154
+ - **Polarisation correction**: standard `1 − PF · sin²(2θ) · cos²(η − plane)`.
155
+ - **Parallax correction**: `R + parallax · sin(2θ) / px` (matches v1).
156
+ - **Q ↔ R bin edge conversion**: `2θ = 2 arcsin(λ/2d)`, `R = (Lsd/px) tan(2θ)`.
157
+ - **BC ↔ PONI 0.5 px convention** for pyFAI interop pinned in
158
+ `compat.pyfai`; the `make_pyfai_integrator(spec)` helper makes it
159
+ impossible to drop the half-pixel shift.
160
+
161
+ ### Five binning kernels (each clearly labeled)
162
+
163
+ | Kernel | Math | Differentiable in geometry? | Use for |
164
+ |---|---|---|---|
165
+ | `PolygonBinGeometry` | **Exact** polygon-arc-arc | No | Production batch + calibration accuracy |
166
+ | `HardBinGeometry` | Hard floor (one sample per pixel) | No | Max throughput, fixed geometry |
167
+ | `SubpixelBinGeometry` | K×K oversampling of hard | No | Mid-fidelity fast path |
168
+ | `SoftBinGeometry` / `integrate_diff` | Linear-interp soft binning | **Yes** | Refinement / autograd |
169
+ | `MapCache` (wraps v1) | Same as v1 | No | v1-cache interop |
170
+
171
+ ### Differentiable refinement
172
+
173
+ - Autograd through every refinable parameter (Lsd, BC, tilts,
174
+ Parallax, wavelength, all 15 distortion coefficients).
175
+ - All 4 v2 corrections as `nn.Module`s with refinable parameters:
176
+ per-ring δr_k, Stage-4 thin-plate spline, polarisation, solid-angle.
177
+ - 9 loss families: profile MSE / weighted, η-uniformity, peak-position,
178
+ Gaussian prior, multi-image, batched-spec, η-slice, wedge, ring-masked.
179
+
180
+ ### Differentiable bad-pixel mask (`LearnableMask`)
181
+
182
+ The MIDAS differentiator no other azimuthal integrator has. Per-pixel
183
+ inclusion weight is a learnable parameter; train jointly with the
184
+ calibration loss + a sparsity prior, and bad pixels (hot, dead,
185
+ cosmic-ray-prone) get auto-zeroed while good pixels stay at weight ≈ 1.
186
+ Notebook 10 walks through the demo with planted hot pixels.
187
+
188
+ ### Production-deployable pipeline
189
+
190
+ - **Streaming**: `TIFFGlobSource` / `HDF5FrameSource` / `ZarrFrameSource`
191
+ iterators; `FrameNormalizer` (monitor / exposure / transmission);
192
+ `reject_cosmic_rays` (per-pixel temporal sigma-clip);
193
+ `integrate_stream` (out-of-core, memory constant in N-frames).
194
+ - **Variance propagation**: every binning kernel has an
195
+ `integrate_*_with_variance` variant returning `(mean, σ)` per bin.
196
+ Default Poisson; user-supplied variance images supported.
197
+ - **Output writers** with embedded provenance metadata (package
198
+ version, geometry hash, mask fraction, source file names): CSV,
199
+ XYE (Rietveld), FXYE (GSAS), DAT (PDF), 2D-CSV, HDF5.
200
+ - **Per-pixel masks** in every binning kernel — applied at *build time*,
201
+ so masked pixels never enter the integration.
202
+ - **3 CLI scripts**: `midas-integrate-v2` (single frame),
203
+ `midas-integrate-v2-batch` (sweep mode), `midas-integrate-v2-write-map`
204
+ (emit v1-format Map.bin / nMap.bin without numba).
205
+
206
+ ### Pedagogical material (11 notebooks, ~3.5 hrs end-to-end)
207
+
208
+ 01 First Diffraction Pattern → 02 Geometry Intuition → 03 Joint
209
+ Refinement → 04 Multi-Distance Calibration → 05 calibrate-v2 ↔
210
+ integrate-v2 Handoff → 06 Custom Losses → 07 Bayesian UQ → 08 PDF
211
+ Analysis → 09 Production Workflow → 10 Differentiable Mask → 11
212
+ Sweep-mode Batch Processing.
213
+
214
+ Each notebook is self-contained, executes end-to-end, and includes
215
+ "try it yourself" exercises.
216
+
217
+ ### Ecosystem
218
+
219
+ - **pyFAI migration guide** (`docs/MIGRATING_FROM_PYFAI.md`).
220
+ - **Performance benchmark** script (`bench/bench_integrate.py`).
221
+ - **Bootstrap helpers** (`estimate_BC_from_image`,
222
+ `estimate_initial_spec`) for users without a starting paramstest.
223
+ - **Ring auto-detect** (`detect_rings`, `suggest_material`) with built-in
224
+ CeO₂ / LaB₆ / Si / Cr₂O₃ d-spacings (Cr₂O₃ uses JCPDS 38-1479).
225
+
226
+ ## Roadmap
227
+
228
+ - **v0.2** — Polygon kernel GPU port (vectorise the scalar Python
229
+ loops onto torch+CUDA, keeping the math exact). Right answer for
230
+ sub-pixel `RBinSize` builds where the trivial fast path doesn't fire.
231
+ - **v0.3** — Multi-GPU integrate; NeXus-strict HDF5 output;
232
+ integration with the wider HEDM pipeline (peak fitting, indexing).
233
+
234
+ ## License
235
+
236
+ BSD-3-Clause.