torchsx 0.2.2__tar.gz → 0.2.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {torchsx-0.2.2 → torchsx-0.2.3}/PKG-INFO +3 -3
- {torchsx-0.2.2 → torchsx-0.2.3}/README.md +1 -1
- {torchsx-0.2.2 → torchsx-0.2.3}/pyproject.toml +5 -5
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_amortized_training.py +1 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_fold_transformer.py +74 -1
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_large_batch_diagnostics.py +0 -2
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/__init__.py +1 -1
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/train.py +4 -1
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scale/amortized.py +10 -3
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/fold_transformer.py +73 -11
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/PKG-INFO +3 -3
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/requires.txt +1 -1
- {torchsx-0.2.2 → torchsx-0.2.3}/LICENSE +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/setup.cfg +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_amortized_scale.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_asym_scatter.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_average_last.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_bg_mean_offset.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_cli_pipeline.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_cli_production.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_core.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_duckdb_load.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_ewald_offset.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_filter.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_geometry.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_geometry_reciprocal.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_grid_tooling.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_hkl.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_io.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_lazy_imports.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_levels.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_measurement_sources.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_merge_stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_merging_layers.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_metadata.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_native_index.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_native_parse.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_observation_filter.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_peer_features.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_phase_classes.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_pooling.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_posterior_nodes.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_posterior_prior.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_posterior_smooth_sigma.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_prepared_store.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_pseudo_translation.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_reciprocal_basis.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_scatter_mixture.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_split_gaussian_mean.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_split_log_grid.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_split_mixture.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_stateless_merge.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_stream.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_structured_prior.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_tether.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_train_schedule.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_wilson_surface.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_writeback.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/tests/test_writeback_native.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/_categorical.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/_core.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/_pipeline.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/merge_nn.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/merge_stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/cli/stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/geometry.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_crystfel.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_duckdb.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_metadata.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_native.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_parser.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_safetensors.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/_writeback.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/sources.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/io/times.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/levels.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/measurement.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merge_stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/config.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/hkl/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/hkl/filter.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/hkl/hkl.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/hkl/noise.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/hkl/stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/likelihood/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/likelihood/base.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/likelihood/detector_noise.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/likelihood/poisson.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/observations.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/prepare_store.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/priors/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/priors/modulation.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/priors/reciprocal_basis.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/_posterior_nodes_triton.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/_split_log_grid_triton.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/_split_mixture_triton.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/base.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/quadrature/split_log_grid.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scale/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scale/pseudo_translation.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scatter/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scatter/base.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scatter/gengauss.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scatter/mixture.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/scatter/predicted.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/store.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/tether/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/tether/batch.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/tether/kmean.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/tether/terms.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/merging/tether/tether.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/__init__.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/heads.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/merge_target.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/metrics.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/objective.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/obs_features.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/pooling.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/posterior_target.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/nn/train_amortized.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/stats.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx/stream.py +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/SOURCES.txt +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/dependency_links.txt +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/entry_points.txt +0 -0
- {torchsx-0.2.2 → torchsx-0.2.3}/torchsx.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: torchsx
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: A PyTorch-based scaling and merging library for serial crystallography
|
|
5
5
|
Author: Hans Peter Seidel
|
|
6
6
|
License-Expression: MIT
|
|
@@ -13,7 +13,7 @@ Requires-Python: >=3.10
|
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
15
|
Requires-Dist: torchref<0.8,>=0.7.0
|
|
16
|
-
Requires-Dist: torch
|
|
16
|
+
Requires-Dist: torch<2.15,>=2.7
|
|
17
17
|
Requires-Dist: numpy>=2
|
|
18
18
|
Requires-Dist: safetensors>=0.4
|
|
19
19
|
Requires-Dist: matplotlib>=3.7
|
|
@@ -37,7 +37,7 @@ Dynamic: license-file
|
|
|
37
37
|
<div align="center">
|
|
38
38
|
|
|
39
39
|
[](https://www.python.org/downloads/)
|
|
40
|
-
[](https://pytorch.org/)
|
|
41
41
|
[](https://github.com/HatPdotS/TorchRef)
|
|
42
42
|
[](https://developer.nvidia.com/cuda-zone)
|
|
43
43
|
[](https://opensource.org/licenses/MIT)
|
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
<div align="center">
|
|
11
11
|
|
|
12
12
|
[](https://www.python.org/downloads/)
|
|
13
|
-
[](https://pytorch.org/)
|
|
14
14
|
[](https://github.com/HatPdotS/TorchRef)
|
|
15
15
|
[](https://developer.nvidia.com/cuda-zone)
|
|
16
16
|
[](https://opensource.org/licenses/MIT)
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "torchsx"
|
|
7
|
-
version = "0.2.
|
|
7
|
+
version = "0.2.3"
|
|
8
8
|
description = "A PyTorch-based scaling and merging library for serial crystallography"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -21,10 +21,10 @@ dependencies = [
|
|
|
21
21
|
# Brings torch, triton and gemmi. TorchRef's minor releases change its API,
|
|
22
22
|
# so the bound follows the minor version torchsx is tested against.
|
|
23
23
|
"torchref>=0.7.0,<0.8",
|
|
24
|
-
# The
|
|
25
|
-
# on
|
|
26
|
-
#
|
|
27
|
-
"torch
|
|
24
|
+
# The fold encoder compiles with flex attention and dynamic shapes from 2.7
|
|
25
|
+
# on (compile_options sets inductor per version); where torch cannot compile
|
|
26
|
+
# it, it runs uncompiled.
|
|
27
|
+
"torch>=2.7,<2.15",
|
|
28
28
|
"numpy>=2", # the Python writer's float32 arithmetic (NEP 50)
|
|
29
29
|
"safetensors>=0.4", # the .sfx working format
|
|
30
30
|
"matplotlib>=3.7", # the figures of torchsx.observation-stats and torchsx.merge-stats
|
|
@@ -454,6 +454,7 @@ def test_deferred_encoder_backward_is_the_one_pass_gradient(tmp_path):
|
|
|
454
454
|
torch.testing.assert_close(grads[1][n], grads[0][n], rtol=1e-5, atol=1e-7, msg=n)
|
|
455
455
|
|
|
456
456
|
|
|
457
|
+
@pytest.mark.compile
|
|
457
458
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled fold path runs on CUDA only")
|
|
458
459
|
def test_fold_predictions_without_autograd_run_flex_and_match_the_masked_kernel():
|
|
459
460
|
"""Without autograd the fold path runs each block under a checkpoint with autograd
|
|
@@ -7,6 +7,7 @@ import pytest
|
|
|
7
7
|
import torch
|
|
8
8
|
|
|
9
9
|
from torchsx.merging.scale import AmortizedScale
|
|
10
|
+
from torchsx.nn import fold_transformer
|
|
10
11
|
from torchsx.nn.fold_transformer import FoldTransformerEncoder, fold_layout, key_prefix_mask
|
|
11
12
|
from torchsx.nn.pooling import fold_ids
|
|
12
13
|
|
|
@@ -165,6 +166,7 @@ def test_packing_is_exact_in_value_and_gradient():
|
|
|
165
166
|
rtol=1e-5, atol=1e-6)
|
|
166
167
|
|
|
167
168
|
|
|
169
|
+
@pytest.mark.compile
|
|
168
170
|
def test_packed_block_compiles_once_for_every_length():
|
|
169
171
|
"""Blocks of different B and L run one graph: no size is specialised."""
|
|
170
172
|
import torch._dynamo
|
|
@@ -216,6 +218,7 @@ def test_key_prefix_mask_is_create_block_mask(q_len, kv_len):
|
|
|
216
218
|
assert sorted(a_idx[row][:k].tolist()) == sorted(b_idx[row][:k].tolist()), (idx, row)
|
|
217
219
|
|
|
218
220
|
|
|
221
|
+
@pytest.mark.compile
|
|
219
222
|
def test_flex_attention_on_the_prefix_mask_is_masked_attention():
|
|
220
223
|
from torch.nn.attention.flex_attention import flex_attention
|
|
221
224
|
g = torch.Generator().manual_seed(0)
|
|
@@ -233,6 +236,7 @@ def test_flex_needs_heads_of_at_least_16():
|
|
|
233
236
|
assert not FoldTransformerEncoder(4, 7, [4, 5, 6], n_heads=4, **kw).flex
|
|
234
237
|
|
|
235
238
|
|
|
239
|
+
@pytest.mark.compile
|
|
236
240
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled stack runs on CUDA only")
|
|
237
241
|
@pytest.mark.parametrize("flex", [False, True])
|
|
238
242
|
def test_compiled_bf16_is_close_and_keeps_the_exclusion(flex):
|
|
@@ -262,6 +266,7 @@ def test_compiled_bf16_is_close_and_keeps_the_exclusion(flex):
|
|
|
262
266
|
z.square().sum().backward()
|
|
263
267
|
outs.append(z.detach())
|
|
264
268
|
grads.append(torch.cat([p.grad.flatten() for p in enc.parameters() if p.grad is not None]))
|
|
269
|
+
assert fast.compile, "the dry run fell back to the uncompiled network"
|
|
265
270
|
assert (outs[1] - outs[0]).norm() / outs[0].norm() < 2e-2
|
|
266
271
|
assert (grads[1] - grads[0]).norm() / grads[0].norm() < 5e-2
|
|
267
272
|
fast.eval()
|
|
@@ -280,6 +285,7 @@ def test_compiled_bf16_is_close_and_keeps_the_exclusion(flex):
|
|
|
280
285
|
assert (moved[same & ~own] - base[same & ~own]).abs().max() > 1e-2
|
|
281
286
|
|
|
282
287
|
|
|
288
|
+
@pytest.mark.compile
|
|
283
289
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled stack runs on CUDA only")
|
|
284
290
|
@pytest.mark.parametrize("q_len", [40, 300])
|
|
285
291
|
def test_compiled_flex_on_the_prefix_mask_is_masked_attention(q_len):
|
|
@@ -307,6 +313,7 @@ def test_compiled_flex_on_the_prefix_mask_is_masked_attention(q_len):
|
|
|
307
313
|
torch.testing.assert_close(got, ref, rtol=1e-4, atol=1e-4)
|
|
308
314
|
|
|
309
315
|
|
|
316
|
+
@pytest.mark.compile
|
|
310
317
|
@pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled stack runs on CUDA only")
|
|
311
318
|
def test_flex_block_is_one_graph_for_every_block_count():
|
|
312
319
|
"""No graph break, and no size specialised by the number of query or key blocks, one
|
|
@@ -320,7 +327,8 @@ def test_flex_block_is_one_graph_for_every_block_count():
|
|
|
320
327
|
for prm in enc.parameters():
|
|
321
328
|
if prm.dim() > 1:
|
|
322
329
|
torch.nn.init.normal_(prm, std=0.1)
|
|
323
|
-
fn = torch.compile(enc.network, dynamic=True, fullgraph=True
|
|
330
|
+
fn = torch.compile(enc.network, dynamic=True, fullgraph=True,
|
|
331
|
+
options=fold_transformer.compile_options())
|
|
324
332
|
# the runs' queries one block (128) or several (144 to 264), their keys 6 to 11 blocks
|
|
325
333
|
sizes = ([650, 700], [636, 100, 90], [900, 820, 777], [40, 1000, 1001], [1300, 1250], [1100])
|
|
326
334
|
for i, s in enumerate(sizes):
|
|
@@ -486,3 +494,68 @@ def test_refine_needs_context_and_geometry():
|
|
|
486
494
|
with pytest.raises(ValueError, match="ctx_layers >= 1"):
|
|
487
495
|
FoldTransformerEncoder(6, 9, [6, 7, 8], d_model=16, n_heads=4, ctx_layers=0, refine_geometry=True,
|
|
488
496
|
geometry={"cols": list(range(6)), "mean": GEO_MEAN, "std": GEO_STD})
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
@pytest.mark.parametrize("version, off", [("2.10.0+cu126", False), ("2.11.0+cu126", True),
|
|
500
|
+
("2.12.1", True), ("2.13.0+cu130", True),
|
|
501
|
+
("2.14.1+cu126", False)])
|
|
502
|
+
def test_compile_options_switch_off_the_failing_passes_on_their_versions(monkeypatch, version, off):
|
|
503
|
+
monkeypatch.setattr(torch, "__version__", torch.torch_version.TorchVersion(version))
|
|
504
|
+
expected = ({"triton.coalesce_tiling_analysis": False, "triton.mix_order_reduction": False}
|
|
505
|
+
if off else {})
|
|
506
|
+
assert fold_transformer.compile_options() == expected
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
class _FailsBackward(torch.autograd.Function):
|
|
510
|
+
@staticmethod
|
|
511
|
+
def forward(ctx, z):
|
|
512
|
+
return z.clone()
|
|
513
|
+
|
|
514
|
+
@staticmethod
|
|
515
|
+
def backward(ctx, g):
|
|
516
|
+
raise RuntimeError("backward compile failed")
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
@pytest.mark.skipif(not torch.cuda.is_available(), reason="the compiled stack runs on CUDA only")
|
|
520
|
+
@pytest.mark.parametrize("fails", ["forward", "backward"])
|
|
521
|
+
def test_a_network_torch_cannot_compile_runs_uncompiled(monkeypatch, fails):
|
|
522
|
+
"""The dry run catches a compile failure in the forward or the backward pass before
|
|
523
|
+
the first block; the network then runs uncompiled, equal to an encoder built
|
|
524
|
+
without compile, and the dry run leaves no gradient behind."""
|
|
525
|
+
def broken_compile(fn, **kw):
|
|
526
|
+
def run(*args):
|
|
527
|
+
if fails == "forward":
|
|
528
|
+
raise RuntimeError("forward compile failed")
|
|
529
|
+
z, corr = fn(*args)
|
|
530
|
+
return _FailsBackward.apply(z), corr
|
|
531
|
+
return run
|
|
532
|
+
monkeypatch.setattr(torch, "compile", broken_compile)
|
|
533
|
+
torch.manual_seed(0)
|
|
534
|
+
kw = dict(d_model=32, n_heads=4, folds=3, ctx_layers=1, query_layers=1, bf16=False)
|
|
535
|
+
ref = FoldTransformerEncoder(6, 9, [6, 7, 8], compile=False, **kw).cuda()
|
|
536
|
+
for prm in ref.parameters():
|
|
537
|
+
if prm.dim() > 1:
|
|
538
|
+
torch.nn.init.normal_(prm, std=0.15)
|
|
539
|
+
enc = FoldTransformerEncoder(6, 9, [6, 7, 8], compile=True, **kw).cuda()
|
|
540
|
+
enc.load_state_dict(ref.state_dict())
|
|
541
|
+
sizes = torch.tensor([3, 11, 40, 7])
|
|
542
|
+
gid = torch.repeat_interleave(torch.arange(sizes.numel()), sizes).cuda()
|
|
543
|
+
xs, xc = torch.randn(gid.numel(), 6, device="cuda"), torch.randn(gid.numel(), 9, device="cuda")
|
|
544
|
+
grads = []
|
|
545
|
+
for e in (ref, enc):
|
|
546
|
+
e.train()
|
|
547
|
+
if e is enc:
|
|
548
|
+
with pytest.warns(RuntimeWarning, match="cannot compile the fold encoder"):
|
|
549
|
+
e.dry_run(xs, xc, None, gid, *e.layout(gid)[0], autograd=True, backward=True)
|
|
550
|
+
assert not e.compile
|
|
551
|
+
assert all(p.grad is None for p in e.parameters())
|
|
552
|
+
z = e(xs, xc, gid, sizes.numel())
|
|
553
|
+
z.square().sum().backward()
|
|
554
|
+
grads.append([p.grad for p in e.parameters()])
|
|
555
|
+
if e is ref:
|
|
556
|
+
z_ref = z.detach()
|
|
557
|
+
assert torch.equal(z.detach(), z_ref)
|
|
558
|
+
for g_enc, g_ref in zip(*reversed(grads)):
|
|
559
|
+
assert (g_enc is None) == (g_ref is None)
|
|
560
|
+
if g_ref is not None:
|
|
561
|
+
assert torch.equal(g_enc, g_ref)
|
|
@@ -16,8 +16,6 @@ class LargeBatchDiagnosticTests(unittest.TestCase):
|
|
|
16
16
|
def check_large_batch(self, device):
|
|
17
17
|
n = 24_456_800
|
|
18
18
|
x = torch.arange(n // 2, device=device, dtype=torch.float32).repeat_interleave(2).flip(0)
|
|
19
|
-
with self.assertRaisesRegex(RuntimeError, 'too large'):
|
|
20
|
-
x.quantile(.01)
|
|
21
19
|
for q in [0., .01, .5, .99, 1.]:
|
|
22
20
|
rank = q * (n - 1)
|
|
23
21
|
lo, hi = math.floor(rank), math.ceil(rank)
|
|
@@ -369,8 +369,11 @@ def main(argv=None):
|
|
|
369
369
|
updates = run_updates(tr, args.steps, eval_every=args.eval_every, probe=probe)
|
|
370
370
|
probe.end()
|
|
371
371
|
summary_path = args.out / "training_summary.json"
|
|
372
|
+
enc = getattr(tr.scale, "encoder", None)
|
|
372
373
|
summary = dict(steps=args.steps, updates_per_state=updates, best_step=tr.best_step,
|
|
373
|
-
elapsed_seconds=time.time()-t0, resources=probe.summary()
|
|
374
|
+
elapsed_seconds=time.time()-t0, resources=probe.summary(),
|
|
375
|
+
torch=torch.__version__,
|
|
376
|
+
compiled=bool(getattr(enc, "compile", False)) and str(tr.device).startswith("cuda"))
|
|
374
377
|
summary_path.write_text(json.dumps(summary, indent=2) + "\n")
|
|
375
378
|
if args.no_merge:
|
|
376
379
|
print(probe.report(), flush=True)
|
|
@@ -164,7 +164,9 @@ class AmortizedScale(nn.Module):
|
|
|
164
164
|
The encoder packs the batch's crystals into blocks once
|
|
165
165
|
(:meth:`~torchsx.nn.fold_transformer.FoldTransformerEncoder.layout`).
|
|
166
166
|
Each block gathers its rows, runs the network (compiled on CUDA,
|
|
167
|
-
``encoder.compile
|
|
167
|
+
``encoder.compile``, after a dry run on the first block,
|
|
168
|
+
:meth:`~torchsx.nn.fold_transformer.FoldTransformerEncoder.dry_run`)
|
|
169
|
+
and the heads, in training under one checkpoint,
|
|
168
170
|
so only the batch tables and the block's outputs are stored. One
|
|
169
171
|
gather puts the outputs back in row order.
|
|
170
172
|
|
|
@@ -187,11 +189,16 @@ class AmortizedScale(nn.Module):
|
|
|
187
189
|
run = self._fold_block
|
|
188
190
|
ckpt = torch.is_grad_enabled() and self.training
|
|
189
191
|
defer = ckpt and self.defer_encoder_backward
|
|
190
|
-
|
|
192
|
+
flex_eval = (not torch.is_grad_enabled() and x_self.is_cuda and enc.flex
|
|
191
193
|
and enc.flex_device and any(p.requires_grad for p in enc.parameters()))
|
|
194
|
+
layout = enc.layout(meta.crystal_id)
|
|
195
|
+
if layout:
|
|
196
|
+
enc.dry_run(x_self, x_ctx, x_crys, meta.crystal_id, *layout[0],
|
|
197
|
+
autograd=torch.is_grad_enabled() or flex_eval, backward=ckpt)
|
|
198
|
+
flex_only = flex_eval and enc.compile
|
|
192
199
|
n = x_self.shape[0]
|
|
193
200
|
parts, dest, corrections, blocks = [], [], [], []
|
|
194
|
-
for idx, valid in
|
|
201
|
+
for idx, valid in layout:
|
|
195
202
|
args = (x_self, x_ctx, x_crys, meta.crystal_id, idx, valid)
|
|
196
203
|
if flex_only:
|
|
197
204
|
with torch.enable_grad():
|
|
@@ -53,6 +53,7 @@ embedding.
|
|
|
53
53
|
from __future__ import annotations
|
|
54
54
|
|
|
55
55
|
import os
|
|
56
|
+
import warnings
|
|
56
57
|
|
|
57
58
|
import torch
|
|
58
59
|
import torch.nn as nn
|
|
@@ -70,6 +71,20 @@ FLEX_KERNEL_OPTIONS: dict | None = dict(bwd_BLOCK_M1=64, bwd_BLOCK_N1=128, bwd_B
|
|
|
70
71
|
bwd_BLOCK_N2=64, bwd_num_warps=4, bwd_num_stages=2)
|
|
71
72
|
|
|
72
73
|
|
|
74
|
+
def compile_options() -> dict:
|
|
75
|
+
"""Inductor settings for the compiled network on the installed torch.
|
|
76
|
+
|
|
77
|
+
Inductor of torch 2.11 to 2.13 fails on this network in two of its
|
|
78
|
+
passes: the memory-coalescing analysis of tiled kernels (2.11, forward)
|
|
79
|
+
and mix-order reduction (2.12 and 2.13, backward). On those versions
|
|
80
|
+
both passes are off; every other version keeps inductor's defaults.
|
|
81
|
+
"""
|
|
82
|
+
version = torch.torch_version.TorchVersion(torch.__version__)
|
|
83
|
+
if "2.11" <= version < "2.14":
|
|
84
|
+
return {"triton.coalesce_tiling_analysis": False, "triton.mix_order_reduction": False}
|
|
85
|
+
return {}
|
|
86
|
+
|
|
87
|
+
|
|
73
88
|
def fold_layout(group_id: Tensor, folds: int, tokens: int = 1 << 18,
|
|
74
89
|
rows: int = 2) -> list[tuple[Tensor, Tensor]]:
|
|
75
90
|
"""Pack each crystal's rows into padded blocks of ``B`` rows of ``L`` slots.
|
|
@@ -310,11 +325,13 @@ class FoldTransformerEncoder(nn.Module):
|
|
|
310
325
|
compile, bf16
|
|
311
326
|
Run the network on each packed block (:meth:`network`; the rows are
|
|
312
327
|
gathered eagerly, :meth:`gather`) through
|
|
313
|
-
``torch.compile(dynamic=True)`` on CUDA
|
|
314
|
-
half precision on CUDA: bfloat16, or
|
|
315
|
-
capability below 8.0, which have no bfloat16
|
|
316
|
-
|
|
317
|
-
(
|
|
328
|
+
``torch.compile(dynamic=True)`` on CUDA (with :func:`compile_options`),
|
|
329
|
+
and the attention itself in half precision on CUDA: bfloat16, or
|
|
330
|
+
float16 on GPUs of compute capability below 8.0, which have no bfloat16
|
|
331
|
+
(:meth:`fit_device`). Before its first block the network is compiled
|
|
332
|
+
on that block (:meth:`dry_run`); when torch cannot compile it, it runs
|
|
333
|
+
uncompiled. ``None`` reads ``TORCHSX_PEER_COMPILE`` /
|
|
334
|
+
``TORCHSX_PEER_BF16`` (default on).
|
|
318
335
|
flex
|
|
319
336
|
Inside the compiled block, with autograd on, attention reads each
|
|
320
337
|
run's keys through flex attention, which skips the padding instead of
|
|
@@ -396,6 +413,8 @@ class FoldTransformerEncoder(nn.Module):
|
|
|
396
413
|
self.flex_device = True
|
|
397
414
|
self._device = None
|
|
398
415
|
self._compiled = None
|
|
416
|
+
#: the (autograd, backward) modes :meth:`dry_run` has compiled
|
|
417
|
+
self._dry_runs: set[tuple[bool, bool]] = set()
|
|
399
418
|
self.ctx_blocks = nn.ModuleList(
|
|
400
419
|
AttentionBlock(d_model, n_heads, d_hidden, self_attention=True,
|
|
401
420
|
activation=activation, bf16=bf16)
|
|
@@ -448,8 +467,13 @@ class FoldTransformerEncoder(nn.Module):
|
|
|
448
467
|
f"{self.n_crystal_features}; x_crys does not match")
|
|
449
468
|
self.fit_device(x_self.device)
|
|
450
469
|
use_ckpt = self.checkpoint and self.training and torch.is_grad_enabled()
|
|
470
|
+
blocks = self.layout(group_id)
|
|
471
|
+
if blocks:
|
|
472
|
+
self.dry_run(x_self, x_ctx, x_crys, group_id, *blocks[0],
|
|
473
|
+
autograd=torch.is_grad_enabled(),
|
|
474
|
+
backward=self.training and torch.is_grad_enabled())
|
|
451
475
|
zs, dest, corrections = [], [], []
|
|
452
|
-
for idx, valid in
|
|
476
|
+
for idx, valid in blocks:
|
|
453
477
|
args = (x_self, x_ctx, x_crys, group_id, idx, valid)
|
|
454
478
|
zb, corr = (torch.utils.checkpoint.checkpoint(self.encode, *args, use_reentrant=False)
|
|
455
479
|
if use_ckpt else self.encode(*args))
|
|
@@ -485,13 +509,51 @@ class FoldTransformerEncoder(nn.Module):
|
|
|
485
509
|
compiled on CUDA. ``(B, L, D)`` embeddings of its slots and each run's
|
|
486
510
|
geometry correction."""
|
|
487
511
|
args = self.gather(x_self, x_ctx, x_crys, group_id, idx, valid)
|
|
488
|
-
run = self.network
|
|
489
|
-
if self.compile and x_self.is_cuda:
|
|
490
|
-
if self._compiled is None:
|
|
491
|
-
self._compiled = torch.compile(self.network, dynamic=True)
|
|
492
|
-
run = self._compiled
|
|
512
|
+
run = self._compiled_network() if self.compile and x_self.is_cuda else self.network
|
|
493
513
|
return run(*args)
|
|
494
514
|
|
|
515
|
+
def _compiled_network(self):
|
|
516
|
+
if self._compiled is None:
|
|
517
|
+
self._compiled = torch.compile(self.network, dynamic=True, options=compile_options())
|
|
518
|
+
return self._compiled
|
|
519
|
+
|
|
520
|
+
def dry_run(self, x_self: Tensor, x_ctx: Tensor, x_crys: Tensor | None, group_id: Tensor,
|
|
521
|
+
idx: Tensor, valid: Tensor, *, autograd: bool, backward: bool) -> None:
|
|
522
|
+
"""Compile the network on one packed block before the run uses it.
|
|
523
|
+
|
|
524
|
+
With ``compile`` on CUDA, the first call per mode runs the compiled
|
|
525
|
+
network on the block (:func:`fold_layout`) as the run will: the
|
|
526
|
+
forward pass, with autograd when ``autograd`` (plain and under a
|
|
527
|
+
checkpoint), and with ``backward`` its backward pass. The outputs are
|
|
528
|
+
discarded and no gradient is accumulated. When torch fails on any of
|
|
529
|
+
them, the network runs uncompiled from then on (``compile`` off) and a
|
|
530
|
+
warning names the error. Callers run it before their first block,
|
|
531
|
+
outside any checkpoint.
|
|
532
|
+
"""
|
|
533
|
+
mode = (bool(autograd), bool(backward))
|
|
534
|
+
if not (self.compile and x_self.is_cuda) or mode in self._dry_runs:
|
|
535
|
+
return
|
|
536
|
+
self._dry_runs.add(mode)
|
|
537
|
+
args = self.gather(x_self, x_ctx, x_crys, group_id, idx, valid)
|
|
538
|
+
params = [p for p in self.parameters() if p.requires_grad]
|
|
539
|
+
try:
|
|
540
|
+
net = self._compiled_network()
|
|
541
|
+
with torch.set_grad_enabled(autograd):
|
|
542
|
+
z, _ = net(*args)
|
|
543
|
+
if backward and params:
|
|
544
|
+
torch.autograd.grad(z, params, torch.ones_like(z), allow_unused=True)
|
|
545
|
+
if autograd:
|
|
546
|
+
torch.utils.checkpoint.checkpoint(net, *args, use_reentrant=False)
|
|
547
|
+
except torch.cuda.OutOfMemoryError:
|
|
548
|
+
raise
|
|
549
|
+
except Exception as e:
|
|
550
|
+
self.compile, self._compiled = False, None
|
|
551
|
+
first = next((line for line in str(e).splitlines() if line.strip()), "")
|
|
552
|
+
warnings.warn(f"torch {torch.__version__} cannot compile the fold encoder "
|
|
553
|
+
f"({type(e).__name__}: {first.strip()}); it runs uncompiled, which "
|
|
554
|
+
f"is slower. TORCHSX_PEER_COMPILE=0 skips the attempt.",
|
|
555
|
+
RuntimeWarning, stacklevel=2)
|
|
556
|
+
|
|
495
557
|
def gather(self, x_self: Tensor, x_ctx: Tensor, x_crys: Tensor | None, group_id: Tensor,
|
|
496
558
|
idx: Tensor, valid: Tensor) -> tuple[Tensor, Tensor, Tensor | None, Tensor]:
|
|
497
559
|
"""The network's inputs for one packed block, read from the batch tables: the
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: torchsx
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.3
|
|
4
4
|
Summary: A PyTorch-based scaling and merging library for serial crystallography
|
|
5
5
|
Author: Hans Peter Seidel
|
|
6
6
|
License-Expression: MIT
|
|
@@ -13,7 +13,7 @@ Requires-Python: >=3.10
|
|
|
13
13
|
Description-Content-Type: text/markdown
|
|
14
14
|
License-File: LICENSE
|
|
15
15
|
Requires-Dist: torchref<0.8,>=0.7.0
|
|
16
|
-
Requires-Dist: torch
|
|
16
|
+
Requires-Dist: torch<2.15,>=2.7
|
|
17
17
|
Requires-Dist: numpy>=2
|
|
18
18
|
Requires-Dist: safetensors>=0.4
|
|
19
19
|
Requires-Dist: matplotlib>=3.7
|
|
@@ -37,7 +37,7 @@ Dynamic: license-file
|
|
|
37
37
|
<div align="center">
|
|
38
38
|
|
|
39
39
|
[](https://www.python.org/downloads/)
|
|
40
|
-
[](https://pytorch.org/)
|
|
41
41
|
[](https://github.com/HatPdotS/TorchRef)
|
|
42
42
|
[](https://developer.nvidia.com/cuda-zone)
|
|
43
43
|
[](https://opensource.org/licenses/MIT)
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|