jaxdecomp 0.2.2__tar.gz → 0.2.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.pre-commit-config.yaml +7 -11
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/PKG-INFO +3 -3
- jaxdecomp-0.2.4/examples/lpt_nbody_demo.py +292 -0
- jaxdecomp-0.2.4/examples/scatter.py +163 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/examples/visualizer.ipynb +31 -19
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/pyproject.toml +6 -3
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/scripts/autotune.py +7 -6
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/scripts/test_fft3d.py +17 -19
- jaxdecomp-0.2.4/src/jaxdecomp/__init__.py +111 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/__init__.py +68 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/cudecomp/fft.py +465 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/cudecomp/halo.py +411 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/cudecomp/transpose.py +563 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/fft_utils.py +183 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/jax/fft.py +563 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/jax/fftfreq.py +101 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/jax/halo.py +466 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/jax/transpose.py +417 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/pencil_utils.py +309 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/sharded_array.py +291 -0
- jaxdecomp-0.2.4/src/jaxdecomp/_src/spmd_ops.py +296 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/fft.py +82 -64
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/halo.py +14 -13
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/transpose.py +110 -110
- jaxdecomp-0.2.4/tests/conftest.py +198 -0
- jaxdecomp-0.2.4/tests/test_fft.py +355 -0
- jaxdecomp-0.2.4/tests/test_halo.py +356 -0
- jaxdecomp-0.2.4/tests/test_transpose.py +254 -0
- jaxdecomp-0.2.2/examples/lpt_nbody_demo.py +0 -280
- jaxdecomp-0.2.2/examples/scatter.py +0 -158
- jaxdecomp-0.2.2/src/jaxdecomp/__init__.py +0 -73
- jaxdecomp-0.2.2/src/jaxdecomp/_src/__init__.py +0 -39
- jaxdecomp-0.2.2/src/jaxdecomp/_src/cudecomp/fft.py +0 -424
- jaxdecomp-0.2.2/src/jaxdecomp/_src/cudecomp/halo.py +0 -379
- jaxdecomp-0.2.2/src/jaxdecomp/_src/cudecomp/transpose.py +0 -519
- jaxdecomp-0.2.2/src/jaxdecomp/_src/fft_utils.py +0 -183
- jaxdecomp-0.2.2/src/jaxdecomp/_src/jax/fft.py +0 -521
- jaxdecomp-0.2.2/src/jaxdecomp/_src/jax/fftfreq.py +0 -91
- jaxdecomp-0.2.2/src/jaxdecomp/_src/jax/halo.py +0 -435
- jaxdecomp-0.2.2/src/jaxdecomp/_src/jax/transpose.py +0 -368
- jaxdecomp-0.2.2/src/jaxdecomp/_src/pencil_utils.py +0 -299
- jaxdecomp-0.2.2/src/jaxdecomp/_src/spmd_ops.py +0 -264
- jaxdecomp-0.2.2/tests/conftest.py +0 -164
- jaxdecomp-0.2.2/tests/test_fft.py +0 -312
- jaxdecomp-0.2.2/tests/test_halo.py +0 -244
- jaxdecomp-0.2.2/tests/test_transpose.py +0 -222
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.clang-format +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.github/workflows/formatting.yml +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.github/workflows/github-deploy.yml +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.github/workflows/joss-paper-pdf.yml +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.github/workflows/tests.yml +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.gitignore +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.gitmodules +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/.style.yapf +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/CHANGELOG.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/CMakeLists.txt +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/CONTRIBUTING.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/LICENSE +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/README.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/examples/README.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/examples/submit_rusty.sbatch +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/joss-paper/assets/benchmark.png +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/joss-paper/assets/fft.svg +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/joss-paper/assets/halo-exchange.svg +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/joss-paper/paper.bib +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/joss-paper/paper.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/slurms/README.md +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/slurms/template.slurm +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/fft.cu +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/grid_descriptor_mgr.cc +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/halo.cu +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/checks.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/fft.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/grid_descriptor_mgr.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/halo.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/helpers.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/jaxdecomp.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/logger.hpp +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/perfostep.hpp +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/include/transpose.h +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/jaxdecomp.cc +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/csrc/transpose.cu +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/_src/cudecomp/__init__.py +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/_src/jax/__init__.py +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/src/jaxdecomp/typing.py +0 -0
- {jaxdecomp-0.2.2 → jaxdecomp-0.2.4}/tests/run_all_tests.sh +0 -0
|
@@ -1,20 +1,16 @@
|
|
|
1
1
|
repos:
|
|
2
2
|
- repo: https://github.com/pre-commit/pre-commit-hooks
|
|
3
|
-
rev:
|
|
3
|
+
rev: 'v5.0.0'
|
|
4
4
|
hooks:
|
|
5
5
|
- id: check-yaml
|
|
6
6
|
- id: end-of-file-fixer
|
|
7
7
|
- id: trailing-whitespace
|
|
8
|
-
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
- id:
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
rev: 5.13.2
|
|
15
|
-
hooks:
|
|
16
|
-
- id: isort
|
|
17
|
-
name: isort (python)
|
|
8
|
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
|
9
|
+
rev: v0.8.2
|
|
10
|
+
hooks:
|
|
11
|
+
- id: ruff-format
|
|
12
|
+
- id: ruff
|
|
13
|
+
|
|
18
14
|
- repo: https://github.com/pre-commit/mirrors-clang-format
|
|
19
15
|
rev: v18.1.4
|
|
20
16
|
hooks:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: jaxdecomp
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.4
|
|
4
4
|
Summary: JAX bindings for the cuDecomp library
|
|
5
5
|
Author: Wassim Kabalan, Francois Lanusse
|
|
6
6
|
License: MIT License
|
|
@@ -34,9 +34,9 @@ Classifier: Intended Audience :: Developers
|
|
|
34
34
|
Classifier: Intended Audience :: Science/Research
|
|
35
35
|
Project-URL: Homepage, https://github.com/DifferentiableUniverseInitiative/jaxDecomp
|
|
36
36
|
Requires-Dist: jaxtyping>=0.2.0
|
|
37
|
-
Requires-Dist: jax>=0.4.
|
|
37
|
+
Requires-Dist: jax>=0.4.35
|
|
38
|
+
Requires-Dist: typing-extensions; python_version < "3.11"
|
|
38
39
|
Requires-Dist: pytest>=8.0.0; extra == "test"
|
|
39
|
-
Requires-Dist: jax>=0.4.30; extra == "test"
|
|
40
40
|
Provides-Extra: test
|
|
41
41
|
Description-Content-Type: text/markdown
|
|
42
42
|
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
import argparse
|
|
2
|
+
import os
|
|
3
|
+
from functools import partial
|
|
4
|
+
from typing import Any, Callable, Hashable
|
|
5
|
+
|
|
6
|
+
Specs = Any
|
|
7
|
+
AxisName = Hashable
|
|
8
|
+
|
|
9
|
+
import jax
|
|
10
|
+
|
|
11
|
+
jax.config.update("jax_enable_x64", False)
|
|
12
|
+
|
|
13
|
+
import jax.numpy as jnp
|
|
14
|
+
import jax_cosmo as jc
|
|
15
|
+
import numpy as np
|
|
16
|
+
from jax._src import mesh as mesh_lib
|
|
17
|
+
from jax.experimental import mesh_utils
|
|
18
|
+
from jax.experimental.shard_map import shard_map
|
|
19
|
+
from jax.sharding import Mesh
|
|
20
|
+
from jax.sharding import PartitionSpec as P
|
|
21
|
+
from scatter import scatter
|
|
22
|
+
|
|
23
|
+
import jaxdecomp
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def shmap(
|
|
27
|
+
f: Callable,
|
|
28
|
+
in_specs: Specs,
|
|
29
|
+
out_specs: Specs,
|
|
30
|
+
check_rep: bool = True,
|
|
31
|
+
auto: frozenset[AxisName] = frozenset(),
|
|
32
|
+
):
|
|
33
|
+
"""Helper function to create a shard_map function that extracts the mesh from the
|
|
34
|
+
context."""
|
|
35
|
+
mesh = mesh_lib.thread_resources.env.physical_mesh
|
|
36
|
+
return shard_map(f, mesh, in_specs, out_specs, check_rep, auto)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _global_to_local_size(nc: int):
|
|
40
|
+
"""Helper function to get the local size of a mesh given the global size."""
|
|
41
|
+
pdims = mesh_lib.thread_resources.env.physical_mesh.devices.shape
|
|
42
|
+
return [nc // pdims[0], nc // pdims[1], nc]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def fttk(nc: int):
|
|
46
|
+
"""
|
|
47
|
+
Generate Fourier transform wave numbers for a given mesh.
|
|
48
|
+
|
|
49
|
+
Args:
|
|
50
|
+
nc (int): Shape of the mesh grid.
|
|
51
|
+
|
|
52
|
+
Returns:
|
|
53
|
+
list: List of wave number arrays for each dimension in
|
|
54
|
+
the order [kx, ky, kz].
|
|
55
|
+
"""
|
|
56
|
+
kd = np.fft.fftfreq(nc) * 2 * np.pi
|
|
57
|
+
|
|
58
|
+
@partial(
|
|
59
|
+
shmap,
|
|
60
|
+
in_specs=(P("x"), P("y"), P(None)),
|
|
61
|
+
out_specs=(P("x"), P(None, "y"), P(None)),
|
|
62
|
+
)
|
|
63
|
+
def get_kvec(ky, kz, kx):
|
|
64
|
+
return (ky.reshape([-1, 1, 1]),
|
|
65
|
+
kz.reshape([1, -1, 1]),
|
|
66
|
+
kx.reshape([1, 1, -1])) # yapf: disable
|
|
67
|
+
|
|
68
|
+
ky, kz, kx = get_kvec(kd, kd, kd) # The order of the output
|
|
69
|
+
# corresponds to the order of dimensions in the transposed FFT
|
|
70
|
+
# output
|
|
71
|
+
return kx, ky, kz
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def gravity_kernel(kx, ky, kz):
|
|
75
|
+
"""Computes a Fourier kernel combining laplace and derivative
|
|
76
|
+
operators to compute gravitational forces.
|
|
77
|
+
|
|
78
|
+
Args:
|
|
79
|
+
kvec (tuple of float): Wave numbers in Fourier space.
|
|
80
|
+
|
|
81
|
+
Returns:
|
|
82
|
+
tuple of jnp.ndarray: kernels for each dimension.
|
|
83
|
+
"""
|
|
84
|
+
kk = kx**2 + ky**2 + kz**2
|
|
85
|
+
laplace_kernel = jnp.where(kk == 0, 1.0, 1.0 / kk)
|
|
86
|
+
|
|
87
|
+
grav_kernel = (laplace_kernel * 1j * kx,
|
|
88
|
+
laplace_kernel * 1j * ky,
|
|
89
|
+
laplace_kernel * 1j * kz) # yapf: disable
|
|
90
|
+
return grav_kernel
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def gaussian_field_and_forces(key, nc, box_size, power_spectrum):
|
|
94
|
+
"""
|
|
95
|
+
Generate a Gaussian field with a given power spectrum, along with gravitational forces.
|
|
96
|
+
|
|
97
|
+
Args:
|
|
98
|
+
key (int): Key for the random number generator.
|
|
99
|
+
nc (int): Number of cells in the mesh.
|
|
100
|
+
box_size (float): Size of the box.
|
|
101
|
+
power_spectrum (callable): Power spectrum function.
|
|
102
|
+
|
|
103
|
+
Returns:
|
|
104
|
+
tuple of jnp.ndarray: The generated Gaussian field and the gravitational forces.
|
|
105
|
+
"""
|
|
106
|
+
local_mesh_shape = _global_to_local_size(nc)
|
|
107
|
+
|
|
108
|
+
# Create a distributed field drawn from a Gaussian distribution in real space
|
|
109
|
+
delta = shmap(
|
|
110
|
+
partial(jax.random.normal, shape=local_mesh_shape, dtype='float32'),
|
|
111
|
+
in_specs=P(None),
|
|
112
|
+
out_specs=P('x', 'y'))(key) # yapf: disable
|
|
113
|
+
|
|
114
|
+
# Compute the Fourier transform of the field
|
|
115
|
+
delta_k = jaxdecomp.fft.pfft3d(delta.astype(jnp.complex64))
|
|
116
|
+
|
|
117
|
+
# Compute the Fourier wavenumbers of the field
|
|
118
|
+
kx, ky, kz = fttk(nc)
|
|
119
|
+
kk = jnp.sqrt(kx**2 + ky**2 + kz**2) * (nc / box_size)
|
|
120
|
+
|
|
121
|
+
# Apply power spectrum to Fourier modes
|
|
122
|
+
delta_k *= (power_spectrum(kk) * (nc / box_size) ** 3) ** 0.5
|
|
123
|
+
|
|
124
|
+
# Compute inverse Fourier transform to recover the initial conditions in real space
|
|
125
|
+
delta = jaxdecomp.fft.pifft3d(delta_k).real
|
|
126
|
+
|
|
127
|
+
# Compute gravitational forces associated with this field
|
|
128
|
+
grav_kernel = gravity_kernel(kx, ky, kz)
|
|
129
|
+
forces_k = [g * delta_k for g in grav_kernel]
|
|
130
|
+
|
|
131
|
+
# Retrieve the forces in real space by inverse Fourier transforming
|
|
132
|
+
forces = jnp.stack([jaxdecomp.fft.pifft3d(f).real for f in forces_k], axis=-1)
|
|
133
|
+
|
|
134
|
+
return delta, forces
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def cic_paint(displacement, halo_size):
|
|
138
|
+
"""Paints particles on a mesh using Cloud-In-Cell interpolation.
|
|
139
|
+
|
|
140
|
+
Args:
|
|
141
|
+
displacement (jnp.ndarray): Displacement of each particle.
|
|
142
|
+
halo_size (int): Halo size for painting.
|
|
143
|
+
|
|
144
|
+
Returns:
|
|
145
|
+
jnp.ndarray: Density field.
|
|
146
|
+
"""
|
|
147
|
+
local_mesh_shape = _global_to_local_size(displacement.shape[0])
|
|
148
|
+
hs = halo_size
|
|
149
|
+
|
|
150
|
+
@partial(shmap, in_specs=(P("x", "y"),), out_specs=P("x", "y"))
|
|
151
|
+
def cic_op(disp):
|
|
152
|
+
"""CiC operation on each local slice of the mesh."""
|
|
153
|
+
# Create a mesh to paint the particles on for the local slice
|
|
154
|
+
mesh = jnp.zeros(disp.shape[:-1], dtype="float32")
|
|
155
|
+
|
|
156
|
+
# Padding the mesh along the two first dimensions
|
|
157
|
+
mesh = jnp.pad(mesh, [[hs, hs], [hs, hs], [0, 0]])
|
|
158
|
+
|
|
159
|
+
# Compute the position of the particles on a regular grid
|
|
160
|
+
pos_x, pos_y, pos_z = jnp.meshgrid(
|
|
161
|
+
jnp.arange(local_mesh_shape[0]),
|
|
162
|
+
jnp.arange(local_mesh_shape[1]),
|
|
163
|
+
jnp.arange(local_mesh_shape[2]),
|
|
164
|
+
indexing="ij",
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
# adding an offset of size halo size
|
|
168
|
+
pos = jnp.stack([pos_x + hs, pos_y + hs, pos_z], axis=-1)
|
|
169
|
+
|
|
170
|
+
# Apply scatter operation to paint the particles on the local mesh
|
|
171
|
+
field = scatter(pos.reshape([-1, 3]), disp.reshape([-1, 3]), mesh)
|
|
172
|
+
|
|
173
|
+
return field
|
|
174
|
+
|
|
175
|
+
# Performs painting on a padded mesh, with halos on the two first dimensions
|
|
176
|
+
field = cic_op(displacement)
|
|
177
|
+
|
|
178
|
+
# Run halo exchange to get the correct values at the boundaries
|
|
179
|
+
field = jaxdecomp.halo_exchange(
|
|
180
|
+
field, halo_extents=(hs // 2, hs // 2, 0), halo_periods=(True, True, True)
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
@partial(shmap, in_specs=(P("x", "y"),), out_specs=P("x", "y"))
|
|
184
|
+
def unpad(x):
|
|
185
|
+
"""Removes the padding and reduce the halo regions"""
|
|
186
|
+
x = x.at[hs : hs + hs // 2].add(x[: hs // 2])
|
|
187
|
+
x = x.at[-(hs + hs // 2) : -hs].add(x[-hs // 2 :])
|
|
188
|
+
x = x.at[:, hs : hs + hs // 2].add(x[:, : hs // 2])
|
|
189
|
+
x = x.at[:, -(hs + hs // 2) : -hs].add(x[:, -hs // 2 :])
|
|
190
|
+
return x[hs:-hs, hs:-hs, :]
|
|
191
|
+
|
|
192
|
+
# Unpad the output array
|
|
193
|
+
field = unpad(field)
|
|
194
|
+
return field
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
@partial(jax.jit, static_argnames=("nc", "box_size", "halo_size"))
|
|
198
|
+
def simulation_fn(key, nc, box_size, halo_size, a=1.0):
|
|
199
|
+
"""
|
|
200
|
+
Run a simulation to generate initial conditions and density field using LPT.
|
|
201
|
+
|
|
202
|
+
Args:
|
|
203
|
+
key (list of int): Jax random key for the random number generator.
|
|
204
|
+
nc (int): Size of the mesh grid.
|
|
205
|
+
box_size (float): Size of the box.
|
|
206
|
+
halo_size (int): Halo size for painting.
|
|
207
|
+
a (float): Scale factor of final field.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
tuple of jnp.ndarray: Initial conditions and final density field.
|
|
211
|
+
"""
|
|
212
|
+
# Build a default cosmology
|
|
213
|
+
cosmo = jc.Planck15()
|
|
214
|
+
|
|
215
|
+
# Create a small function to generate the linear matter power spectrum at arbitrary k
|
|
216
|
+
k = jnp.logspace(-4, 1, 128)
|
|
217
|
+
pk = jc.power.linear_matter_power(cosmo, k)
|
|
218
|
+
pk_fn = jax.jit(
|
|
219
|
+
lambda x: jc.scipy.interpolate.interp(x.reshape([-1]), k, pk).reshape(x.shape)
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
# Generate a Gaussian field and gravitational forces from a power spectrum
|
|
223
|
+
intial_conditions, initial_forces = gaussian_field_and_forces(
|
|
224
|
+
key=key, nc=nc, box_size=box_size, power_spectrum=pk_fn
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
# Compute the LPT displacement that particles initialy placed on a regular grid
|
|
228
|
+
# would experience at scale factor a, by simple Zeldovich approximation
|
|
229
|
+
initial_displacement = (
|
|
230
|
+
jc.background.growth_factor(cosmo, jnp.atleast_1d(a)) * initial_forces
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
# Paints the displaced particles on a mesh to obtain the density field
|
|
234
|
+
final_field = cic_paint(initial_displacement, halo_size)
|
|
235
|
+
|
|
236
|
+
return intial_conditions, final_field
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def main(args):
|
|
240
|
+
print(f"Running with arguments {args}")
|
|
241
|
+
|
|
242
|
+
# Setting up distributed jax
|
|
243
|
+
jax.distributed.initialize()
|
|
244
|
+
rank = jax.process_index()
|
|
245
|
+
size = jax.process_count()
|
|
246
|
+
|
|
247
|
+
# Setting up distributed random numbers
|
|
248
|
+
master_key = jax.random.PRNGKey(42)
|
|
249
|
+
key = jax.random.split(master_key, size)[rank]
|
|
250
|
+
|
|
251
|
+
# Create computing mesh and sharding information
|
|
252
|
+
pdims = tuple(map(int, args.pdims.split("x")))
|
|
253
|
+
devices = mesh_utils.create_device_mesh(pdims)
|
|
254
|
+
mesh = Mesh(devices.T, axis_names=("x", "y"))
|
|
255
|
+
|
|
256
|
+
# Run the simulation on the compute mesh
|
|
257
|
+
with mesh:
|
|
258
|
+
initial_conds, final_field = simulation_fn(
|
|
259
|
+
key=key, nc=args.nc, box_size=args.box_size, halo_size=args.halo_size
|
|
260
|
+
)
|
|
261
|
+
|
|
262
|
+
# Create output directory to save the results
|
|
263
|
+
output_dir = args.output
|
|
264
|
+
os.makedirs(output_dir, exist_ok=True)
|
|
265
|
+
np.save(
|
|
266
|
+
f"{output_dir}/initial_conditions_{rank}.npy", initial_conds.addressable_data(0)
|
|
267
|
+
)
|
|
268
|
+
np.save(f"{output_dir}/field_{rank}.npy", final_field.addressable_data(0))
|
|
269
|
+
print(f"Finished saved to {output_dir}")
|
|
270
|
+
|
|
271
|
+
# Closing distributed jax
|
|
272
|
+
jax.distributed.shutdown()
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
if __name__ == "__main__":
|
|
276
|
+
parser = argparse.ArgumentParser("Distributed LPT N-body simulation.")
|
|
277
|
+
parser.add_argument(
|
|
278
|
+
"--pdims", type=str, default="1x1", help="Processor grid dimensions"
|
|
279
|
+
)
|
|
280
|
+
parser.add_argument(
|
|
281
|
+
"--nc", type=int, default=256, help="Number of cells in the mesh"
|
|
282
|
+
)
|
|
283
|
+
parser.add_argument(
|
|
284
|
+
"--box_size", type=float, default=512.0, help="Box size in Mpc/h"
|
|
285
|
+
)
|
|
286
|
+
parser.add_argument(
|
|
287
|
+
"--halo_size", type=int, default=32, help="Halo size for painting"
|
|
288
|
+
)
|
|
289
|
+
parser.add_argument("--output", type=str, default="out")
|
|
290
|
+
args = parser.parse_args()
|
|
291
|
+
|
|
292
|
+
main(args)
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
# This file is adapted from the scatter implementation of the pmwd library
|
|
2
|
+
# https://github.com/eelregit/pmwd/blob/master/pmwd/scatter.py
|
|
3
|
+
# It provides a simple way to perform a scatter operation by chunks and saves
|
|
4
|
+
# memory compared to a native jax.lax.scatter.
|
|
5
|
+
# Below is the orginal license of the pmwd library:
|
|
6
|
+
###############################################################################
|
|
7
|
+
# BSD 3-Clause License
|
|
8
|
+
#
|
|
9
|
+
# Copyright (c) 2021, the pmwd developers
|
|
10
|
+
# All rights reserved.
|
|
11
|
+
#
|
|
12
|
+
# Redistribution and use in source and binary forms, with or without
|
|
13
|
+
# modification, are permitted provided that the following conditions are met:
|
|
14
|
+
#
|
|
15
|
+
# 1. Redistributions of source code must retain the above copyright notice, this
|
|
16
|
+
# list of conditions and the following disclaimer.
|
|
17
|
+
#
|
|
18
|
+
# 2. Redistributions in binary form must reproduce the above copyright notice,
|
|
19
|
+
# this list of conditions and the following disclaimer in the documentation
|
|
20
|
+
# and/or other materials provided with the distribution.
|
|
21
|
+
#
|
|
22
|
+
# 3. Neither the name of the copyright holder nor the names of its
|
|
23
|
+
# contributors may be used to endorse or promote products derived from
|
|
24
|
+
# this software without specific prior written permission.
|
|
25
|
+
#
|
|
26
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
27
|
+
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
28
|
+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
29
|
+
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
30
|
+
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
31
|
+
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
32
|
+
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
33
|
+
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
34
|
+
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
35
|
+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
36
|
+
import jax.numpy as jnp
|
|
37
|
+
from jax.lax import scan
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _chunk_split(ptcl_num, chunk_size, *arrays):
|
|
41
|
+
"""Split and reshape particle arrays into chunks and remainders, with the remainders
|
|
42
|
+
preceding the chunks. 0D ones are duplicated as full arrays in the chunks."""
|
|
43
|
+
chunk_size = ptcl_num if chunk_size is None else min(chunk_size, ptcl_num)
|
|
44
|
+
remainder_size = ptcl_num % chunk_size
|
|
45
|
+
chunk_num = ptcl_num // chunk_size
|
|
46
|
+
|
|
47
|
+
remainder = None
|
|
48
|
+
chunks = arrays
|
|
49
|
+
if remainder_size:
|
|
50
|
+
remainder = [x[:remainder_size] if x.ndim != 0 else x for x in arrays]
|
|
51
|
+
chunks = [x[remainder_size:] if x.ndim != 0 else x for x in arrays]
|
|
52
|
+
|
|
53
|
+
# `scan` triggers errors in scatter and gather without the `full`
|
|
54
|
+
chunks = [
|
|
55
|
+
x.reshape(chunk_num, chunk_size, *x.shape[1:])
|
|
56
|
+
if x.ndim != 0
|
|
57
|
+
else jnp.full(chunk_num, x)
|
|
58
|
+
for x in chunks
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
return remainder, chunks
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def enmesh(i1, d1, a1, s1, b12, a2, s2):
|
|
65
|
+
"""Multilinear enmeshing."""
|
|
66
|
+
i1 = jnp.asarray(i1)
|
|
67
|
+
d1 = jnp.asarray(d1)
|
|
68
|
+
a1 = jnp.float64(a1) if a2 is not None else jnp.array(a1, dtype=d1.dtype)
|
|
69
|
+
if s1 is not None:
|
|
70
|
+
s1 = jnp.array(s1, dtype=i1.dtype)
|
|
71
|
+
b12 = jnp.float64(b12)
|
|
72
|
+
if a2 is not None:
|
|
73
|
+
a2 = jnp.float64(a2)
|
|
74
|
+
if s2 is not None:
|
|
75
|
+
s2 = jnp.array(s2, dtype=i1.dtype)
|
|
76
|
+
|
|
77
|
+
dim = i1.shape[1]
|
|
78
|
+
neighbors = (
|
|
79
|
+
jnp.arange(2**dim, dtype=i1.dtype)[:, jnp.newaxis]
|
|
80
|
+
>> jnp.arange(dim, dtype=i1.dtype)
|
|
81
|
+
) & 1
|
|
82
|
+
|
|
83
|
+
if a2 is not None:
|
|
84
|
+
P = i1 * a1 + d1 - b12
|
|
85
|
+
P = P[:, jnp.newaxis] # insert neighbor axis
|
|
86
|
+
i2 = P + neighbors * a2 # multilinear
|
|
87
|
+
|
|
88
|
+
if s1 is not None:
|
|
89
|
+
L = s1 * a1
|
|
90
|
+
i2 %= L
|
|
91
|
+
|
|
92
|
+
i2 //= a2
|
|
93
|
+
d2 = P - i2 * a2
|
|
94
|
+
|
|
95
|
+
if s1 is not None:
|
|
96
|
+
d2 -= jnp.rint(d2 / L) * L # also abs(d2) < a2 is expected
|
|
97
|
+
|
|
98
|
+
i2 = i2.astype(i1.dtype)
|
|
99
|
+
d2 = d2.astype(d1.dtype)
|
|
100
|
+
a2 = a2.astype(d1.dtype)
|
|
101
|
+
|
|
102
|
+
d2 /= a2
|
|
103
|
+
else:
|
|
104
|
+
i12, d12 = jnp.divmod(b12, a1)
|
|
105
|
+
i1 -= i12.astype(i1.dtype)
|
|
106
|
+
d1 -= d12.astype(d1.dtype)
|
|
107
|
+
|
|
108
|
+
# insert neighbor axis
|
|
109
|
+
i1 = i1[:, jnp.newaxis]
|
|
110
|
+
d1 = d1[:, jnp.newaxis]
|
|
111
|
+
|
|
112
|
+
# multilinear
|
|
113
|
+
d1 /= a1
|
|
114
|
+
i2 = jnp.floor(d1).astype(i1.dtype)
|
|
115
|
+
i2 += neighbors
|
|
116
|
+
d2 = d1 - i2
|
|
117
|
+
i2 += i1
|
|
118
|
+
|
|
119
|
+
if s1 is not None:
|
|
120
|
+
i2 %= s1
|
|
121
|
+
|
|
122
|
+
f2 = 1 - jnp.abs(d2)
|
|
123
|
+
|
|
124
|
+
if s1 is None and s2 is not None: # all i2 >= 0 if s1 is not None
|
|
125
|
+
i2 = jnp.where(i2 < 0, s2, i2)
|
|
126
|
+
|
|
127
|
+
f2 = f2.prod(axis=-1)
|
|
128
|
+
|
|
129
|
+
return i2, f2
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def scatter(pmid, disp, mesh, chunk_size=2**24, val=1.0, offset=0, cell_size=1.0):
|
|
133
|
+
ptcl_num, spatial_ndim = pmid.shape
|
|
134
|
+
val = jnp.asarray(val)
|
|
135
|
+
mesh = jnp.asarray(mesh)
|
|
136
|
+
|
|
137
|
+
remainder, chunks = _chunk_split(ptcl_num, chunk_size, pmid, disp, val)
|
|
138
|
+
|
|
139
|
+
carry = mesh, offset, cell_size
|
|
140
|
+
if remainder is not None:
|
|
141
|
+
carry = _scatter_chunk(carry, remainder)[0]
|
|
142
|
+
carry = scan(_scatter_chunk, carry, chunks)[0]
|
|
143
|
+
mesh = carry[0]
|
|
144
|
+
|
|
145
|
+
return mesh
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _scatter_chunk(carry, chunk):
|
|
149
|
+
mesh, offset, cell_size = carry
|
|
150
|
+
pmid, disp, val = chunk
|
|
151
|
+
spatial_ndim = pmid.shape[1]
|
|
152
|
+
spatial_shape = mesh.shape
|
|
153
|
+
|
|
154
|
+
# multilinear mesh indices and fractions
|
|
155
|
+
ind, frac = enmesh(
|
|
156
|
+
pmid, disp, cell_size, spatial_shape, offset, cell_size, spatial_shape
|
|
157
|
+
)
|
|
158
|
+
# scatter
|
|
159
|
+
ind = tuple(ind[..., i] for i in range(spatial_ndim))
|
|
160
|
+
mesh = mesh.at[ind].add(val * frac)
|
|
161
|
+
|
|
162
|
+
carry = mesh, offset, cell_size
|
|
163
|
+
return carry, None
|
|
@@ -70,8 +70,8 @@
|
|
|
70
70
|
}
|
|
71
71
|
],
|
|
72
72
|
"source": [
|
|
73
|
-
"folder =
|
|
74
|
-
"pdims=(4,4)\n",
|
|
73
|
+
"folder = \"../out\"\n",
|
|
74
|
+
"pdims = (4, 4)\n",
|
|
75
75
|
"\n",
|
|
76
76
|
"init_field_slices = []\n",
|
|
77
77
|
"field_slices = []\n",
|
|
@@ -79,11 +79,11 @@
|
|
|
79
79
|
"for i in range(pdims[0]):\n",
|
|
80
80
|
" row_init_field = []\n",
|
|
81
81
|
" row_field = []\n",
|
|
82
|
-
"
|
|
82
|
+
"\n",
|
|
83
83
|
" for j in range(pdims[1]):\n",
|
|
84
|
-
" slice_index = i * pdims[1]
|
|
85
|
-
" row_field.append(np.load(f
|
|
86
|
-
" row_init_field.append(np.load(f
|
|
84
|
+
" slice_index = i * pdims[1] + j\n",
|
|
85
|
+
" row_field.append(np.load(f\"{folder}/field_{slice_index}.npy\"))\n",
|
|
86
|
+
" row_init_field.append(np.load(f\"{folder}/initial_conditions_{slice_index}.npy\"))\n",
|
|
87
87
|
"\n",
|
|
88
88
|
" field_slices.append(np.vstack(row_field))\n",
|
|
89
89
|
" init_field_slices.append(np.vstack(row_init_field))\n",
|
|
@@ -148,23 +148,31 @@
|
|
|
148
148
|
" slicing[proj_axis] = slice(None, sum_over)\n",
|
|
149
149
|
" slicing = tuple(slicing)\n",
|
|
150
150
|
"\n",
|
|
151
|
-
"\n",
|
|
152
151
|
" fig, axes = plt.subplots(1, 2, figsize=(12, 6))\n",
|
|
153
152
|
"\n",
|
|
154
153
|
" # Flatten axes for easy indexing\n",
|
|
155
154
|
" axes = axes.flatten()\n",
|
|
156
155
|
"\n",
|
|
157
156
|
" # Plot initial conditions\n",
|
|
158
|
-
" axes[0].imshow(
|
|
159
|
-
"
|
|
160
|
-
"
|
|
161
|
-
"
|
|
157
|
+
" axes[0].imshow(\n",
|
|
158
|
+
" initial_conditions[slicing].sum(axis=proj_axis),\n",
|
|
159
|
+
" cmap=\"magma\",\n",
|
|
160
|
+
" extent=[0, box_size + 5, 0, box_size + 5],\n",
|
|
161
|
+
" )\n",
|
|
162
|
+
" axes[0].set_xlabel(\"Mpc/h\")\n",
|
|
163
|
+
" axes[0].set_ylabel(\"Mpc/h\")\n",
|
|
164
|
+
" axes[0].set_title(\"Initial conditions\")\n",
|
|
162
165
|
"\n",
|
|
163
166
|
" # Plot LPT density field at z=0\n",
|
|
164
|
-
" axes[1].imshow(
|
|
165
|
-
"
|
|
166
|
-
"
|
|
167
|
-
"
|
|
167
|
+
" axes[1].imshow(\n",
|
|
168
|
+
" field[slicing].sum(axis=proj_axis),\n",
|
|
169
|
+
" cmap=\"magma\",\n",
|
|
170
|
+
" extent=[0, box_size + 5, 0, box_size + 5],\n",
|
|
171
|
+
" )\n",
|
|
172
|
+
" axes[1].set_xlabel(\"Mpc/h\")\n",
|
|
173
|
+
" axes[1].set_ylabel(\"Mpc/h\")\n",
|
|
174
|
+
" axes[1].set_title(\"LPT density field at z=0\")\n",
|
|
175
|
+
"\n",
|
|
168
176
|
"\n",
|
|
169
177
|
"for i in range(3):\n",
|
|
170
178
|
" plot(i)"
|
|
@@ -201,10 +209,14 @@
|
|
|
201
209
|
"box_size = initial_conditions.shape[0] + 5\n",
|
|
202
210
|
"plt.figure(figsize=(20, 20))\n",
|
|
203
211
|
"# Generate the plot\n",
|
|
204
|
-
"plt.imshow(
|
|
205
|
-
"
|
|
206
|
-
"
|
|
207
|
-
"
|
|
212
|
+
"plt.imshow(\n",
|
|
213
|
+
" np.log10(field[:16].sum(axis=proj_axis) + 1),\n",
|
|
214
|
+
" cmap=\"magma\",\n",
|
|
215
|
+
" extent=[0, box_size, 0, box_size],\n",
|
|
216
|
+
")\n",
|
|
217
|
+
"plt.xlabel(\"Mpc/h\")\n",
|
|
218
|
+
"plt.ylabel(\"Mpc/h\")\n",
|
|
219
|
+
"plt.title(\"LPT density field at z=0\")\n",
|
|
208
220
|
"\n",
|
|
209
221
|
"# Display the plot\n",
|
|
210
222
|
"plt.show()"
|
|
@@ -3,7 +3,7 @@ requires = ["scikit-build-core>=0.4.0", "pybind11>=2.9.0"]
|
|
|
3
3
|
build-backend = "scikit_build_core.build"
|
|
4
4
|
[project]
|
|
5
5
|
name = "jaxdecomp"
|
|
6
|
-
version = "0.2.
|
|
6
|
+
version = "0.2.4"
|
|
7
7
|
description = "JAX bindings for the cuDecomp library"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name = "Wassim Kabalan" },
|
|
@@ -24,11 +24,12 @@ classifiers = [
|
|
|
24
24
|
]
|
|
25
25
|
dependencies = [
|
|
26
26
|
"jaxtyping>=0.2.0",
|
|
27
|
-
"jax>=0.4.
|
|
27
|
+
"jax>=0.4.35",
|
|
28
|
+
"typing-extensions; python_version < '3.11'",
|
|
28
29
|
]
|
|
29
30
|
|
|
30
31
|
[project.optional-dependencies]
|
|
31
|
-
test = ["pytest>=8.0.0"
|
|
32
|
+
test = ["pytest>=8.0.0"]
|
|
32
33
|
|
|
33
34
|
[tool.scikit-build]
|
|
34
35
|
minimum-version = "0.8"
|
|
@@ -42,3 +43,5 @@ CMAKE_LIBRARY_OUTPUT_DIRECTORY = ""
|
|
|
42
43
|
[tool.cibuildwheel]
|
|
43
44
|
test-extras = "test"
|
|
44
45
|
test-command = "pytest {project}/tests"
|
|
46
|
+
[tool.ruff]
|
|
47
|
+
extend-ignore = ["E402"]
|
|
@@ -17,11 +17,12 @@ config.halo_comm_backend = jaxdecomp.config.halo_comm_backend
|
|
|
17
17
|
config.transpose_comm_backend = jaxdecomp.config.transpose_comm_backend
|
|
18
18
|
|
|
19
19
|
# Run autotune
|
|
20
|
-
tuned_config = jaxdecomp.get_autotuned_config(
|
|
21
|
-
|
|
20
|
+
tuned_config = jaxdecomp.get_autotuned_config(
|
|
21
|
+
config, False, False, True, True, (32, 32, 32), (True, True, True)
|
|
22
|
+
)
|
|
22
23
|
|
|
23
24
|
if rank == 0:
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
25
|
+
print(rank, "*** Results of optimization ***")
|
|
26
|
+
print(rank, "pdims", tuned_config.pdims)
|
|
27
|
+
print(rank, "halo_comm_backend", tuned_config.halo_comm_backend)
|
|
28
|
+
print(rank, "transpose_comm_backend", tuned_config.transpose_comm_backend)
|