morphogen 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (381) hide show
  1. morphogen/__init__.py +60 -0
  2. morphogen/__main__.py +7 -0
  3. morphogen/_platform.py +55 -0
  4. morphogen/_public.py +153 -0
  5. morphogen/_utils.py +80 -0
  6. morphogen/_version.py +1 -0
  7. morphogen/analysis/__init__.py +174 -0
  8. morphogen/analysis/anisotropy.py +135 -0
  9. morphogen/analysis/basic.py +38 -0
  10. morphogen/analysis/cells.py +54 -0
  11. morphogen/analysis/connectivity.py +156 -0
  12. morphogen/analysis/correlation.py +195 -0
  13. morphogen/analysis/covariance.py +80 -0
  14. morphogen/analysis/fractal.py +54 -0
  15. morphogen/analysis/measures.py +188 -0
  16. morphogen/analysis/morphology.py +265 -0
  17. morphogen/analysis/particles.py +160 -0
  18. morphogen/analysis/percolation.py +2034 -0
  19. morphogen/analysis/power_laws.py +143 -0
  20. morphogen/analysis/random_sets.py +351 -0
  21. morphogen/analysis/stereology.py +289 -0
  22. morphogen/analysis/surface.py +107 -0
  23. morphogen/analysis/topology.py +92 -0
  24. morphogen/automata/__init__.py +74 -0
  25. morphogen/automata/engine.py +211 -0
  26. morphogen/automata/growth.py +686 -0
  27. morphogen/automata/lattice.py +143 -0
  28. morphogen/automata/reaction.py +255 -0
  29. morphogen/automata/rules.py +353 -0
  30. morphogen/automata/tilings.py +1715 -0
  31. morphogen/cli.py +3526 -0
  32. morphogen/config.py +2167 -0
  33. morphogen/contact/__init__.py +113 -0
  34. morphogen/contact/experiments.py +368 -0
  35. morphogen/contact/hertz.py +104 -0
  36. morphogen/contact/kernels.py +827 -0
  37. morphogen/contact/periodic.py +278 -0
  38. morphogen/contact/problem.py +578 -0
  39. morphogen/contact/surfaces.py +310 -0
  40. morphogen/coupling/__init__.py +43 -0
  41. morphogen/coupling/ageing.py +143 -0
  42. morphogen/coupling/diffusion.py +226 -0
  43. morphogen/coupling/fenicsx.py +484 -0
  44. morphogen/coupling/oxidation.py +447 -0
  45. morphogen/coupling/plates.py +247 -0
  46. morphogen/design/__init__.py +51 -0
  47. morphogen/design/csp.py +616 -0
  48. morphogen/design/frames.py +277 -0
  49. morphogen/design/search.py +287 -0
  50. morphogen/design/tiles.py +336 -0
  51. morphogen/distributions.py +312 -0
  52. morphogen/domain.py +203 -0
  53. morphogen/dynamics/__init__.py +179 -0
  54. morphogen/dynamics/model.py +638 -0
  55. morphogen/dynamics/openradioss.py +643 -0
  56. morphogen/dynamics/results.py +515 -0
  57. morphogen/fracture/__init__.py +170 -0
  58. morphogen/fracture/bulk.py +349 -0
  59. morphogen/fracture/cohesive.py +1614 -0
  60. morphogen/fracture/ductile.py +557 -0
  61. morphogen/fracture/lefm.py +1168 -0
  62. morphogen/fracture/mixed.py +1178 -0
  63. morphogen/fracture/periodic.py +733 -0
  64. morphogen/fracture/phasefield.py +714 -0
  65. morphogen/fracture/phasefield_finite.py +368 -0
  66. morphogen/fracture/surface.py +572 -0
  67. morphogen/fracture/toughness.py +566 -0
  68. morphogen/gpu.py +108 -0
  69. morphogen/gui/__init__.py +8 -0
  70. morphogen/gui/api.py +1927 -0
  71. morphogen/gui/desktop.py +387 -0
  72. morphogen/gui/icon.png +0 -0
  73. morphogen/gui/index.html +1909 -0
  74. morphogen/gui/server.py +229 -0
  75. morphogen/gui/tools/__init__.py +629 -0
  76. morphogen/gui/tools/compare.py +149 -0
  77. morphogen/gui/tools/contact.py +391 -0
  78. morphogen/gui/tools/couplings.py +1085 -0
  79. morphogen/gui/tools/crackfront.py +245 -0
  80. morphogen/gui/tools/design.py +224 -0
  81. morphogen/gui/tools/dynamics.py +824 -0
  82. morphogen/gui/tools/fracture.py +2586 -0
  83. morphogen/gui/tools/interop.py +1654 -0
  84. morphogen/gui/tools/learning.py +595 -0
  85. morphogen/gui/tools/mechanics.py +3881 -0
  86. morphogen/gui/tools/nonlocal_damage.py +1321 -0
  87. morphogen/gui/tools/phasefield.py +215 -0
  88. morphogen/gui/tools/physics.py +597 -0
  89. morphogen/gui/tools/stability.py +1756 -0
  90. morphogen/gui/tools/statistics.py +1408 -0
  91. morphogen/gui/viewer.js +1100 -0
  92. morphogen/homogenization/__init__.py +210 -0
  93. morphogen/homogenization/acoustics.py +566 -0
  94. morphogen/homogenization/amg.py +1005 -0
  95. morphogen/homogenization/apparent.py +1046 -0
  96. morphogen/homogenization/behaviours.cu +2285 -0
  97. morphogen/homogenization/behaviours.py +944 -0
  98. morphogen/homogenization/beremin.py +263 -0
  99. morphogen/homogenization/bifurcation.py +1085 -0
  100. morphogen/homogenization/bloch.py +507 -0
  101. morphogen/homogenization/bounds.py +226 -0
  102. morphogen/homogenization/core.py +458 -0
  103. morphogen/homogenization/coupled.py +1135 -0
  104. morphogen/homogenization/crystal.py +2967 -0
  105. morphogen/homogenization/dilatation.py +454 -0
  106. morphogen/homogenization/elements.py +157 -0
  107. morphogen/homogenization/fatigue.py +768 -0
  108. morphogen/homogenization/fe.py +1739 -0
  109. morphogen/homogenization/fenicsx.py +307 -0
  110. morphogen/homogenization/fft.py +499 -0
  111. morphogen/homogenization/fft_nonlinear.py +391 -0
  112. morphogen/homogenization/finite.py +613 -0
  113. morphogen/homogenization/flow.py +361 -0
  114. morphogen/homogenization/frames.py +1125 -0
  115. morphogen/homogenization/gpu.py +1241 -0
  116. morphogen/homogenization/harmonic.py +427 -0
  117. morphogen/homogenization/identification.py +579 -0
  118. morphogen/homogenization/kernels.py +760 -0
  119. morphogen/homogenization/laws.py +2061 -0
  120. morphogen/homogenization/materials.py +1080 -0
  121. morphogen/homogenization/mfront.py +589 -0
  122. morphogen/homogenization/networks.py +880 -0
  123. morphogen/homogenization/nonlinear.py +2612 -0
  124. morphogen/homogenization/ntfa.py +577 -0
  125. morphogen/homogenization/paths.py +476 -0
  126. morphogen/homogenization/photonic.py +467 -0
  127. morphogen/homogenization/poroelastic.py +212 -0
  128. morphogen/homogenization/porous.py +1120 -0
  129. morphogen/homogenization/postprocess.py +290 -0
  130. morphogen/homogenization/solvers.py +1140 -0
  131. morphogen/homogenization/tensors.py +240 -0
  132. morphogen/homogenization/voxel_fe.py +457 -0
  133. morphogen/homogenization/xfem.py +1016 -0
  134. morphogen/homogenization/zmat.py +1258 -0
  135. morphogen/iga/__init__.py +48 -0
  136. morphogen/iga/analysis.py +508 -0
  137. morphogen/iga/basis.py +214 -0
  138. morphogen/iga/geometry.py +501 -0
  139. morphogen/iga/homogenization.py +219 -0
  140. morphogen/iga/optimization.py +700 -0
  141. morphogen/iga/shapes.py +331 -0
  142. morphogen/image.py +128 -0
  143. morphogen/io/__init__.py +246 -0
  144. morphogen/io/amitex.py +741 -0
  145. morphogen/io/ebsd.py +881 -0
  146. morphogen/io/export.py +196 -0
  147. morphogen/io/geometry.py +158 -0
  148. morphogen/io/medit.py +147 -0
  149. morphogen/io/meshes.py +844 -0
  150. morphogen/io/metaimage.py +97 -0
  151. morphogen/io/mfront.py +1104 -0
  152. morphogen/io/native.py +57 -0
  153. morphogen/io/radioss.py +764 -0
  154. morphogen/io/solvers.py +371 -0
  155. morphogen/io/tables.py +278 -0
  156. morphogen/io/tiff.py +66 -0
  157. morphogen/io/vtk.py +701 -0
  158. morphogen/io/xdmf.py +200 -0
  159. morphogen/kernels/__init__.py +5 -0
  160. morphogen/kernels/annealing.py +192 -0
  161. morphogen/kernels/cells.py +69 -0
  162. morphogen/kernels/collective.py +87 -0
  163. morphogen/kernels/force_biased.py +595 -0
  164. morphogen/kernels/implicit.py +145 -0
  165. morphogen/kernels/leaves.py +132 -0
  166. morphogen/kernels/packing.py +304 -0
  167. morphogen/kernels/raster.py +279 -0
  168. morphogen/kernels/rsa.py +371 -0
  169. morphogen/kernels/union.py +135 -0
  170. morphogen/kernels/watershed.py +190 -0
  171. morphogen/laser/__init__.py +234 -0
  172. morphogen/laser/architectures.py +375 -0
  173. morphogen/laser/bending.py +216 -0
  174. morphogen/laser/cells.py +1694 -0
  175. morphogen/laser/design.py +940 -0
  176. morphogen/laser/figures.py +227 -0
  177. morphogen/laser/heat.py +804 -0
  178. morphogen/laser/materials.py +691 -0
  179. morphogen/laser/mechanics.py +1670 -0
  180. morphogen/laser/metallurgy.py +615 -0
  181. morphogen/laser/patterns.py +828 -0
  182. morphogen/laser/periodic.py +392 -0
  183. morphogen/laser/process.py +362 -0
  184. morphogen/laser/run.py +534 -0
  185. morphogen/laser/solid.py +216 -0
  186. morphogen/laser/specimen.py +200 -0
  187. morphogen/laser/structures.py +708 -0
  188. morphogen/learning/__init__.py +81 -0
  189. morphogen/learning/behaviour.py +398 -0
  190. morphogen/learning/data.py +328 -0
  191. morphogen/learning/learned.cpp +668 -0
  192. morphogen/learning/logarithmic.py +202 -0
  193. morphogen/learning/mfront.py +409 -0
  194. morphogen/learning/networks.py +767 -0
  195. morphogen/learning/run.py +212 -0
  196. morphogen/learning/standard.py +814 -0
  197. morphogen/learning/training.py +235 -0
  198. morphogen/mesh/__init__.py +29 -0
  199. morphogen/mesh/conforming.py +1423 -0
  200. morphogen/mesh/curved.py +600 -0
  201. morphogen/mesh/exact.py +1717 -0
  202. morphogen/mesh/octree.py +350 -0
  203. morphogen/mesh/polycrystal.py +1041 -0
  204. morphogen/mesh/polyhedra.py +677 -0
  205. morphogen/mesh/regularize.py +570 -0
  206. morphogen/mesh/surface.py +97 -0
  207. morphogen/mesh/tetra.py +512 -0
  208. morphogen/mesh/voxel.py +124 -0
  209. morphogen/micromorphic/__init__.py +31 -0
  210. morphogen/micromorphic/damage.py +359 -0
  211. morphogen/micromorphic/ductile.py +534 -0
  212. morphogen/micromorphic/finite.py +220 -0
  213. morphogen/micromorphic/microfoam.py +787 -0
  214. morphogen/micromorphic/space.py +341 -0
  215. morphogen/models/__init__.py +195 -0
  216. morphogen/models/affine.py +384 -0
  217. morphogen/models/automata.py +592 -0
  218. morphogen/models/base.py +500 -0
  219. morphogen/models/boolean.py +310 -0
  220. morphogen/models/bouligand.py +331 -0
  221. morphogen/models/branching.py +241 -0
  222. morphogen/models/combination.py +430 -0
  223. morphogen/models/deadleaves.py +766 -0
  224. morphogen/models/design.py +289 -0
  225. morphogen/models/ebsd.py +206 -0
  226. morphogen/models/fibers.py +1476 -0
  227. morphogen/models/foam.py +594 -0
  228. morphogen/models/gaussian.py +314 -0
  229. morphogen/models/growth.py +459 -0
  230. morphogen/models/hardcore.py +317 -0
  231. morphogen/models/hierarchical.py +269 -0
  232. morphogen/models/honeycomb.py +749 -0
  233. morphogen/models/implicit.py +177 -0
  234. morphogen/models/laminate.py +470 -0
  235. morphogen/models/lattice.py +351 -0
  236. morphogen/models/lattice_field.py +345 -0
  237. morphogen/models/levelset_foam.py +563 -0
  238. morphogen/models/nacre.py +411 -0
  239. morphogen/models/nested.py +924 -0
  240. morphogen/models/osteons.py +374 -0
  241. morphogen/models/pieces.py +167 -0
  242. morphogen/models/planar.py +126 -0
  243. morphogen/models/reconstruction.py +203 -0
  244. morphogen/models/scale_rich.py +1409 -0
  245. morphogen/models/scale_rich_3d.py +1616 -0
  246. morphogen/models/tessellation.py +735 -0
  247. morphogen/models/tiling_field.py +302 -0
  248. morphogen/models/tpms.py +197 -0
  249. morphogen/models/transparent.py +406 -0
  250. morphogen/models/union.py +141 -0
  251. morphogen/optimization/__init__.py +43 -0
  252. morphogen/optimization/cell.py +295 -0
  253. morphogen/optimization/filters.py +115 -0
  254. morphogen/optimization/mma.py +226 -0
  255. morphogen/optimization/run.py +219 -0
  256. morphogen/optimization/topology.py +632 -0
  257. morphogen/orientations.py +568 -0
  258. morphogen/parallel.py +599 -0
  259. morphogen/pipeline.py +978 -0
  260. morphogen/primitives.py +188 -0
  261. morphogen/results.py +564 -0
  262. morphogen/rve/__init__.py +73 -0
  263. morphogen/rve/report.py +627 -0
  264. morphogen/rve/scaling.py +363 -0
  265. morphogen/rve/study.py +1054 -0
  266. morphogen/rve/windows.py +345 -0
  267. morphogen/seeding.py +51 -0
  268. morphogen/settings.py +118 -0
  269. morphogen/stopping.py +48 -0
  270. morphogen/symmetry.py +384 -0
  271. morphogen/templates/activator_inhibitor.yaml +22 -0
  272. morphogen/templates/aggregation.yaml +23 -0
  273. morphogen/templates/boolean_cylinders.yaml +20 -0
  274. morphogen/templates/boolean_spheres.yaml +22 -0
  275. morphogen/templates/bouligand.yaml +28 -0
  276. morphogen/templates/cellular_automaton.yaml +25 -0
  277. morphogen/templates/coated_spheres.yaml +26 -0
  278. morphogen/templates/contact_fibers.yaml +27 -0
  279. morphogen/templates/cox_nanocomposite.yaml +22 -0
  280. morphogen/templates/cross_linked_fibers.yaml +26 -0
  281. morphogen/templates/curved_arcs.yaml +28 -0
  282. morphogen/templates/curved_fibers.yaml +24 -0
  283. morphogen/templates/dead_leaves.yaml +22 -0
  284. morphogen/templates/delaunay_network.yaml +22 -0
  285. morphogen/templates/dense_packing.yaml +22 -0
  286. morphogen/templates/diffusion.yaml +33 -0
  287. morphogen/templates/ebsd.yaml +34 -0
  288. morphogen/templates/ebsd_map.ctf +3087 -0
  289. morphogen/templates/fracture_composite.yaml +42 -0
  290. morphogen/templates/fracture_rubber.yaml +47 -0
  291. morphogen/templates/gaussian_excursion.yaml +17 -0
  292. morphogen/templates/graded_lattice.yaml +24 -0
  293. morphogen/templates/hardcore_fibers.yaml +22 -0
  294. morphogen/templates/hardcore_spheres.yaml +22 -0
  295. morphogen/templates/hierarchical_lattice.yaml +22 -0
  296. morphogen/templates/homogenization_acoustic_foam.yaml +36 -0
  297. morphogen/templates/homogenization_beremin.yaml +50 -0
  298. morphogen/templates/homogenization_cleavage.yaml +51 -0
  299. morphogen/templates/homogenization_cleavage_band.yaml +55 -0
  300. morphogen/templates/homogenization_contact.yaml +43 -0
  301. morphogen/templates/homogenization_crack_closure.yaml +49 -0
  302. morphogen/templates/homogenization_crystal_plasticity.yaml +50 -0
  303. morphogen/templates/homogenization_crystal_plasticity_fft.yaml +52 -0
  304. morphogen/templates/homogenization_crystallization.yaml +30 -0
  305. morphogen/templates/homogenization_debonding.yaml +43 -0
  306. morphogen/templates/homogenization_detwinning.yaml +58 -0
  307. morphogen/templates/homogenization_ductile_damage.yaml +47 -0
  308. morphogen/templates/homogenization_fatigue.yaml +40 -0
  309. morphogen/templates/homogenization_grain_fracture.yaml +56 -0
  310. morphogen/templates/homogenization_gyroid.yaml +30 -0
  311. morphogen/templates/homogenization_intergranular.yaml +43 -0
  312. morphogen/templates/homogenization_interphase.yaml +42 -0
  313. morphogen/templates/homogenization_mixed_loading.yaml +33 -0
  314. morphogen/templates/homogenization_ntfa.yaml +56 -0
  315. morphogen/templates/homogenization_piezoelectric.yaml +27 -0
  316. morphogen/templates/homogenization_polycrystal.yaml +30 -0
  317. morphogen/templates/homogenization_polycrystal_inclusions.yaml +30 -0
  318. morphogen/templates/homogenization_poroelastic.yaml +26 -0
  319. morphogen/templates/homogenization_rubber.yaml +40 -0
  320. morphogen/templates/homogenization_slip_cleavage.yaml +50 -0
  321. morphogen/templates/homogenization_spheres.yaml +31 -0
  322. morphogen/templates/homogenization_transport.yaml +29 -0
  323. morphogen/templates/homogenization_twinning.yaml +54 -0
  324. morphogen/templates/homogenization_viscoelastic.yaml +34 -0
  325. morphogen/templates/homogenization_viscoplastic.yaml +42 -0
  326. morphogen/templates/homogenization_voronoi.yaml +26 -0
  327. morphogen/templates/homogenization_xfem.yaml +23 -0
  328. morphogen/templates/hyperuniform_cells.yaml +24 -0
  329. morphogen/templates/laguerre.yaml +18 -0
  330. morphogen/templates/laser_bending.yaml +35 -0
  331. morphogen/templates/laser_cells.yaml +38 -0
  332. morphogen/templates/laser_corrugated.yaml +36 -0
  333. morphogen/templates/level_set_foam.yaml +29 -0
  334. morphogen/templates/menger_sponge.yaml +17 -0
  335. morphogen/templates/nacre.yaml +25 -0
  336. morphogen/templates/nucleation_growth.yaml +25 -0
  337. morphogen/templates/optimize_auxetic.yaml +29 -0
  338. morphogen/templates/optimize_auxetic_3d.yaml +27 -0
  339. morphogen/templates/optimize_bulk.yaml +23 -0
  340. morphogen/templates/optimize_conductivity.yaml +20 -0
  341. morphogen/templates/osteons.yaml +28 -0
  342. morphogen/templates/penetrable_spheres.yaml +21 -0
  343. morphogen/templates/poisson_fibers.yaml +21 -0
  344. morphogen/templates/random_laminate.yaml +26 -0
  345. morphogen/templates/random_lattice.yaml +27 -0
  346. morphogen/templates/random_walk_fibers.yaml +26 -0
  347. morphogen/templates/reaction_diffusion.yaml +22 -0
  348. morphogen/templates/reconstruction.yaml +22 -0
  349. morphogen/templates/rve_cellular_automaton.yaml +23 -0
  350. morphogen/templates/rve_crystal_plasticity_fft.yaml +51 -0
  351. morphogen/templates/rve_multiphysics.yaml +43 -0
  352. morphogen/templates/rve_poisson_fibers.yaml +22 -0
  353. morphogen/templates/rve_poisson_fibers_fe.yaml +33 -0
  354. morphogen/templates/rve_voronoi_windows.yaml +20 -0
  355. morphogen/templates/scale_rich_network.yaml +29 -0
  356. morphogen/templates/scale_rich_plates.yaml +30 -0
  357. morphogen/templates/sequential_dead_leaves.yaml +25 -0
  358. morphogen/templates/strut_lattice.yaml +21 -0
  359. morphogen/templates/tile_assembly.yaml +28 -0
  360. morphogen/templates/tpms.yaml +19 -0
  361. morphogen/templates/trabecular_bone.yaml +29 -0
  362. morphogen/templates/transparent_dead_leaves.yaml +23 -0
  363. morphogen/templates/vascular_tree.yaml +23 -0
  364. morphogen/templates/voronoi.yaml +20 -0
  365. morphogen/templates/wood.yaml +34 -0
  366. morphogen/templates/workflow.yaml +56 -0
  367. morphogen/texture.py +665 -0
  368. morphogen/theory.py +162 -0
  369. morphogen/viz/__init__.py +12 -0
  370. morphogen/viz/export.py +244 -0
  371. morphogen/viz/figures.py +707 -0
  372. morphogen/viz/png.py +148 -0
  373. morphogen/viz/sections.py +696 -0
  374. morphogen/viz/surfaces.py +1352 -0
  375. morphogen/viz/video.py +904 -0
  376. morphogen/viz/workflow.py +488 -0
  377. morphogen-1.0.0.dist-info/METADATA +2316 -0
  378. morphogen-1.0.0.dist-info/RECORD +381 -0
  379. morphogen-1.0.0.dist-info/WHEEL +4 -0
  380. morphogen-1.0.0.dist-info/entry_points.txt +5 -0
  381. morphogen-1.0.0.dist-info/licenses/LICENSE +674 -0
morphogen/parallel.py ADDED
@@ -0,0 +1,599 @@
1
+ """Parallel runs: processes on one machine, job arrays on clusters, MPI.
2
+
3
+ A run is a list of realizations (items), each with its own seed derived from the seed
4
+ of the run and its indices (:mod:`morphogen.seeding`): the results do not depend on how
5
+ the items are spread over processes, tasks or nodes.
6
+
7
+ * Every realization leaves a *record* (``records/<name>.json``: its row of results and
8
+ its rows of apparent properties) as soon as it is done. A run started again with
9
+ ``resume`` skips the realizations that have a record, so that an interrupted task (a
10
+ time limit on a cluster) loses at most the realizations in progress.
11
+ * ``jobs`` processes run the realizations of a task on one machine (``spawn`` start
12
+ method, on every platform); the threads (``threads``, by default all the cores) are
13
+ shared between them, so that the compiled kernels and the linear algebra do not
14
+ oversubscribe the cores.
15
+ * A run split in ``n_tasks`` tasks (``task_id`` = 0 ... n - 1) gives task ``i`` the items
16
+ ``i, i + n, i + 2n...``: a job array of a scheduler, or the ranks of an MPI program
17
+ (:func:`run_mpi`). :func:`merge` gathers the records of all the tasks into the results
18
+ tables and writes the RVE report; :func:`status` counts them.
19
+ * :func:`job_script` writes the script of a job array for SLURM, PBS Pro, Torque, SGE,
20
+ OAR or LSF, and the script of the merging job.
21
+ """
22
+
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import logging
27
+ import os
28
+ import re
29
+ import shlex
30
+ import subprocess
31
+ import sys
32
+ import time
33
+ from collections.abc import Iterator
34
+ from concurrent.futures import ProcessPoolExecutor, as_completed
35
+ from contextlib import contextmanager
36
+ from dataclasses import dataclass, field
37
+ from pathlib import Path
38
+
39
+ import numpy as np
40
+
41
+ from morphogen._utils import to_jsonable
42
+ from morphogen.stopping import Stopped, checkpoint
43
+
44
+ log = logging.getLogger("morphogen")
45
+
46
+ #: Environment variables bounding the threads of the linear algebra libraries.
47
+ THREAD_VARIABLES = (
48
+ "OMP_NUM_THREADS",
49
+ "OPENBLAS_NUM_THREADS",
50
+ "MKL_NUM_THREADS",
51
+ "VECLIB_MAXIMUM_THREADS",
52
+ "NUMEXPR_NUM_THREADS",
53
+ )
54
+
55
+
56
+ @dataclass(frozen=True)
57
+ class Item:
58
+ """One realization of a run: its position, seed key, file stem and domain size
59
+ (``None``: the domain of the configuration), and the columns identifying its rows."""
60
+
61
+ index: int
62
+ key: tuple[int, ...]
63
+ stem: str
64
+ size: float | None
65
+ ident: dict = field(default_factory=dict, hash=False, compare=False)
66
+
67
+
68
+ def items(cfg) -> list[Item]:
69
+ """Realizations of a run (a batch, or the sizes and realizations of an ensemble
70
+ RVE study)."""
71
+ if cfg.rve is None:
72
+ width = max(4, len(str(cfg.realizations - 1)))
73
+ return [
74
+ Item(r, (r,), f"{cfg.outputs.name}_{r:0{width}d}", None, {"realization": r})
75
+ for r in range(cfg.realizations)
76
+ ]
77
+ if cfg.rve.method != "ensemble":
78
+ raise ValueError("the window method of RVE studies runs one realization")
79
+ out: list[Item] = []
80
+ for s, n in enumerate(cfg.rve.realizations):
81
+ size = float(cfg.rve.sizes[s])
82
+ ident = {"size_index": s, "size": size, "volume": size**3}
83
+ for r in range(n):
84
+ stem = f"{cfg.outputs.name}_L{size:g}_{r:04d}"
85
+ out.append(Item(len(out), (s, r), stem, size, {**ident, "realization": r}))
86
+ return out
87
+
88
+
89
+ def task_items(all_items: list[Item], task_id: int, n_tasks: int) -> list[Item]:
90
+ """The items of a task among ``n_tasks`` (``task_id`` from 0): one item in ``n_tasks``, so that
91
+ the tasks share them evenly."""
92
+ if n_tasks < 1 or not 0 <= task_id < n_tasks:
93
+ raise ValueError(f"task id {task_id} must lie in [0, {n_tasks})")
94
+ return all_items[task_id::n_tasks]
95
+
96
+
97
+ # -- records -----------------------------------------------------------------------------
98
+
99
+
100
+ def records_dir(outdir) -> Path:
101
+ """The folder of the records of the items of a run, in its output folder."""
102
+ return Path(outdir) / "records"
103
+
104
+
105
+ def record_path(outdir, item: Item) -> Path:
106
+ """The record of an item of a run (``records/<item>.json`` in the output folder)."""
107
+ return records_dir(outdir) / f"{item.stem}.json"
108
+
109
+
110
+ def write_record(outdir, item: Item, record: dict) -> Path:
111
+ """Write a record atomically (a partial file never looks complete)."""
112
+ path = record_path(outdir, item)
113
+ path.parent.mkdir(parents=True, exist_ok=True)
114
+ tmp = path.with_suffix(f".{os.getpid()}.tmp")
115
+ tmp.write_text(json.dumps(to_jsonable(record)), encoding="utf-8")
116
+ os.replace(tmp, path)
117
+ return path
118
+
119
+
120
+ def read_records(outdir) -> list[dict]:
121
+ """All the records of a run directory, in the order of the run."""
122
+ folder = records_dir(outdir)
123
+ out = []
124
+ for path in sorted(folder.glob("*.json")) if folder.is_dir() else []:
125
+ try:
126
+ out.append(json.loads(path.read_text(encoding="utf-8")))
127
+ except (OSError, ValueError):
128
+ log.warning("unreadable record %s (skipped)", path)
129
+ return sorted(out, key=lambda r: r["index"])
130
+
131
+
132
+ # -- execution -----------------------------------------------------------------------------
133
+
134
+
135
+ def realize(cfg, item: Item, outdir) -> dict:
136
+ """Generate and process one realization; returns its record."""
137
+ from morphogen.domain import Domain
138
+ from morphogen.pipeline import _process
139
+ from morphogen.seeding import derive_seed
140
+
141
+ t0 = time.time()
142
+ domain = cfg.domain
143
+ if item.size is not None:
144
+ domain = Domain(
145
+ size=(item.size,) * 3,
146
+ origin=cfg.domain.origin,
147
+ periodic=cfg.domain.periodic,
148
+ unit=cfg.domain.unit,
149
+ )
150
+ micro = cfg.model.generate(domain, derive_seed(cfg.seed, *item.key))
151
+ measures = cfg.analysis if cfg.rve is None else cfg.rve.properties
152
+ values, fe = _process(micro, cfg, item.stem, Path(outdir), measures, item.index)
153
+ if cfg.rve is None:
154
+ row = {**item.ident, "seed": cfg.seed if cfg.seed is not None else "", **micro.summary()}
155
+ else:
156
+ row = {**item.ident, **micro.summary()}
157
+ row.update(values)
158
+ row["time_s"] = round(time.time() - t0, 4)
159
+ record = {
160
+ "index": item.index,
161
+ "stem": item.stem,
162
+ "row": row,
163
+ "fe_rows": [{**item.ident, **f} for f in fe],
164
+ }
165
+ return json.loads(json.dumps(to_jsonable(record))) # as read back from a record
166
+
167
+
168
+ def _worker_init(threads: int, level: int) -> None:
169
+ from morphogen import settings
170
+ from morphogen._platform import full_speed
171
+
172
+ logging.basicConfig(level=level, format="%(levelname)s: %(message)s")
173
+ log.setLevel(level)
174
+ settings.set_num_threads(threads)
175
+ full_speed() # Windows: not inherited from the parent process
176
+
177
+
178
+ def _worker(cfg, item: Item, outdir) -> dict:
179
+ record = realize(cfg, item, outdir)
180
+ write_record(outdir, item, record)
181
+ return record
182
+
183
+
184
+ def threads_per_job(jobs: int, threads: int | None = None) -> int:
185
+ """Threads of each of ``jobs`` processes sharing ``threads`` (default: the cores)."""
186
+ total = threads if threads is not None else (os.cpu_count() or 1)
187
+ return max(1, int(total) // max(1, int(jobs)))
188
+
189
+
190
+ @contextmanager
191
+ def process_pool(jobs: int, threads: int) -> Iterator[ProcessPoolExecutor]:
192
+ """Pool of ``jobs`` processes (``spawn``) with ``threads`` threads each."""
193
+ import multiprocessing
194
+
195
+ env = {name: str(threads) for name in THREAD_VARIABLES}
196
+ saved = {name: os.environ.get(name) for name in env}
197
+ os.environ.update(env) # inherited by the processes: read when they import NumPy
198
+ try:
199
+ with ProcessPoolExecutor(
200
+ max_workers=jobs,
201
+ mp_context=multiprocessing.get_context("spawn"),
202
+ initializer=_worker_init,
203
+ initargs=(threads, log.getEffectiveLevel()),
204
+ ) as pool:
205
+ yield pool
206
+ finally:
207
+ for name, value in saved.items():
208
+ if value is None:
209
+ os.environ.pop(name, None)
210
+ else:
211
+ os.environ[name] = value
212
+
213
+
214
+ def execute(cfg, todo: list[Item], outdir, *, jobs: int = 1, resume: bool = False) -> list[dict]:
215
+ """Records of the items ``todo`` (in their order), computed by ``jobs`` processes;
216
+ with ``resume``, the items that have a record are not computed again. Each
217
+ realization is a checkpoint of :mod:`morphogen.stopping` (a stop cancels those the
218
+ processes have not started)."""
219
+ outdir = Path(outdir)
220
+ records: dict[int, dict] = {}
221
+ pending = []
222
+ for item in todo:
223
+ path = record_path(outdir, item)
224
+ if resume and path.exists():
225
+ try:
226
+ records[item.index] = json.loads(path.read_text(encoding="utf-8"))
227
+ continue
228
+ except ValueError:
229
+ pass
230
+ pending.append(item)
231
+ if resume and records:
232
+ log.info("resuming: %d of %d realizations already done", len(records), len(todo))
233
+ if jobs <= 1 or len(pending) <= 1:
234
+ for item in pending:
235
+ checkpoint()
236
+ t0 = time.time()
237
+ records[item.index] = _worker(cfg, item, outdir)
238
+ log.info("%s done in %.2f s", item.stem, time.time() - t0)
239
+ else:
240
+ threads = threads_per_job(jobs, cfg.threads)
241
+ log.info("%d processes of %d threads", jobs, threads)
242
+ with process_pool(jobs, threads) as pool:
243
+ futures = {pool.submit(_worker, cfg, item, outdir): item for item in pending}
244
+ try:
245
+ for future in as_completed(futures):
246
+ item = futures[future]
247
+ records[item.index] = future.result()
248
+ log.info("%s done (%d/%d)", item.stem, len(records), len(todo))
249
+ checkpoint()
250
+ except Stopped:
251
+ pool.shutdown(wait=False, cancel_futures=True)
252
+ raise
253
+ return [records[item.index] for item in todo]
254
+
255
+
256
+ def rows_of(records: list[dict]) -> tuple[list[dict], list[dict]]:
257
+ """Results rows and homogenization rows of records."""
258
+ rows = [r["row"] for r in records]
259
+ fe_rows = [f for r in records for f in r["fe_rows"]]
260
+ return rows, fe_rows
261
+
262
+
263
+ # -- MPI -------------------------------------------------------------------------------------
264
+
265
+
266
+ def run_mpi(cfg, *, jobs: int = 1, resume: bool = False) -> Path:
267
+ """Run with the ranks of an MPI program (``mpirun -n N morphogen run config --mpi``):
268
+ rank ``i`` of ``N`` is task ``i``; rank 0 merges the results when all are done."""
269
+ try:
270
+ from mpi4py import MPI
271
+ except ImportError: # pragma: no cover - depends on the environment
272
+ raise ImportError("MPI runs need the optional 'mpi4py' package") from None
273
+ from morphogen import pipeline
274
+
275
+ comm = MPI.COMM_WORLD
276
+ rank, size = comm.Get_rank(), comm.Get_size()
277
+ if cfg.seed is None: # one seed for all the ranks
278
+ cfg.seed = comm.bcast(np.random.SeedSequence().entropy if rank == 0 else None, root=0)
279
+ outdir = pipeline.run(cfg, task_id=rank, n_tasks=size, jobs=jobs, resume=resume)
280
+ comm.Barrier()
281
+ if rank == 0 and size > 1:
282
+ merge(outdir)
283
+ comm.Barrier()
284
+ return outdir
285
+
286
+
287
+ # -- merging and status ------------------------------------------------------------------
288
+
289
+
290
+ def _manifest_value(outdir: Path, key: str):
291
+ """An entry of the manifest of a run (``run.json``, or that of one of its tasks)."""
292
+ for path in [outdir / "run.json", *sorted(outdir.glob("run_task*of*.json"))]:
293
+ if path.exists():
294
+ return json.loads(path.read_text(encoding="utf-8")).get(key)
295
+ return None
296
+
297
+
298
+ def _run_config(outdir: Path):
299
+ """Configuration of a run directory (from its run.json or run_task*.json)."""
300
+ from morphogen.config import load_config
301
+
302
+ files = [outdir / "run.json", *sorted(outdir.glob("run_task*of*.json"))]
303
+ for path in files:
304
+ if path.exists():
305
+ data = json.loads(path.read_text(encoding="utf-8"))
306
+ source = dict(data["config"])
307
+ if data.get("seed") is not None:
308
+ source["seed"] = data["seed"]
309
+ cfg = load_config(source, base_dir=outdir)
310
+ cfg.outputs.directory = outdir
311
+ return cfg
312
+ raise ValueError(f"{outdir}: no run.json (not a morphoGen run directory)")
313
+
314
+
315
+ def merge(outdir, *, report: bool = True) -> Path:
316
+ """Gather the records of all the tasks of a run into ``results.csv`` and
317
+ ``homogenization.csv``, and write the RVE report of an ensemble study."""
318
+ from morphogen import pipeline
319
+
320
+ outdir = Path(outdir)
321
+ cfg = _run_config(outdir)
322
+ expected = items(cfg)
323
+ records = read_records(outdir)
324
+ done = {r["stem"] for r in records}
325
+ missing = [it.stem for it in expected if it.stem not in done]
326
+ if missing:
327
+ raise ValueError(
328
+ f"{len(missing)} of {len(expected)} realizations have no record yet (e.g. "
329
+ f"{missing[0]}); run the missing tasks again with --resume, see 'morphogen status'"
330
+ )
331
+ rows, fe_rows = rows_of(records)
332
+ pipeline.write_rows(outdir / "results.csv", rows)
333
+ if fe_rows:
334
+ pipeline.write_rows(outdir / "homogenization.csv", fe_rows)
335
+ if report and cfg.rve is not None and rows:
336
+ pipeline.write_report(rows, outdir, cfg.rve, cfg, fe_rows=fe_rows)
337
+ database = _manifest_value(outdir, "database")
338
+ if database is not None:
339
+ from morphogen.results import record_run
340
+
341
+ record_run(outdir, database)
342
+ log.info("merged %d realizations in %s", len(records), outdir)
343
+ return outdir
344
+
345
+
346
+ def status(outdir) -> dict:
347
+ """Realizations done and missing in a run directory."""
348
+ outdir = Path(outdir)
349
+ cfg = _run_config(outdir)
350
+ expected = items(cfg)
351
+ records = read_records(outdir)
352
+ done = {r["stem"] for r in records}
353
+ times = [r["row"].get("time_s", 0.0) for r in records]
354
+ missing = [it.stem for it in expected if it.stem not in done]
355
+ return {
356
+ "directory": str(outdir),
357
+ "expected": len(expected),
358
+ "done": len(expected) - len(missing),
359
+ "missing": missing,
360
+ "mean_time_s": float(np.mean(times)) if times else float("nan"),
361
+ "merged": (outdir / "results.csv").exists(),
362
+ }
363
+
364
+
365
+ # -- job scripts -------------------------------------------------------------------------------
366
+
367
+ #: Array directive, task index (0-based) and submission command of each scheduler.
368
+ SCHEDULERS = {
369
+ "slurm": {
370
+ "prefix": "#SBATCH",
371
+ "array": "--array=0-{last}",
372
+ "index": "${SLURM_ARRAY_TASK_ID}",
373
+ "name": "--job-name={name}",
374
+ "cores": "--cpus-per-task={cores}",
375
+ "time": "--time={time}",
376
+ "memory": "--mem={memory}",
377
+ "queue": "--partition={queue}",
378
+ "account": "--account={account}",
379
+ "log": ["--output={logs}/%x_%A_%a.log"],
380
+ "submit": ["sbatch", "--parsable"],
381
+ "after": ["--dependency=afterok:{job}"],
382
+ },
383
+ "pbs": {
384
+ "prefix": "#PBS",
385
+ "array": "-J 0-{last}",
386
+ "index": "${PBS_ARRAY_INDEX}",
387
+ "name": "-N {name}",
388
+ "cores": "-l select=1:ncpus={cores}",
389
+ "time": "-l walltime={time}",
390
+ "memory": "-l mem={memory}",
391
+ "queue": "-q {queue}",
392
+ "account": "-A {account}",
393
+ "log": ["-o {logs}/", "-j oe"],
394
+ "submit": ["qsub"],
395
+ "after": ["-W", "depend=afterok:{job}"],
396
+ },
397
+ "torque": {
398
+ "prefix": "#PBS",
399
+ "array": "-t 0-{last}",
400
+ "index": "${PBS_ARRAYID}",
401
+ "name": "-N {name}",
402
+ "cores": "-l nodes=1:ppn={cores}",
403
+ "time": "-l walltime={time}",
404
+ "memory": "-l mem={memory}",
405
+ "queue": "-q {queue}",
406
+ "account": "-A {account}",
407
+ "log": ["-o {logs}/", "-j oe"],
408
+ "submit": ["qsub"],
409
+ "after": ["-W", "depend=afterokarray:{job}"],
410
+ },
411
+ "sge": {
412
+ "prefix": "#$",
413
+ "array": "-t 1-{count}",
414
+ "index": "$((SGE_TASK_ID - 1))",
415
+ "name": "-N {name}",
416
+ "cores": "-pe smp {cores}",
417
+ "time": "-l h_rt={time}",
418
+ "memory": "-l h_vmem={memory}",
419
+ "queue": "-q {queue}",
420
+ "account": "-A {account}",
421
+ "log": ["-o {logs}/", "-j y"],
422
+ "submit": ["qsub", "-terse"],
423
+ "after": ["-hold_jid", "{job}"],
424
+ },
425
+ "oar": {
426
+ "prefix": "#OAR",
427
+ "array": "--array {count}",
428
+ "index": "$((OAR_ARRAY_INDEX - 1))",
429
+ "name": "-n {name}",
430
+ "cores": "-l /nodes=1/core={cores},walltime={time}",
431
+ "time": None, # with the cores
432
+ "memory": None,
433
+ "queue": "-q {queue}",
434
+ "account": "--project {account}",
435
+ "log": ["-O {logs}/%jobid%.log", "-E {logs}/%jobid%.log"],
436
+ "submit": ["oarsub", "-S"],
437
+ "after": ["--anterior={job}"],
438
+ },
439
+ "lsf": {
440
+ "prefix": "#BSUB",
441
+ "array": None, # in the job name
442
+ "index": "$((LSB_JOBINDEX - 1))",
443
+ "name": "-J {name}[1-{count}]",
444
+ "cores": "-n {cores}",
445
+ "time": "-W {time}",
446
+ "memory": "-M {memory}",
447
+ "queue": "-q {queue}",
448
+ "account": "-P {account}",
449
+ "log": ["-o {logs}/%J_%I.log"],
450
+ "submit": ["bsub"],
451
+ "after": ["-w", "done({job})"],
452
+ },
453
+ }
454
+
455
+
456
+ @dataclass
457
+ class JobOptions:
458
+ """Resources of the tasks of a job array."""
459
+
460
+ tasks: int
461
+ jobs: int = 1
462
+ threads: int = 1
463
+ time: str = "01:00:00"
464
+ memory: str | None = None
465
+ queue: str | None = None
466
+ account: str | None = None
467
+ name: str = "morphogen"
468
+ setup: list[str] = field(default_factory=list) # shell lines (modules, environments)
469
+
470
+ @property
471
+ def cores(self) -> int:
472
+ return self.jobs * self.threads
473
+
474
+
475
+ def _directives(scheduler: str, opts: JobOptions, logs: Path, *, array: bool) -> list[str]:
476
+ spec = SCHEDULERS[scheduler]
477
+ time_limit = opts.time
478
+ if scheduler == "lsf": # LSF: [hours:]minutes
479
+ parts = [int(x) for x in time_limit.split(":")]
480
+ minutes = parts[0] * 60 + parts[1] + (1 if len(parts) > 2 and parts[2] else 0)
481
+ time_limit = f"{minutes // 60}:{minutes % 60:02d}" if len(parts) > 2 else time_limit
482
+ values = {
483
+ "name": opts.name if array else f"{opts.name}-merge",
484
+ "last": opts.tasks - 1,
485
+ "count": opts.tasks,
486
+ "cores": opts.cores if array else 1,
487
+ "time": time_limit,
488
+ "memory": opts.memory,
489
+ "queue": opts.queue,
490
+ "account": opts.account,
491
+ "logs": logs,
492
+ }
493
+ array = array and opts.tasks > 1
494
+ name = spec["name"] if (array or scheduler != "lsf") else "-J {name}"
495
+ lines = [name]
496
+ if array and spec["array"]:
497
+ lines.append(spec["array"])
498
+ lines += [spec["cores"], *spec["log"]]
499
+ for key in ("time", "memory", "queue", "account"):
500
+ if spec[key] and values[key] is not None:
501
+ lines.append(spec[key])
502
+ return [f"{spec['prefix']} {line.format(**values)}" for line in lines]
503
+
504
+
505
+ def job_script(
506
+ scheduler: str,
507
+ config,
508
+ outdir,
509
+ opts: JobOptions,
510
+ *,
511
+ seed: int | None = None,
512
+ executable: str | None = None,
513
+ ) -> tuple[str, str]:
514
+ """Scripts of the job array running a configuration in ``opts.tasks`` tasks (each
515
+ with ``opts.jobs`` processes of ``opts.threads`` threads, resumable), and of the job
516
+ merging the results."""
517
+ if scheduler not in SCHEDULERS:
518
+ raise ValueError(f"scheduler: one of {sorted(SCHEDULERS)}")
519
+ if opts.tasks < 1 or opts.jobs < 1 or opts.threads < 1:
520
+ raise ValueError("tasks, jobs and threads: positive integers")
521
+ outdir = Path(outdir).resolve()
522
+ logs = outdir / "logs"
523
+ exe = executable or f"{shlex.quote(sys.executable)} -m morphogen"
524
+ index = SCHEDULERS[scheduler]["index"] if opts.tasks > 1 else "0"
525
+ command = [
526
+ exe,
527
+ "run",
528
+ shlex.quote(str(Path(config).resolve())),
529
+ "-o",
530
+ shlex.quote(str(outdir)),
531
+ "--task-id",
532
+ f'"{index}"',
533
+ "--n-tasks",
534
+ str(opts.tasks),
535
+ "--jobs",
536
+ str(opts.jobs),
537
+ "--threads",
538
+ str(opts.cores),
539
+ "--resume",
540
+ "--quiet",
541
+ ]
542
+ if seed is not None:
543
+ command += ["--seed", str(seed)]
544
+ header = ["#!/bin/bash"]
545
+ body = [
546
+ "set -euo pipefail",
547
+ *opts.setup,
548
+ f"mkdir -p {shlex.quote(str(logs))}",
549
+ ]
550
+ array = "\n".join(
551
+ [*header, *_directives(scheduler, opts, logs, array=True), *body, " ".join(command)]
552
+ )
553
+ merge_cmd = f"{exe} merge {shlex.quote(str(outdir))}"
554
+ merging = "\n".join(
555
+ [*header, *_directives(scheduler, opts, logs, array=False), *body, merge_cmd]
556
+ )
557
+ return array + "\n", merging + "\n"
558
+
559
+
560
+ def _job_id(scheduler: str, output: str) -> str:
561
+ """Identifier of a submitted job in the output of the submission command."""
562
+ text = output.strip()
563
+ last = text.splitlines()[-1].strip() if text else ""
564
+ if scheduler == "slurm":
565
+ return last.split(";")[0]
566
+ if scheduler in ("pbs", "torque"):
567
+ return last
568
+ if scheduler == "lsf":
569
+ match = re.search(r"<(\d+)>", text)
570
+ elif scheduler == "oar":
571
+ match = re.search(r"OAR_ARRAY_ID=(\d+)", text) or re.search(r"OAR_JOB_ID=(\d+)", text)
572
+ else:
573
+ match = re.match(r"\d+", last)
574
+ if match is None:
575
+ raise RuntimeError(f"no job id in the output of the submission: {text!r}")
576
+ return match.group(1) if match.groups() else match.group(0)
577
+
578
+
579
+ def submit(scheduler: str, array_script: Path, merge_script: Path) -> list[str]:
580
+ """Submit the job array, then the merging job after it; returns their ids."""
581
+ spec = SCHEDULERS[scheduler]
582
+
583
+ def call(extra, script):
584
+ if scheduler == "lsf": # bsub reads the directives of a script from its input
585
+ with open(script, encoding="utf-8") as f:
586
+ res = subprocess.run(
587
+ [*spec["submit"], *extra], stdin=f, capture_output=True, text=True, check=True
588
+ )
589
+ else:
590
+ res = subprocess.run(
591
+ [*spec["submit"], *extra, str(script)], capture_output=True, text=True, check=True
592
+ )
593
+ return _job_id(scheduler, res.stdout)
594
+
595
+ for script in (array_script, merge_script):
596
+ Path(script).chmod(0o755)
597
+ first = call([], array_script)
598
+ after = [part.format(job=first) for part in spec["after"]]
599
+ return [first, call(after, merge_script)]