parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. parsl/__init__.py +9 -10
  2. parsl/addresses.py +6 -4
  3. parsl/app/app.py +3 -6
  4. parsl/app/bash.py +4 -4
  5. parsl/app/errors.py +5 -3
  6. parsl/app/futures.py +3 -3
  7. parsl/app/python.py +2 -1
  8. parsl/benchmark/perf.py +2 -1
  9. parsl/channels/__init__.py +2 -2
  10. parsl/channels/base.py +0 -1
  11. parsl/channels/errors.py +2 -1
  12. parsl/channels/oauth_ssh/oauth_ssh.py +4 -3
  13. parsl/channels/ssh/ssh.py +9 -1
  14. parsl/channels/ssh_il/ssh_il.py +1 -0
  15. parsl/concurrent/__init__.py +2 -2
  16. parsl/config.py +5 -5
  17. parsl/configs/ASPIRE1.py +3 -3
  18. parsl/configs/Azure.py +6 -7
  19. parsl/configs/ad_hoc.py +4 -3
  20. parsl/configs/bridges.py +3 -3
  21. parsl/configs/cc_in2p3.py +2 -2
  22. parsl/configs/ec2.py +1 -1
  23. parsl/configs/expanse.py +1 -2
  24. parsl/configs/frontera.py +2 -3
  25. parsl/configs/htex_local.py +1 -2
  26. parsl/configs/illinoiscluster.py +1 -1
  27. parsl/configs/kubernetes.py +1 -2
  28. parsl/configs/midway.py +3 -3
  29. parsl/configs/osg.py +1 -1
  30. parsl/configs/polaris.py +1 -1
  31. parsl/configs/stampede2.py +4 -5
  32. parsl/configs/summit.py +1 -3
  33. parsl/configs/toss3_llnl.py +1 -2
  34. parsl/configs/vineex_local.py +3 -3
  35. parsl/configs/wqex_local.py +2 -2
  36. parsl/data_provider/data_manager.py +3 -3
  37. parsl/data_provider/file_noop.py +1 -2
  38. parsl/data_provider/files.py +3 -3
  39. parsl/data_provider/ftp.py +1 -3
  40. parsl/data_provider/globus.py +7 -6
  41. parsl/data_provider/http.py +2 -2
  42. parsl/data_provider/rsync.py +1 -1
  43. parsl/data_provider/staging.py +2 -2
  44. parsl/data_provider/zip.py +4 -5
  45. parsl/dataflow/dflow.py +21 -16
  46. parsl/dataflow/errors.py +2 -1
  47. parsl/dataflow/futures.py +1 -2
  48. parsl/dataflow/memoization.py +5 -5
  49. parsl/dataflow/rundirs.py +1 -1
  50. parsl/dataflow/taskrecord.py +4 -5
  51. parsl/executors/__init__.py +3 -3
  52. parsl/executors/base.py +1 -0
  53. parsl/executors/flux/execute_parsl_task.py +2 -2
  54. parsl/executors/flux/executor.py +11 -12
  55. parsl/executors/flux/flux_instance_manager.py +3 -3
  56. parsl/executors/high_throughput/executor.py +29 -35
  57. parsl/executors/high_throughput/interchange.py +16 -18
  58. parsl/executors/high_throughput/manager_record.py +1 -0
  59. parsl/executors/high_throughput/monitoring_info.py +2 -1
  60. parsl/executors/high_throughput/mpi_executor.py +5 -2
  61. parsl/executors/high_throughput/mpi_prefix_composer.py +1 -1
  62. parsl/executors/high_throughput/mpi_resource_management.py +1 -2
  63. parsl/executors/high_throughput/probe.py +6 -4
  64. parsl/executors/high_throughput/process_worker_pool.py +31 -20
  65. parsl/executors/high_throughput/zmq_pipes.py +28 -14
  66. parsl/executors/radical/executor.py +15 -15
  67. parsl/executors/radical/rpex_master.py +1 -2
  68. parsl/executors/radical/rpex_resources.py +1 -2
  69. parsl/executors/radical/rpex_worker.py +2 -1
  70. parsl/executors/status_handling.py +5 -4
  71. parsl/executors/taskvine/__init__.py +1 -1
  72. parsl/executors/taskvine/errors.py +1 -1
  73. parsl/executors/taskvine/exec_parsl_function.py +2 -2
  74. parsl/executors/taskvine/executor.py +23 -24
  75. parsl/executors/taskvine/factory.py +1 -1
  76. parsl/executors/taskvine/manager.py +11 -13
  77. parsl/executors/threads.py +4 -5
  78. parsl/executors/workqueue/errors.py +1 -1
  79. parsl/executors/workqueue/exec_parsl_function.py +5 -4
  80. parsl/executors/workqueue/executor.py +26 -27
  81. parsl/executors/workqueue/parsl_coprocess.py +1 -1
  82. parsl/jobs/error_handlers.py +1 -1
  83. parsl/jobs/job_status_poller.py +2 -5
  84. parsl/jobs/states.py +1 -1
  85. parsl/jobs/strategy.py +2 -2
  86. parsl/launchers/__init__.py +12 -3
  87. parsl/launchers/errors.py +1 -1
  88. parsl/log_utils.py +1 -2
  89. parsl/monitoring/db_manager.py +16 -10
  90. parsl/monitoring/monitoring.py +11 -15
  91. parsl/monitoring/queries/pandas.py +1 -2
  92. parsl/monitoring/radios.py +2 -4
  93. parsl/monitoring/remote.py +13 -8
  94. parsl/monitoring/router.py +8 -11
  95. parsl/monitoring/types.py +2 -0
  96. parsl/monitoring/visualization/app.py +4 -2
  97. parsl/monitoring/visualization/models.py +0 -1
  98. parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
  99. parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
  100. parsl/monitoring/visualization/utils.py +0 -1
  101. parsl/monitoring/visualization/views.py +16 -9
  102. parsl/multiprocessing.py +0 -1
  103. parsl/process_loggers.py +1 -2
  104. parsl/providers/__init__.py +9 -12
  105. parsl/providers/ad_hoc/ad_hoc.py +1 -1
  106. parsl/providers/aws/aws.py +2 -3
  107. parsl/providers/azure/azure.py +4 -5
  108. parsl/providers/base.py +1 -1
  109. parsl/providers/cluster_provider.py +1 -1
  110. parsl/providers/cobalt/cobalt.py +3 -3
  111. parsl/providers/condor/condor.py +4 -2
  112. parsl/providers/errors.py +2 -2
  113. parsl/providers/googlecloud/googlecloud.py +2 -1
  114. parsl/providers/grid_engine/grid_engine.py +2 -2
  115. parsl/providers/kubernetes/kube.py +5 -3
  116. parsl/providers/local/local.py +5 -1
  117. parsl/providers/lsf/lsf.py +2 -2
  118. parsl/providers/pbspro/pbspro.py +1 -1
  119. parsl/providers/slurm/slurm.py +5 -5
  120. parsl/providers/torque/torque.py +1 -1
  121. parsl/serialize/__init__.py +8 -3
  122. parsl/serialize/base.py +1 -2
  123. parsl/serialize/concretes.py +5 -4
  124. parsl/serialize/proxystore.py +3 -2
  125. parsl/tests/__init__.py +1 -1
  126. parsl/tests/configs/ad_hoc_cluster_htex.py +4 -4
  127. parsl/tests/configs/azure_single_node.py +4 -5
  128. parsl/tests/configs/bridges.py +3 -2
  129. parsl/tests/configs/cc_in2p3.py +2 -2
  130. parsl/tests/configs/comet.py +2 -1
  131. parsl/tests/configs/ec2_single_node.py +1 -2
  132. parsl/tests/configs/ec2_spot.py +1 -2
  133. parsl/tests/configs/frontera.py +3 -2
  134. parsl/tests/configs/htex_ad_hoc_cluster.py +2 -4
  135. parsl/tests/configs/htex_local.py +2 -3
  136. parsl/tests/configs/htex_local_alternate.py +8 -11
  137. parsl/tests/configs/htex_local_intask_staging.py +5 -7
  138. parsl/tests/configs/htex_local_rsync_staging.py +4 -6
  139. parsl/tests/configs/local_adhoc.py +1 -1
  140. parsl/tests/configs/local_radical.py +1 -3
  141. parsl/tests/configs/local_radical_mpi.py +2 -2
  142. parsl/tests/configs/midway.py +2 -2
  143. parsl/tests/configs/nscc_singapore.py +3 -3
  144. parsl/tests/configs/osg_htex.py +1 -1
  145. parsl/tests/configs/petrelkube.py +3 -2
  146. parsl/tests/configs/summit.py +1 -0
  147. parsl/tests/configs/swan_htex.py +2 -2
  148. parsl/tests/configs/taskvine_ex.py +3 -5
  149. parsl/tests/configs/theta.py +2 -2
  150. parsl/tests/configs/workqueue_ex.py +3 -4
  151. parsl/tests/conftest.py +4 -4
  152. parsl/tests/integration/test_channels/test_ssh_errors.py +1 -1
  153. parsl/tests/integration/test_stress/test_python_simple.py +3 -4
  154. parsl/tests/integration/test_stress/test_python_threads.py +3 -5
  155. parsl/tests/manual_tests/htex_local.py +4 -4
  156. parsl/tests/manual_tests/test_ad_hoc_htex.py +2 -1
  157. parsl/tests/manual_tests/test_basic.py +1 -0
  158. parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +4 -4
  159. parsl/tests/manual_tests/test_log_filter.py +3 -1
  160. parsl/tests/manual_tests/test_memory_limits.py +6 -6
  161. parsl/tests/manual_tests/test_regression_220.py +2 -1
  162. parsl/tests/manual_tests/test_udp_simple.py +4 -3
  163. parsl/tests/manual_tests/test_worker_count.py +3 -2
  164. parsl/tests/scaling_tests/htex_local.py +2 -2
  165. parsl/tests/scaling_tests/test_scale.py +0 -9
  166. parsl/tests/scaling_tests/vineex_condor.py +1 -2
  167. parsl/tests/scaling_tests/vineex_local.py +1 -2
  168. parsl/tests/site_tests/test_provider.py +3 -1
  169. parsl/tests/site_tests/test_site.py +2 -0
  170. parsl/tests/sites/test_affinity.py +7 -5
  171. parsl/tests/sites/test_dynamic_executor.py +3 -3
  172. parsl/tests/sites/test_ec2.py +3 -2
  173. parsl/tests/sites/test_local_adhoc.py +2 -1
  174. parsl/tests/sites/test_worker_info.py +4 -3
  175. parsl/tests/test_aalst_patterns.py +0 -1
  176. parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
  177. parsl/tests/test_bash_apps/test_error_codes.py +1 -4
  178. parsl/tests/test_bash_apps/test_memoize_ignore_args.py +1 -0
  179. parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +2 -2
  180. parsl/tests/test_bash_apps/test_pipeline.py +1 -1
  181. parsl/tests/test_bash_apps/test_std_uri.py +4 -3
  182. parsl/tests/test_callables.py +2 -2
  183. parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
  184. parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -1
  185. parsl/tests/test_checkpointing/test_regression_239.py +1 -1
  186. parsl/tests/test_checkpointing/test_task_exit.py +1 -1
  187. parsl/tests/test_docs/test_from_slides.py +2 -2
  188. parsl/tests/test_docs/test_kwargs.py +1 -1
  189. parsl/tests/test_docs/test_tutorial_1.py +1 -2
  190. parsl/tests/test_docs/test_workflow1.py +2 -2
  191. parsl/tests/test_docs/test_workflow2.py +0 -1
  192. parsl/tests/test_error_handling/test_rand_fail.py +2 -2
  193. parsl/tests/test_error_handling/test_resource_spec.py +4 -2
  194. parsl/tests/test_error_handling/test_retries.py +2 -1
  195. parsl/tests/test_error_handling/test_retry_handler.py +1 -0
  196. parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
  197. parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
  198. parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
  199. parsl/tests/test_flux.py +1 -1
  200. parsl/tests/test_htex/test_command_client_timeout.py +9 -12
  201. parsl/tests/test_htex/test_connected_blocks.py +3 -2
  202. parsl/tests/test_htex/test_cpu_affinity_explicit.py +5 -2
  203. parsl/tests/test_htex/test_disconnected_blocks.py +6 -4
  204. parsl/tests/test_htex/test_drain.py +5 -5
  205. parsl/tests/test_htex/test_htex.py +1 -2
  206. parsl/tests/test_htex/test_managers_command.py +3 -2
  207. parsl/tests/test_htex/test_multiple_disconnected_blocks.py +6 -4
  208. parsl/tests/test_monitoring/test_app_names.py +3 -2
  209. parsl/tests/test_monitoring/test_basic.py +4 -4
  210. parsl/tests/test_monitoring/test_db_locks.py +6 -3
  211. parsl/tests/test_monitoring/test_fuzz_zmq.py +6 -3
  212. parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +5 -5
  213. parsl/tests/test_monitoring/test_incomplete_futures.py +5 -3
  214. parsl/tests/test_monitoring/test_memoization_representation.py +4 -1
  215. parsl/tests/test_monitoring/test_stdouterr.py +4 -4
  216. parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
  217. parsl/tests/test_mpi_apps/test_bad_mpi_config.py +1 -1
  218. parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +2 -0
  219. parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +7 -5
  220. parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
  221. parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
  222. parsl/tests/test_mpi_apps/test_mpiex.py +4 -3
  223. parsl/tests/test_mpi_apps/test_resource_spec.py +9 -10
  224. parsl/tests/test_providers/test_cobalt_deprecation_warning.py +2 -0
  225. parsl/tests/test_providers/test_local_provider.py +2 -1
  226. parsl/tests/test_providers/test_pbspro_template.py +1 -1
  227. parsl/tests/test_providers/test_slurm_template.py +1 -1
  228. parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
  229. parsl/tests/test_python_apps/test_context_manager.py +4 -3
  230. parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
  231. parsl/tests/test_python_apps/test_futures.py +2 -1
  232. parsl/tests/test_python_apps/test_join.py +0 -1
  233. parsl/tests/test_python_apps/test_lifted.py +3 -3
  234. parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
  235. parsl/tests/test_python_apps/test_pluggable_future_resolution.py +1 -1
  236. parsl/tests/test_radical/test_mpi_funcs.py +1 -1
  237. parsl/tests/test_regression/test_1480.py +2 -1
  238. parsl/tests/test_regression/test_1653.py +2 -1
  239. parsl/tests/test_regression/test_2652.py +1 -0
  240. parsl/tests/test_regression/test_69a.py +0 -1
  241. parsl/tests/test_regression/test_854.py +4 -2
  242. parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
  243. parsl/tests/test_regression/test_98.py +0 -1
  244. parsl/tests/test_scaling/test_block_error_handler.py +9 -4
  245. parsl/tests/test_scaling/test_scale_down.py +2 -3
  246. parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +4 -5
  247. parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +3 -4
  248. parsl/tests/test_scaling/test_shutdown_scalein.py +1 -2
  249. parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
  250. parsl/tests/test_serialization/test_basic.py +2 -1
  251. parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
  252. parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
  253. parsl/tests/test_serialization/test_proxystore_configured.py +10 -5
  254. parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
  255. parsl/tests/test_shutdown/test_kill_monitoring.py +3 -2
  256. parsl/tests/test_staging/staging_provider.py +2 -2
  257. parsl/tests/test_staging/test_1316.py +3 -2
  258. parsl/tests/test_staging/test_docs_1.py +1 -1
  259. parsl/tests/test_staging/test_docs_2.py +2 -1
  260. parsl/tests/test_staging/test_elaborate_noop_file.py +2 -2
  261. parsl/tests/test_staging/test_staging_https.py +2 -2
  262. parsl/tests/test_staging/test_staging_stdout.py +4 -3
  263. parsl/tests/test_staging/test_zip_in.py +6 -8
  264. parsl/tests/test_staging/test_zip_out.py +7 -9
  265. parsl/tests/test_staging/test_zip_to_zip.py +6 -8
  266. parsl/tests/test_summary.py +2 -1
  267. parsl/tests/test_thread_parallelism.py +0 -1
  268. parsl/tests/test_threads/test_configs.py +1 -1
  269. parsl/tests/test_threads/test_lazy_errors.py +2 -1
  270. parsl/usage_tracking/api.py +2 -3
  271. parsl/usage_tracking/usage.py +8 -18
  272. parsl/utils.py +13 -2
  273. parsl/version.py +1 -1
  274. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py +5 -4
  275. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py +31 -20
  276. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/METADATA +2 -2
  277. parsl-2024.6.3.dist-info/RECORD +471 -0
  278. parsl-2024.5.27.dist-info/RECORD +0 -471
  279. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/parsl_coprocess.py +1 -1
  280. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/LICENSE +0 -0
  281. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/WHEEL +0 -0
  282. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/entry_points.txt +0 -0
  283. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/top_level.txt +0 -0
@@ -1,4 +1,5 @@
1
1
  import os
2
+
2
3
  import pytest
3
4
 
4
5
  import parsl
@@ -1,9 +1,9 @@
1
1
  import copy
2
2
  import os
3
- import pytest
4
-
5
3
  from typing import List
6
4
 
5
+ import pytest
6
+
7
7
  import parsl
8
8
  from parsl.app.app import bash_app
9
9
 
@@ -1,8 +1,8 @@
1
1
  import pytest
2
2
 
3
3
  from parsl.app.app import bash_app
4
- from parsl.data_provider.files import File
5
4
  from parsl.app.futures import DataFuture
5
+ from parsl.data_provider.files import File
6
6
 
7
7
 
8
8
  @bash_app
@@ -1,9 +1,10 @@
1
1
  import logging
2
- import parsl
3
- import pytest
4
2
  import zipfile
5
-
6
3
  from functools import partial
4
+
5
+ import pytest
6
+
7
+ import parsl
7
8
  from parsl.app.futures import DataFuture
8
9
  from parsl.data_provider.files import File
9
10
  from parsl.executors import ThreadPoolExecutor
@@ -5,10 +5,10 @@
5
5
 
6
6
  import importlib
7
7
  import pathlib
8
- import parsl
9
-
10
8
  from functools import partial
11
9
 
10
+ import parsl
11
+
12
12
 
13
13
  @parsl.python_app
14
14
  def app(x):
@@ -1,4 +1,5 @@
1
1
  import os
2
+
2
3
  import pytest
3
4
 
4
5
  import parsl
@@ -1,9 +1,10 @@
1
1
  import contextlib
2
2
  import os
3
+
3
4
  import pytest
5
+
4
6
  import parsl
5
7
  from parsl import python_app
6
-
7
8
  from parsl.tests.configs.local_threads_checkpoint import fresh_config
8
9
 
9
10
 
@@ -2,7 +2,7 @@ import pickle
2
2
 
3
3
  import pytest
4
4
 
5
- from parsl import python_app, DataFlowKernel
5
+ from parsl import DataFlowKernel, python_app
6
6
  from parsl.utils import time_limited_open
7
7
 
8
8
 
@@ -5,8 +5,8 @@ import pytest
5
5
 
6
6
  import parsl
7
7
  from parsl.app.app import python_app
8
- from parsl.utils import time_limited_open
9
8
  from parsl.tests.configs.local_threads_checkpoint_task_exit import config
9
+ from parsl.utils import time_limited_open
10
10
 
11
11
 
12
12
  def local_setup():
@@ -1,8 +1,8 @@
1
+ import os
2
+
1
3
  from parsl.app.app import bash_app, python_app
2
4
  from parsl.data_provider.files import File
3
5
 
4
- import os
5
-
6
6
 
7
7
  @bash_app
8
8
  def echo(message, outputs=[]):
@@ -1,7 +1,7 @@
1
1
  """Functions used to explain kwargs"""
2
2
  from pathlib import Path
3
3
 
4
- from parsl import python_app, File
4
+ from parsl import File, python_app
5
5
 
6
6
 
7
7
  def test_inputs():
@@ -1,12 +1,11 @@
1
1
  import argparse
2
- import parsl
3
2
 
4
3
  import pytest
5
4
 
5
+ import parsl
6
6
  from parsl.app.app import bash_app
7
7
  from parsl.tests.configs.local_threads import config
8
8
 
9
-
10
9
  local_config = config
11
10
 
12
11
 
@@ -1,12 +1,12 @@
1
1
  import os
2
+
2
3
  import pytest
3
- import parsl
4
4
 
5
+ import parsl
5
6
  from parsl.app.app import bash_app, python_app
6
7
  from parsl.data_provider.files import File
7
8
  from parsl.tests.configs.local_threads import config
8
9
 
9
-
10
10
  # parsl.set_stream_logger()
11
11
 
12
12
 
@@ -5,7 +5,6 @@ import pytest
5
5
  from parsl.app.app import python_app
6
6
  from parsl.tests.configs.local_threads import config
7
7
 
8
-
9
8
  local_config = config
10
9
 
11
10
 
@@ -15,8 +15,8 @@ def local_config():
15
15
 
16
16
  @python_app
17
17
  def sleep_fail(sleep_dur, sleep_rand_max, fail_prob, inputs=[]):
18
- import time
19
18
  import random
19
+ import time
20
20
 
21
21
  s = sleep_dur + random.randint(-sleep_rand_max, sleep_rand_max)
22
22
 
@@ -144,8 +144,8 @@ def test_deps(numtasks=10):
144
144
 
145
145
  @python_app
146
146
  def sleep_then_fail(sleep_dur=0.1):
147
- import time
148
147
  import math
148
+ import time
149
149
  time.sleep(sleep_dur)
150
150
  math.ceil("Trigger TypeError")
151
151
  return 0
@@ -1,9 +1,11 @@
1
1
  import parsl
2
2
  from parsl.app.app import python_app
3
- from parsl.executors.errors import UnsupportedFeatureError, ExecutorError
4
3
  from parsl.executors import WorkQueueExecutor
5
- from parsl.executors.high_throughput.mpi_prefix_composer import InvalidResourceSpecification
4
+ from parsl.executors.errors import ExecutorError, UnsupportedFeatureError
6
5
  from parsl.executors.high_throughput.executor import HighThroughputExecutor
6
+ from parsl.executors.high_throughput.mpi_prefix_composer import (
7
+ InvalidResourceSpecification,
8
+ )
7
9
 
8
10
 
9
11
  @python_app
@@ -1,5 +1,6 @@
1
1
  import argparse
2
2
  import os
3
+
3
4
  import pytest
4
5
 
5
6
  import parsl
@@ -15,8 +16,8 @@ def local_config():
15
16
 
16
17
  @python_app
17
18
  def sleep_then_fail(inputs=[], sleep_dur=0.1):
18
- import time
19
19
  import math
20
+ import time
20
21
  time.sleep(sleep_dur)
21
22
  math.ceil("Trigger TypeError")
22
23
  return 0
@@ -1,4 +1,5 @@
1
1
  import os
2
+
2
3
  import pytest
3
4
 
4
5
  import parsl
@@ -1,6 +1,7 @@
1
- import parsl
2
1
  import pytest
3
2
 
3
+ import parsl
4
+
4
5
 
5
6
  @parsl.python_app
6
7
  def always_fails():
@@ -1,8 +1,8 @@
1
1
  import pytest
2
2
 
3
3
  from parsl import python_app
4
- from parsl.tests.configs.htex_local import fresh_config
5
4
  from parsl.serialize.errors import SerializationError
5
+ from parsl.tests.configs.htex_local import fresh_config
6
6
 
7
7
 
8
8
  def local_config():
@@ -1,4 +1,5 @@
1
1
  import logging
2
+
2
3
  import pytest
3
4
 
4
5
  from parsl.process_loggers import wrap_with_logs
parsl/tests/test_flux.py CHANGED
@@ -1,5 +1,5 @@
1
- import os
2
1
  import concurrent.futures as cf
2
+ import os
3
3
 
4
4
  import pytest
5
5
 
@@ -1,11 +1,15 @@
1
- import pytest
2
1
  import threading
3
2
  import time
3
+
4
+ import pytest
4
5
  import zmq
6
+
5
7
  from parsl import curvezmq
8
+ from parsl.executors.high_throughput.errors import (
9
+ CommandClientBadError,
10
+ CommandClientTimeoutError,
11
+ )
6
12
  from parsl.executors.high_throughput.zmq_pipes import CommandClient
7
- from parsl.executors.high_throughput.errors import CommandClientTimeoutError, CommandClientBadError
8
-
9
13
 
10
14
  # Time constant used for timeout tests: various delays and
11
15
  # timeouts will be appropriate multiples of this, but the
@@ -18,10 +22,8 @@ T = 0.25
18
22
  def test_command_not_sent() -> None:
19
23
  """Tests timeout on command send.
20
24
  """
21
- ctx = curvezmq.ClientContext(None)
22
-
23
25
  # RFC6335 ephemeral port range
24
- cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
26
+ cc = CommandClient("127.0.0.1", (49152, 65535))
25
27
 
26
28
  # cc will now wait for a connection, but we won't do anything to make the
27
29
  # other side of the connection exist, so any command given to cc should
@@ -43,10 +45,8 @@ def test_command_ignored() -> None:
43
45
  htex makes multithreaded use of the command client: see issue #3376 about
44
46
  that lack of thread safety.
45
47
  """
46
- ctx = curvezmq.ClientContext(None)
47
-
48
48
  # RFC6335 ephemeral port range
49
- cc = CommandClient(ctx, "127.0.0.1", (49152, 65535))
49
+ cc = CommandClient("127.0.0.1", (49152, 65535))
50
50
 
51
51
  ic_ctx = curvezmq.ServerContext(None)
52
52
  ic_channel = ic_ctx.socket(zmq.REP)
@@ -63,7 +63,4 @@ def test_command_ignored() -> None:
63
63
  cc.run("ANOTHER_COMMAND")
64
64
 
65
65
  cc.close()
66
- ctx.term()
67
-
68
66
  ic_channel.close()
69
- ic_ctx.term()
@@ -1,7 +1,8 @@
1
- import parsl
2
1
  import pytest
3
- from parsl.executors import HighThroughputExecutor
2
+
3
+ import parsl
4
4
  from parsl import Config
5
+ from parsl.executors import HighThroughputExecutor
5
6
  from parsl.providers import LocalProvider
6
7
 
7
8
 
@@ -1,8 +1,10 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import random
4
+
5
+ import pytest
6
+
7
+ import parsl
6
8
  from parsl.tests.configs.htex_local import fresh_config
7
9
 
8
10
  logger = logging.getLogger(__name__)
@@ -18,6 +20,7 @@ def my_affinity():
18
20
 
19
21
  @pytest.mark.local
20
22
  @pytest.mark.multiple_cores_required
23
+ @pytest.mark.skipif('sched_getaffinity' not in dir(os), reason='System does not support sched_setaffinity')
21
24
  def test_cpu_affinity_explicit():
22
25
  available_cores = os.sched_getaffinity(0)
23
26
 
@@ -1,11 +1,13 @@
1
1
  import logging
2
- import parsl
2
+
3
3
  import pytest
4
- from parsl.executors import HighThroughputExecutor
4
+
5
+ import parsl
5
6
  from parsl import Config
6
- from parsl.providers import LocalProvider
7
+ from parsl.executors import HighThroughputExecutor
7
8
  from parsl.executors.errors import BadStateException
8
- from parsl.jobs.states import JobStatus, JobState
9
+ from parsl.jobs.states import JobState, JobStatus
10
+ from parsl.providers import LocalProvider
9
11
 
10
12
 
11
13
  def local_config():
@@ -1,13 +1,13 @@
1
- import parsl
2
- import pytest
3
1
  import time
4
2
 
5
- from parsl.providers import LocalProvider
6
- from parsl.channels import LocalChannel
7
- from parsl.launchers import SimpleLauncher
3
+ import pytest
8
4
 
5
+ import parsl
6
+ from parsl.channels import LocalChannel
9
7
  from parsl.config import Config
10
8
  from parsl.executors import HighThroughputExecutor
9
+ from parsl.launchers import SimpleLauncher
10
+ from parsl.providers import LocalProvider
11
11
 
12
12
  # this constant is used to scale some durations that happen
13
13
  # based around the expected drain period: the drain period
@@ -4,8 +4,7 @@ from unittest import mock
4
4
 
5
5
  import pytest
6
6
 
7
- from parsl import curvezmq
8
- from parsl import HighThroughputExecutor
7
+ from parsl import HighThroughputExecutor, curvezmq
9
8
  from parsl.multiprocessing import ForkProcess
10
9
 
11
10
  _MOCK_BASE = "parsl.executors.high_throughput.executor"
@@ -1,7 +1,8 @@
1
- import pytest
2
1
  import logging
3
-
4
2
  import sys
3
+
4
+ import pytest
5
+
5
6
  import parsl
6
7
  from parsl.app.app import python_app
7
8
  from parsl.tests.configs.htex_local import fresh_config
@@ -1,11 +1,13 @@
1
1
  import logging
2
- import parsl
2
+
3
3
  import pytest
4
- from parsl.executors import HighThroughputExecutor
4
+
5
+ import parsl
5
6
  from parsl import Config
6
- from parsl.providers import LocalProvider
7
+ from parsl.executors import HighThroughputExecutor
7
8
  from parsl.executors.errors import BadStateException
8
- from parsl.jobs.states import JobStatus, JobState
9
+ from parsl.jobs.states import JobState, JobStatus
10
+ from parsl.providers import LocalProvider
9
11
 
10
12
 
11
13
  def local_config():
@@ -2,10 +2,11 @@
2
2
  """
3
3
 
4
4
  import os
5
- import parsl
6
- import pytest
7
5
  import time
8
6
 
7
+ import pytest
8
+
9
+ import parsl
9
10
  from parsl.tests.configs.htex_local_alternate import fresh_config
10
11
 
11
12
 
@@ -1,12 +1,12 @@
1
1
  import os
2
- import parsl
3
- import pytest
4
2
  import time
5
3
 
4
+ import pytest
5
+
6
+ import parsl
6
7
  from parsl import HighThroughputExecutor
7
8
  from parsl.config import Config
8
- from parsl.executors.taskvine import TaskVineExecutor
9
- from parsl.executors.taskvine import TaskVineManagerConfig
9
+ from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
10
10
  from parsl.monitoring import MonitoringHub
11
11
 
12
12
 
@@ -1,9 +1,11 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import time
6
4
 
5
+ import pytest
6
+
7
+ import parsl
8
+
7
9
  logger = logging.getLogger(__name__)
8
10
 
9
11
 
@@ -14,9 +16,10 @@ def this_app():
14
16
 
15
17
  @pytest.mark.local
16
18
  def test_row_counts():
17
- from parsl.tests.configs.htex_local_alternate import fresh_config
18
19
  import sqlalchemy
19
20
  from sqlalchemy import text
21
+
22
+ from parsl.tests.configs.htex_local_alternate import fresh_config
20
23
  if os.path.exists("runinfo/monitoring.db"):
21
24
  logger.info("Monitoring database already exists - deleting")
22
25
  os.remove("runinfo/monitoring.db")
@@ -1,11 +1,13 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import socket
6
4
  import time
5
+
6
+ import pytest
7
7
  import zmq
8
8
 
9
+ import parsl
10
+
9
11
  logger = logging.getLogger(__name__)
10
12
 
11
13
 
@@ -16,10 +18,11 @@ def this_app():
16
18
 
17
19
  @pytest.mark.local
18
20
  def test_row_counts():
19
- from parsl.tests.configs.htex_local_alternate import fresh_config
20
21
  import sqlalchemy
21
22
  from sqlalchemy import text
22
23
 
24
+ from parsl.tests.configs.htex_local_alternate import fresh_config
25
+
23
26
  if os.path.exists("runinfo/monitoring.db"):
24
27
  logger.info("Monitoring database already exists - deleting")
25
28
  os.remove("runinfo/monitoring.db")
@@ -1,16 +1,16 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import time
6
4
 
7
- from parsl.providers import LocalProvider
8
- from parsl.channels import LocalChannel
9
- from parsl.launchers import SimpleLauncher
5
+ import pytest
10
6
 
7
+ import parsl
8
+ from parsl.channels import LocalChannel
11
9
  from parsl.config import Config
12
10
  from parsl.executors import HighThroughputExecutor
11
+ from parsl.launchers import SimpleLauncher
13
12
  from parsl.monitoring import MonitoringHub
13
+ from parsl.providers import LocalProvider
14
14
 
15
15
 
16
16
  def fresh_config(run_dir, strategy, db_url):
@@ -1,11 +1,12 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import random
6
-
7
4
  from concurrent.futures import Future
8
5
 
6
+ import pytest
7
+
8
+ import parsl
9
+
9
10
 
10
11
  @parsl.python_app
11
12
  def this_app(inputs=()):
@@ -16,6 +17,7 @@ def this_app(inputs=()):
16
17
  def test_future_representation(tmpd_cwd):
17
18
  import sqlalchemy
18
19
  from sqlalchemy import text
20
+
19
21
  from parsl.tests.configs.htex_local_alternate import fresh_config
20
22
 
21
23
  monitoring_db = str(tmpd_cwd / "monitoring.db")
@@ -1,9 +1,11 @@
1
1
 
2
2
  import logging
3
3
  import os
4
- import parsl
4
+
5
5
  import pytest
6
6
 
7
+ import parsl
8
+
7
9
  logger = logging.getLogger(__name__)
8
10
 
9
11
 
@@ -16,6 +18,7 @@ def this_app(x):
16
18
  def test_hashsum():
17
19
  import sqlalchemy
18
20
  from sqlalchemy import text
21
+
19
22
  from parsl.tests.configs.htex_local_alternate import fresh_config
20
23
 
21
24
  if os.path.exists("runinfo/monitoring.db"):
@@ -3,16 +3,16 @@
3
3
 
4
4
  import logging
5
5
  import os
6
- import parsl
7
- import pytest
8
6
  import re
9
7
  import time
10
-
11
8
  from typing import Union
12
9
 
10
+ import pytest
11
+
12
+ import parsl
13
13
  from parsl.config import Config
14
- from parsl.data_provider.files import File
15
14
  from parsl.data_provider.data_manager import default_staging
15
+ from parsl.data_provider.files import File
16
16
  from parsl.data_provider.staging import Staging
17
17
  from parsl.executors import HighThroughputExecutor
18
18
  from parsl.monitoring import MonitoringHub
@@ -1,4 +1,5 @@
1
1
  import pytest
2
+
2
3
  from parsl.dataflow.states import States
3
4
 
4
5
 
@@ -2,7 +2,7 @@ import pytest
2
2
 
3
3
  from parsl import Config
4
4
  from parsl.executors import HighThroughputExecutor
5
- from parsl.launchers import SrunLauncher, AprunLauncher, SimpleLauncher
5
+ from parsl.launchers import AprunLauncher, SimpleLauncher, SrunLauncher
6
6
  from parsl.providers import SlurmProvider
7
7
 
8
8
 
@@ -1,5 +1,7 @@
1
1
  from typing import Dict
2
+
2
3
  import pytest
4
+
3
5
  import parsl
4
6
  from parsl import python_app
5
7
  from parsl.tests.configs.htex_local import fresh_config
@@ -1,15 +1,17 @@
1
1
  import logging
2
+ import os
2
3
  import random
3
4
  from typing import Dict
5
+
4
6
  import pytest
7
+
5
8
  import parsl
6
- from parsl import python_app, bash_app
9
+ from parsl import bash_app, python_app
10
+ from parsl.executors.high_throughput.mpi_prefix_composer import (
11
+ MissingResourceSpecification,
12
+ )
7
13
  from parsl.tests.configs.htex_local import fresh_config
8
14
 
9
- from parsl.executors.high_throughput.mpi_prefix_composer import MissingResourceSpecification
10
-
11
- import os
12
-
13
15
  EXECUTOR_LABEL = "MPI_TEST"
14
16
 
15
17
 
@@ -1,14 +1,14 @@
1
1
  import logging
2
+
2
3
  import pytest
3
4
 
4
- from parsl.executors.high_throughput.mpi_resource_management import Scheduler
5
5
  from parsl.executors.high_throughput.mpi_prefix_composer import (
6
- compose_srun_launch_cmd,
6
+ compose_all,
7
7
  compose_aprun_launch_cmd,
8
8
  compose_mpiexec_launch_cmd,
9
- compose_all,
9
+ compose_srun_launch_cmd,
10
10
  )
11
-
11
+ from parsl.executors.high_throughput.mpi_resource_management import Scheduler
12
12
 
13
13
  resource_spec = {"num_nodes": 2,
14
14
  "num_ranks": 8,