parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. parsl/__init__.py +9 -10
  2. parsl/addresses.py +6 -4
  3. parsl/app/app.py +3 -6
  4. parsl/app/bash.py +4 -4
  5. parsl/app/errors.py +5 -3
  6. parsl/app/futures.py +3 -3
  7. parsl/app/python.py +2 -1
  8. parsl/benchmark/perf.py +2 -1
  9. parsl/channels/__init__.py +2 -2
  10. parsl/channels/base.py +0 -1
  11. parsl/channels/errors.py +2 -1
  12. parsl/channels/oauth_ssh/oauth_ssh.py +4 -3
  13. parsl/channels/ssh/ssh.py +9 -1
  14. parsl/channels/ssh_il/ssh_il.py +1 -0
  15. parsl/concurrent/__init__.py +2 -2
  16. parsl/config.py +5 -5
  17. parsl/configs/ASPIRE1.py +3 -3
  18. parsl/configs/Azure.py +6 -7
  19. parsl/configs/ad_hoc.py +4 -3
  20. parsl/configs/bridges.py +3 -3
  21. parsl/configs/cc_in2p3.py +2 -2
  22. parsl/configs/ec2.py +1 -1
  23. parsl/configs/expanse.py +1 -2
  24. parsl/configs/frontera.py +2 -3
  25. parsl/configs/htex_local.py +1 -2
  26. parsl/configs/illinoiscluster.py +1 -1
  27. parsl/configs/kubernetes.py +1 -2
  28. parsl/configs/midway.py +3 -3
  29. parsl/configs/osg.py +1 -1
  30. parsl/configs/polaris.py +1 -1
  31. parsl/configs/stampede2.py +4 -5
  32. parsl/configs/summit.py +1 -3
  33. parsl/configs/toss3_llnl.py +1 -2
  34. parsl/configs/vineex_local.py +3 -3
  35. parsl/configs/wqex_local.py +2 -2
  36. parsl/data_provider/data_manager.py +3 -3
  37. parsl/data_provider/file_noop.py +1 -2
  38. parsl/data_provider/files.py +3 -3
  39. parsl/data_provider/ftp.py +1 -3
  40. parsl/data_provider/globus.py +7 -6
  41. parsl/data_provider/http.py +2 -2
  42. parsl/data_provider/rsync.py +1 -1
  43. parsl/data_provider/staging.py +2 -2
  44. parsl/data_provider/zip.py +4 -5
  45. parsl/dataflow/dflow.py +21 -16
  46. parsl/dataflow/errors.py +2 -1
  47. parsl/dataflow/futures.py +1 -2
  48. parsl/dataflow/memoization.py +5 -5
  49. parsl/dataflow/rundirs.py +1 -1
  50. parsl/dataflow/taskrecord.py +4 -5
  51. parsl/executors/__init__.py +3 -3
  52. parsl/executors/base.py +1 -0
  53. parsl/executors/flux/execute_parsl_task.py +2 -2
  54. parsl/executors/flux/executor.py +11 -12
  55. parsl/executors/flux/flux_instance_manager.py +3 -3
  56. parsl/executors/high_throughput/executor.py +29 -35
  57. parsl/executors/high_throughput/interchange.py +16 -18
  58. parsl/executors/high_throughput/manager_record.py +1 -0
  59. parsl/executors/high_throughput/monitoring_info.py +2 -1
  60. parsl/executors/high_throughput/mpi_executor.py +5 -2
  61. parsl/executors/high_throughput/mpi_prefix_composer.py +1 -1
  62. parsl/executors/high_throughput/mpi_resource_management.py +1 -2
  63. parsl/executors/high_throughput/probe.py +6 -4
  64. parsl/executors/high_throughput/process_worker_pool.py +31 -20
  65. parsl/executors/high_throughput/zmq_pipes.py +28 -14
  66. parsl/executors/radical/executor.py +15 -15
  67. parsl/executors/radical/rpex_master.py +1 -2
  68. parsl/executors/radical/rpex_resources.py +1 -2
  69. parsl/executors/radical/rpex_worker.py +2 -1
  70. parsl/executors/status_handling.py +5 -4
  71. parsl/executors/taskvine/__init__.py +1 -1
  72. parsl/executors/taskvine/errors.py +1 -1
  73. parsl/executors/taskvine/exec_parsl_function.py +2 -2
  74. parsl/executors/taskvine/executor.py +23 -24
  75. parsl/executors/taskvine/factory.py +1 -1
  76. parsl/executors/taskvine/manager.py +11 -13
  77. parsl/executors/threads.py +4 -5
  78. parsl/executors/workqueue/errors.py +1 -1
  79. parsl/executors/workqueue/exec_parsl_function.py +5 -4
  80. parsl/executors/workqueue/executor.py +26 -27
  81. parsl/executors/workqueue/parsl_coprocess.py +1 -1
  82. parsl/jobs/error_handlers.py +1 -1
  83. parsl/jobs/job_status_poller.py +2 -5
  84. parsl/jobs/states.py +1 -1
  85. parsl/jobs/strategy.py +2 -2
  86. parsl/launchers/__init__.py +12 -3
  87. parsl/launchers/errors.py +1 -1
  88. parsl/log_utils.py +1 -2
  89. parsl/monitoring/db_manager.py +16 -10
  90. parsl/monitoring/monitoring.py +11 -15
  91. parsl/monitoring/queries/pandas.py +1 -2
  92. parsl/monitoring/radios.py +2 -4
  93. parsl/monitoring/remote.py +13 -8
  94. parsl/monitoring/router.py +8 -11
  95. parsl/monitoring/types.py +2 -0
  96. parsl/monitoring/visualization/app.py +4 -2
  97. parsl/monitoring/visualization/models.py +0 -1
  98. parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
  99. parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
  100. parsl/monitoring/visualization/utils.py +0 -1
  101. parsl/monitoring/visualization/views.py +16 -9
  102. parsl/multiprocessing.py +0 -1
  103. parsl/process_loggers.py +1 -2
  104. parsl/providers/__init__.py +9 -12
  105. parsl/providers/ad_hoc/ad_hoc.py +1 -1
  106. parsl/providers/aws/aws.py +2 -3
  107. parsl/providers/azure/azure.py +4 -5
  108. parsl/providers/base.py +1 -1
  109. parsl/providers/cluster_provider.py +1 -1
  110. parsl/providers/cobalt/cobalt.py +3 -3
  111. parsl/providers/condor/condor.py +4 -2
  112. parsl/providers/errors.py +2 -2
  113. parsl/providers/googlecloud/googlecloud.py +2 -1
  114. parsl/providers/grid_engine/grid_engine.py +2 -2
  115. parsl/providers/kubernetes/kube.py +5 -3
  116. parsl/providers/local/local.py +5 -1
  117. parsl/providers/lsf/lsf.py +2 -2
  118. parsl/providers/pbspro/pbspro.py +1 -1
  119. parsl/providers/slurm/slurm.py +5 -5
  120. parsl/providers/torque/torque.py +1 -1
  121. parsl/serialize/__init__.py +8 -3
  122. parsl/serialize/base.py +1 -2
  123. parsl/serialize/concretes.py +5 -4
  124. parsl/serialize/proxystore.py +3 -2
  125. parsl/tests/__init__.py +1 -1
  126. parsl/tests/configs/ad_hoc_cluster_htex.py +4 -4
  127. parsl/tests/configs/azure_single_node.py +4 -5
  128. parsl/tests/configs/bridges.py +3 -2
  129. parsl/tests/configs/cc_in2p3.py +2 -2
  130. parsl/tests/configs/comet.py +2 -1
  131. parsl/tests/configs/ec2_single_node.py +1 -2
  132. parsl/tests/configs/ec2_spot.py +1 -2
  133. parsl/tests/configs/frontera.py +3 -2
  134. parsl/tests/configs/htex_ad_hoc_cluster.py +2 -4
  135. parsl/tests/configs/htex_local.py +2 -3
  136. parsl/tests/configs/htex_local_alternate.py +8 -11
  137. parsl/tests/configs/htex_local_intask_staging.py +5 -7
  138. parsl/tests/configs/htex_local_rsync_staging.py +4 -6
  139. parsl/tests/configs/local_adhoc.py +1 -1
  140. parsl/tests/configs/local_radical.py +1 -3
  141. parsl/tests/configs/local_radical_mpi.py +2 -2
  142. parsl/tests/configs/midway.py +2 -2
  143. parsl/tests/configs/nscc_singapore.py +3 -3
  144. parsl/tests/configs/osg_htex.py +1 -1
  145. parsl/tests/configs/petrelkube.py +3 -2
  146. parsl/tests/configs/summit.py +1 -0
  147. parsl/tests/configs/swan_htex.py +2 -2
  148. parsl/tests/configs/taskvine_ex.py +3 -5
  149. parsl/tests/configs/theta.py +2 -2
  150. parsl/tests/configs/workqueue_ex.py +3 -4
  151. parsl/tests/conftest.py +4 -4
  152. parsl/tests/integration/test_channels/test_ssh_errors.py +1 -1
  153. parsl/tests/integration/test_stress/test_python_simple.py +3 -4
  154. parsl/tests/integration/test_stress/test_python_threads.py +3 -5
  155. parsl/tests/manual_tests/htex_local.py +4 -4
  156. parsl/tests/manual_tests/test_ad_hoc_htex.py +2 -1
  157. parsl/tests/manual_tests/test_basic.py +1 -0
  158. parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +4 -4
  159. parsl/tests/manual_tests/test_log_filter.py +3 -1
  160. parsl/tests/manual_tests/test_memory_limits.py +6 -6
  161. parsl/tests/manual_tests/test_regression_220.py +2 -1
  162. parsl/tests/manual_tests/test_udp_simple.py +4 -3
  163. parsl/tests/manual_tests/test_worker_count.py +3 -2
  164. parsl/tests/scaling_tests/htex_local.py +2 -2
  165. parsl/tests/scaling_tests/test_scale.py +0 -9
  166. parsl/tests/scaling_tests/vineex_condor.py +1 -2
  167. parsl/tests/scaling_tests/vineex_local.py +1 -2
  168. parsl/tests/site_tests/test_provider.py +3 -1
  169. parsl/tests/site_tests/test_site.py +2 -0
  170. parsl/tests/sites/test_affinity.py +7 -5
  171. parsl/tests/sites/test_dynamic_executor.py +3 -3
  172. parsl/tests/sites/test_ec2.py +3 -2
  173. parsl/tests/sites/test_local_adhoc.py +2 -1
  174. parsl/tests/sites/test_worker_info.py +4 -3
  175. parsl/tests/test_aalst_patterns.py +0 -1
  176. parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
  177. parsl/tests/test_bash_apps/test_error_codes.py +1 -4
  178. parsl/tests/test_bash_apps/test_memoize_ignore_args.py +1 -0
  179. parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +2 -2
  180. parsl/tests/test_bash_apps/test_pipeline.py +1 -1
  181. parsl/tests/test_bash_apps/test_std_uri.py +4 -3
  182. parsl/tests/test_callables.py +2 -2
  183. parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
  184. parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -1
  185. parsl/tests/test_checkpointing/test_regression_239.py +1 -1
  186. parsl/tests/test_checkpointing/test_task_exit.py +1 -1
  187. parsl/tests/test_docs/test_from_slides.py +2 -2
  188. parsl/tests/test_docs/test_kwargs.py +1 -1
  189. parsl/tests/test_docs/test_tutorial_1.py +1 -2
  190. parsl/tests/test_docs/test_workflow1.py +2 -2
  191. parsl/tests/test_docs/test_workflow2.py +0 -1
  192. parsl/tests/test_error_handling/test_rand_fail.py +2 -2
  193. parsl/tests/test_error_handling/test_resource_spec.py +4 -2
  194. parsl/tests/test_error_handling/test_retries.py +2 -1
  195. parsl/tests/test_error_handling/test_retry_handler.py +1 -0
  196. parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
  197. parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
  198. parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
  199. parsl/tests/test_flux.py +1 -1
  200. parsl/tests/test_htex/test_command_client_timeout.py +9 -12
  201. parsl/tests/test_htex/test_connected_blocks.py +3 -2
  202. parsl/tests/test_htex/test_cpu_affinity_explicit.py +5 -2
  203. parsl/tests/test_htex/test_disconnected_blocks.py +6 -4
  204. parsl/tests/test_htex/test_drain.py +5 -5
  205. parsl/tests/test_htex/test_htex.py +1 -2
  206. parsl/tests/test_htex/test_managers_command.py +3 -2
  207. parsl/tests/test_htex/test_multiple_disconnected_blocks.py +6 -4
  208. parsl/tests/test_monitoring/test_app_names.py +3 -2
  209. parsl/tests/test_monitoring/test_basic.py +4 -4
  210. parsl/tests/test_monitoring/test_db_locks.py +6 -3
  211. parsl/tests/test_monitoring/test_fuzz_zmq.py +6 -3
  212. parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +5 -5
  213. parsl/tests/test_monitoring/test_incomplete_futures.py +5 -3
  214. parsl/tests/test_monitoring/test_memoization_representation.py +4 -1
  215. parsl/tests/test_monitoring/test_stdouterr.py +4 -4
  216. parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
  217. parsl/tests/test_mpi_apps/test_bad_mpi_config.py +1 -1
  218. parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +2 -0
  219. parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +7 -5
  220. parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
  221. parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
  222. parsl/tests/test_mpi_apps/test_mpiex.py +4 -3
  223. parsl/tests/test_mpi_apps/test_resource_spec.py +9 -10
  224. parsl/tests/test_providers/test_cobalt_deprecation_warning.py +2 -0
  225. parsl/tests/test_providers/test_local_provider.py +2 -1
  226. parsl/tests/test_providers/test_pbspro_template.py +1 -1
  227. parsl/tests/test_providers/test_slurm_template.py +1 -1
  228. parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
  229. parsl/tests/test_python_apps/test_context_manager.py +4 -3
  230. parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
  231. parsl/tests/test_python_apps/test_futures.py +2 -1
  232. parsl/tests/test_python_apps/test_join.py +0 -1
  233. parsl/tests/test_python_apps/test_lifted.py +3 -3
  234. parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
  235. parsl/tests/test_python_apps/test_pluggable_future_resolution.py +1 -1
  236. parsl/tests/test_radical/test_mpi_funcs.py +1 -1
  237. parsl/tests/test_regression/test_1480.py +2 -1
  238. parsl/tests/test_regression/test_1653.py +2 -1
  239. parsl/tests/test_regression/test_2652.py +1 -0
  240. parsl/tests/test_regression/test_69a.py +0 -1
  241. parsl/tests/test_regression/test_854.py +4 -2
  242. parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
  243. parsl/tests/test_regression/test_98.py +0 -1
  244. parsl/tests/test_scaling/test_block_error_handler.py +9 -4
  245. parsl/tests/test_scaling/test_scale_down.py +2 -3
  246. parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +4 -5
  247. parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +3 -4
  248. parsl/tests/test_scaling/test_shutdown_scalein.py +1 -2
  249. parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
  250. parsl/tests/test_serialization/test_basic.py +2 -1
  251. parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
  252. parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
  253. parsl/tests/test_serialization/test_proxystore_configured.py +10 -5
  254. parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
  255. parsl/tests/test_shutdown/test_kill_monitoring.py +3 -2
  256. parsl/tests/test_staging/staging_provider.py +2 -2
  257. parsl/tests/test_staging/test_1316.py +3 -2
  258. parsl/tests/test_staging/test_docs_1.py +1 -1
  259. parsl/tests/test_staging/test_docs_2.py +2 -1
  260. parsl/tests/test_staging/test_elaborate_noop_file.py +2 -2
  261. parsl/tests/test_staging/test_staging_https.py +2 -2
  262. parsl/tests/test_staging/test_staging_stdout.py +4 -3
  263. parsl/tests/test_staging/test_zip_in.py +6 -8
  264. parsl/tests/test_staging/test_zip_out.py +7 -9
  265. parsl/tests/test_staging/test_zip_to_zip.py +6 -8
  266. parsl/tests/test_summary.py +2 -1
  267. parsl/tests/test_thread_parallelism.py +0 -1
  268. parsl/tests/test_threads/test_configs.py +1 -1
  269. parsl/tests/test_threads/test_lazy_errors.py +2 -1
  270. parsl/usage_tracking/api.py +2 -3
  271. parsl/usage_tracking/usage.py +8 -18
  272. parsl/utils.py +13 -2
  273. parsl/version.py +1 -1
  274. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py +5 -4
  275. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py +31 -20
  276. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/METADATA +2 -2
  277. parsl-2024.6.3.dist-info/RECORD +471 -0
  278. parsl-2024.5.27.dist-info/RECORD +0 -471
  279. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/parsl_coprocess.py +1 -1
  280. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/LICENSE +0 -0
  281. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/WHEEL +0 -0
  282. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/entry_points.txt +0 -0
  283. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/top_level.txt +0 -0
@@ -1,10 +1,11 @@
1
- from parsl.config import Config
2
1
  from parsl.channels import LocalChannel
3
- from parsl.providers import SlurmProvider
2
+ from parsl.config import Config
4
3
  from parsl.executors import HighThroughputExecutor
5
4
  from parsl.launchers import SrunLauncher
5
+ from parsl.providers import SlurmProvider
6
6
 
7
7
  from .user_opts import user_opts
8
+
8
9
  """ This config assumes that it is used to launch parsl tasks from the login nodes
9
10
  of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
10
11
  """
@@ -1,9 +1,7 @@
1
- from parsl.providers import AdHocProvider
2
1
  from parsl.channels import SSHChannel
3
- from parsl.executors import HighThroughputExecutor
4
-
5
2
  from parsl.config import Config
6
-
3
+ from parsl.executors import HighThroughputExecutor
4
+ from parsl.providers import AdHocProvider
7
5
  from parsl.tests.configs.user_opts import user_opts
8
6
 
9
7
  config = Config(
@@ -1,9 +1,8 @@
1
- from parsl.providers import LocalProvider
2
1
  from parsl.channels import LocalChannel
3
- from parsl.launchers import SimpleLauncher
4
-
5
2
  from parsl.config import Config
6
3
  from parsl.executors import HighThroughputExecutor
4
+ from parsl.launchers import SimpleLauncher
5
+ from parsl.providers import LocalProvider
7
6
 
8
7
 
9
8
  def fresh_config():
@@ -15,23 +15,20 @@ will cause substantially different behaviour on whatever
15
15
  those timing parameters control.
16
16
  """
17
17
 
18
- # imports for monitoring:
19
- from parsl.monitoring import MonitoringHub
20
-
21
18
  import os
22
19
 
23
- from parsl.providers import LocalProvider
24
20
  from parsl.channels import LocalChannel
25
- from parsl.launchers import SingleNodeLauncher
26
-
27
21
  from parsl.config import Config
28
- from parsl.executors import HighThroughputExecutor
29
-
30
-
31
- from parsl.data_provider.http import HTTPInTaskStaging
32
- from parsl.data_provider.ftp import FTPInTaskStaging
33
22
  from parsl.data_provider.file_noop import NoOpFileStaging
23
+ from parsl.data_provider.ftp import FTPInTaskStaging
24
+ from parsl.data_provider.http import HTTPInTaskStaging
34
25
  from parsl.data_provider.zip import ZipFileStaging
26
+ from parsl.executors import HighThroughputExecutor
27
+ from parsl.launchers import SingleNodeLauncher
28
+
29
+ # imports for monitoring:
30
+ from parsl.monitoring import MonitoringHub
31
+ from parsl.providers import LocalProvider
35
32
 
36
33
  working_dir = os.getcwd() + "/" + "test_htex_alternate"
37
34
 
@@ -1,13 +1,11 @@
1
- from parsl.providers import LocalProvider
2
1
  from parsl.channels import LocalChannel
3
- from parsl.launchers import SimpleLauncher
4
-
5
- from parsl.data_provider.http import HTTPInTaskStaging
6
- from parsl.data_provider.ftp import FTPInTaskStaging
7
- from parsl.data_provider.file_noop import NoOpFileStaging
8
-
9
2
  from parsl.config import Config
3
+ from parsl.data_provider.file_noop import NoOpFileStaging
4
+ from parsl.data_provider.ftp import FTPInTaskStaging
5
+ from parsl.data_provider.http import HTTPInTaskStaging
10
6
  from parsl.executors import HighThroughputExecutor
7
+ from parsl.launchers import SimpleLauncher
8
+ from parsl.providers import LocalProvider
11
9
 
12
10
  config = Config(
13
11
  executors=[
@@ -1,13 +1,11 @@
1
- from parsl.providers import LocalProvider
2
1
  from parsl.channels import LocalChannel
3
- from parsl.launchers import SimpleLauncher
4
-
5
- from parsl.data_provider.http import HTTPInTaskStaging
2
+ from parsl.config import Config
6
3
  from parsl.data_provider.ftp import FTPInTaskStaging
4
+ from parsl.data_provider.http import HTTPInTaskStaging
7
5
  from parsl.data_provider.rsync import RSyncStaging
8
-
9
- from parsl.config import Config
10
6
  from parsl.executors import HighThroughputExecutor
7
+ from parsl.launchers import SimpleLauncher
8
+ from parsl.providers import LocalProvider
11
9
 
12
10
  config = Config(
13
11
  executors=[
@@ -1,5 +1,5 @@
1
- from parsl.config import Config
2
1
  from parsl.channels import LocalChannel
2
+ from parsl.config import Config
3
3
  from parsl.executors import HighThroughputExecutor
4
4
  from parsl.providers import AdHocProvider
5
5
 
@@ -1,9 +1,7 @@
1
1
  import os
2
2
 
3
3
  from parsl.config import Config
4
- from parsl.executors.radical import RadicalPilotExecutor
5
- from parsl.executors.radical import ResourceConfig
6
-
4
+ from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
7
5
 
8
6
  rpex_cfg = ResourceConfig()
9
7
 
@@ -1,10 +1,10 @@
1
1
  import os
2
+
2
3
  from parsl.config import Config
3
4
 
4
5
 
5
6
  def fresh_config():
6
- from parsl.executors.radical import ResourceConfig
7
- from parsl.executors.radical import RadicalPilotExecutor
7
+ from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
8
8
 
9
9
  rpex_cfg = ResourceConfig()
10
10
  rpex_cfg.worker_type = "MPI"
@@ -1,7 +1,7 @@
1
1
  from parsl.config import Config
2
- from parsl.providers import SlurmProvider
3
- from parsl.launchers import SrunLauncher
4
2
  from parsl.executors import HighThroughputExecutor
3
+ from parsl.launchers import SrunLauncher
4
+ from parsl.providers import SlurmProvider
5
5
 
6
6
  from .user_opts import user_opts
7
7
 
@@ -1,8 +1,8 @@
1
- from parsl.providers import PBSProProvider
2
- from parsl.executors import HighThroughputExecutor
3
- from parsl.launchers import MpiRunLauncher
4
1
  from parsl.addresses import address_by_interface
5
2
  from parsl.config import Config
3
+ from parsl.executors import HighThroughputExecutor
4
+ from parsl.launchers import MpiRunLauncher
5
+ from parsl.providers import PBSProProvider
6
6
 
7
7
  from .user_opts import user_opts
8
8
 
@@ -1,6 +1,6 @@
1
1
  from parsl.config import Config
2
- from parsl.providers import CondorProvider
3
2
  from parsl.executors import HighThroughputExecutor
3
+ from parsl.providers import CondorProvider
4
4
 
5
5
  # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
6
6
  # If you are a user copying-and-pasting this as an example, make sure to either
@@ -1,10 +1,11 @@
1
+ import os
2
+
3
+ from parsl.addresses import address_by_route
1
4
  from parsl.config import Config
2
5
  from parsl.executors import HighThroughputExecutor
3
6
  from parsl.providers import KubernetesProvider
4
- from parsl.addresses import address_by_route
5
7
 
6
8
  from .user_opts import user_opts
7
- import os
8
9
 
9
10
 
10
11
  def fresh_config():
@@ -4,6 +4,7 @@ from parsl.launchers import JsrunLauncher
4
4
  from parsl.providers import LSFProvider
5
5
 
6
6
  from .user_opts import user_opts
7
+
7
8
  """ This config assumes that it is used to launch parsl tasks from the login nodes
8
9
  of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
9
10
  """
@@ -8,10 +8,10 @@
8
8
  ==================
9
9
  """
10
10
  from parsl.channels import SSHChannel
11
- from parsl.launchers import AprunLauncher
12
- from parsl.providers import TorqueProvider
13
11
  from parsl.config import Config
14
12
  from parsl.executors import HighThroughputExecutor
13
+ from parsl.launchers import AprunLauncher
14
+ from parsl.providers import TorqueProvider
15
15
 
16
16
  # If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
17
17
  # If you are a user copying-and-pasting this as an example, make sure to either
@@ -1,10 +1,8 @@
1
1
  from parsl.config import Config
2
- from parsl.executors.taskvine import TaskVineExecutor
3
- from parsl.executors.taskvine import TaskVineManagerConfig
4
-
5
- from parsl.data_provider.http import HTTPInTaskStaging
6
- from parsl.data_provider.ftp import FTPInTaskStaging
7
2
  from parsl.data_provider.file_noop import NoOpFileStaging
3
+ from parsl.data_provider.ftp import FTPInTaskStaging
4
+ from parsl.data_provider.http import HTTPInTaskStaging
5
+ from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
8
6
 
9
7
 
10
8
  def fresh_config():
@@ -1,7 +1,7 @@
1
1
  from parsl.config import Config
2
- from parsl.providers import CobaltProvider
3
- from parsl.launchers import AprunLauncher
4
2
  from parsl.executors import HighThroughputExecutor
3
+ from parsl.launchers import AprunLauncher
4
+ from parsl.providers import CobaltProvider
5
5
 
6
6
  from .user_opts import user_opts
7
7
 
@@ -1,9 +1,8 @@
1
1
  from parsl.config import Config
2
- from parsl.executors import WorkQueueExecutor
3
-
4
- from parsl.data_provider.http import HTTPInTaskStaging
5
- from parsl.data_provider.ftp import FTPInTaskStaging
6
2
  from parsl.data_provider.file_noop import NoOpFileStaging
3
+ from parsl.data_provider.ftp import FTPInTaskStaging
4
+ from parsl.data_provider.http import HTTPInTaskStaging
5
+ from parsl.executors import WorkQueueExecutor
7
6
 
8
7
 
9
8
  def fresh_config():
parsl/tests/conftest.py CHANGED
@@ -6,21 +6,21 @@ import pathlib
6
6
  import random
7
7
  import re
8
8
  import shutil
9
- import string
10
- import time
11
- import types
12
9
  import signal
10
+ import string
13
11
  import sys
14
12
  import tempfile
15
13
  import threading
14
+ import time
16
15
  import traceback
16
+ import types
17
17
  import typing as t
18
18
  from datetime import datetime
19
19
  from glob import glob
20
20
  from itertools import chain
21
21
 
22
- import pytest
23
22
  import _pytest.runner as runner
23
+ import pytest
24
24
 
25
25
  import parsl
26
26
  from parsl.dataflow.dflow import DataFlowKernelLoader
@@ -1,4 +1,4 @@
1
- from parsl.channels.errors import SSHException, BadHostKeyException
1
+ from parsl.channels.errors import BadHostKeyException, SSHException
2
2
  from parsl.channels.ssh.ssh import SSHChannel as SSH
3
3
 
4
4
 
@@ -1,10 +1,9 @@
1
+ import argparse
2
+ import time
3
+
1
4
  import parsl
2
5
  from parsl import python_app
3
6
  from parsl.configs.htex_local import config
4
- # from parsl.configs.local_threads import config
5
-
6
- import time
7
- import argparse
8
7
 
9
8
 
10
9
  @python_app
@@ -1,14 +1,12 @@
1
1
  ''' Testing bash apps
2
2
  '''
3
- import parsl
4
- from parsl import python_app
5
-
6
- import time
7
3
  import argparse
4
+ import time
8
5
 
6
+ import parsl
7
+ from parsl import python_app
9
8
  from parsl.tests.configs.local_threads import config
10
9
 
11
-
12
10
  local_config = config
13
11
 
14
12
 
@@ -1,11 +1,11 @@
1
- from parsl.providers import LocalProvider
2
1
  from parsl.channels import LocalChannel
3
- # from parsl.launchers import SimpleLauncher
4
- from parsl.launchers import SingleNodeLauncher
5
-
6
2
  from parsl.config import Config
7
3
  from parsl.executors import HighThroughputExecutor
8
4
 
5
+ # from parsl.launchers import SimpleLauncher
6
+ from parsl.launchers import SingleNodeLauncher
7
+ from parsl.providers import LocalProvider
8
+
9
9
  config = Config(
10
10
  executors=[
11
11
  HighThroughputExecutor(
@@ -1,11 +1,12 @@
1
1
  import parsl
2
2
  from parsl import python_app
3
+
3
4
  parsl.set_stream_logger()
4
5
 
5
- from parsl.providers import AdHocProvider
6
6
  from parsl.channels import SSHChannel
7
7
  from parsl.config import Config
8
8
  from parsl.executors import HighThroughputExecutor
9
+ from parsl.providers import AdHocProvider
9
10
 
10
11
  remotes = ['midway2-login2.rcc.uchicago.edu', 'midway2-login1.rcc.uchicago.edu']
11
12
 
@@ -2,6 +2,7 @@ import argparse
2
2
  import time
3
3
 
4
4
  import parsl
5
+
5
6
  # Tested. Confirmed. Local X Local X SingleNodeLauncher
6
7
  # from parsl.tests.configs.local_ipp import config
7
8
 
@@ -1,13 +1,13 @@
1
+ import logging
2
+
1
3
  import parsl
2
- from parsl.monitoring.monitoring import MonitoringHub
4
+ from parsl.app.app import python_app
3
5
  from parsl.config import Config
4
6
  from parsl.executors import HighThroughputExecutor
5
7
  from parsl.launchers import AprunLauncher
8
+ from parsl.monitoring.monitoring import MonitoringHub
6
9
  from parsl.providers import CobaltProvider
7
10
 
8
- import logging
9
- from parsl.app.app import python_app
10
-
11
11
 
12
12
  def local_setup():
13
13
  threads_config = Config(
@@ -1,6 +1,8 @@
1
1
  import argparse
2
- import parsl
3
2
  import logging
3
+
4
+ import parsl
5
+
4
6
  parsl.load()
5
7
 
6
8
  from parsl import python_app
@@ -1,15 +1,15 @@
1
1
  import argparse
2
- import parsl
3
- import psutil
4
2
  import multiprocessing
5
3
 
6
- from parsl.providers import LocalProvider
4
+ import psutil
5
+
6
+ import parsl
7
+ from parsl.app.app import python_app # , bash_app
7
8
  from parsl.channels import LocalChannel
8
- from parsl.launchers import SingleNodeLauncher
9
9
  from parsl.config import Config
10
10
  from parsl.executors import HighThroughputExecutor
11
-
12
- from parsl.app.app import python_app # , bash_app
11
+ from parsl.launchers import SingleNodeLauncher
12
+ from parsl.providers import LocalProvider
13
13
 
14
14
 
15
15
  @python_app
@@ -1,7 +1,8 @@
1
+ import time
2
+
1
3
  import pytest
2
4
 
3
5
  from parsl import DataFlowKernel, set_stream_logger
4
- import time
5
6
  from parsl.tests.configs.local_threads import config
6
7
 
7
8
 
@@ -1,9 +1,10 @@
1
+ import logging
2
+
3
+ import parsl
1
4
  from parsl import python_app
2
- from parsl.monitoring.monitoring import MonitoringHub
3
5
  from parsl.config import Config
4
6
  from parsl.executors import ThreadPoolExecutor
5
- import parsl
6
- import logging
7
+ from parsl.monitoring.monitoring import MonitoringHub
7
8
 
8
9
 
9
10
  def local_setup():
@@ -1,7 +1,7 @@
1
1
  import argparse
2
- import time
3
2
  import math
4
3
  import multiprocessing
4
+ import time
5
5
 
6
6
  import parsl
7
7
 
@@ -10,10 +10,11 @@ CORES_PER_WORKER = 1
10
10
  EXPECTED_WORKERS = math.floor(CORES / CORES_PER_WORKER)
11
11
 
12
12
 
13
+ from parsl.executors import HighThroughputExecutor
14
+
13
15
  # from parsl.tests.configs.htex_local import config
14
16
  from parsl.tests.manual_tests.htex_local import config
15
17
 
16
- from parsl.executors import HighThroughputExecutor
17
18
  assert isinstance(config.executors[0], HighThroughputExecutor)
18
19
  config.executors[0].cores_per_worker = CORES_PER_WORKER
19
20
  config.executors[0].provider.init_blocks = 1
@@ -1,8 +1,8 @@
1
- from parsl.providers import LocalProvider
2
1
  from parsl.channels import LocalChannel
3
-
4
2
  from parsl.config import Config
5
3
  from parsl.executors import HighThroughputExecutor
4
+ from parsl.providers import LocalProvider
5
+
6
6
  # import os
7
7
  config = Config(
8
8
  executors=[
@@ -4,15 +4,6 @@ import argparse
4
4
  import time
5
5
 
6
6
  import parsl
7
-
8
- # from parsl.tests.configs.htex_local import config
9
- # from htex_local import config
10
- # from parsl.configs.local_threads import config
11
- # from parsl.configs.local_ipp import config
12
-
13
- # parsl.set_stream_logger()
14
- # config.executors[0].provider.tasks_per_node = 4
15
- # parsl.load(config)
16
7
  from parsl.app.app import python_app # , bash_app
17
8
 
18
9
 
@@ -1,6 +1,5 @@
1
1
  from parsl.config import Config
2
- from parsl.executors.taskvine import TaskVineExecutor
3
- from parsl.executors.taskvine import TaskVineManagerConfig
2
+ from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
4
3
  from parsl.providers import CondorProvider
5
4
 
6
5
  config = Config(
@@ -1,6 +1,5 @@
1
1
  from parsl.config import Config
2
- from parsl.executors.taskvine import TaskVineExecutor
3
- from parsl.executors.taskvine import TaskVineManagerConfig
2
+ from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
4
3
  from parsl.providers import LocalProvider
5
4
 
6
5
  config = Config(
@@ -1,8 +1,10 @@
1
1
  import argparse
2
2
  import logging
3
+ import time
4
+
3
5
  import pytest
6
+
4
7
  import parsl
5
- import time
6
8
  from parsl.app.app import python_app # , bash_app
7
9
  from parsl.jobs.states import JobState
8
10
  from parsl.tests.site_tests.site_config_selector import fresh_config
@@ -1,5 +1,7 @@
1
1
  import argparse
2
+
2
3
  import pytest
4
+
3
5
  import parsl
4
6
  from parsl.app.app import python_app # , bash_app
5
7
  from parsl.tests.site_tests.site_config_selector import fresh_config
@@ -1,12 +1,14 @@
1
1
  """Tests related to assigning workers to specific compute units"""
2
2
 
3
- from parsl.providers import LocalProvider
3
+ import os
4
+
5
+ import pytest
6
+
7
+ from parsl import python_app
4
8
  from parsl.channels import LocalChannel
5
9
  from parsl.config import Config
6
10
  from parsl.executors import HighThroughputExecutor
7
- from parsl import python_app
8
- import pytest
9
- import os
11
+ from parsl.providers import LocalProvider
10
12
 
11
13
 
12
14
  def local_config():
@@ -32,8 +34,8 @@ def local_config():
32
34
 
33
35
  @python_app
34
36
  def get_worker_info():
35
- from time import sleep
36
37
  import os
38
+ from time import sleep
37
39
  rank = int(os.environ['PARSL_WORKER_RANK'])
38
40
  aff = os.sched_getaffinity(0)
39
41
  device = os.environ.get('CUDA_VISIBLE_DEVICES')
@@ -1,10 +1,10 @@
1
+ import pytest
2
+
1
3
  import parsl
4
+ from parsl.app.app import python_app
2
5
  from parsl.executors import HighThroughputExecutor
3
6
  from parsl.executors.threads import ThreadPoolExecutor
4
7
  from parsl.providers import LocalProvider
5
- from parsl.app.app import python_app
6
-
7
- import pytest
8
8
 
9
9
 
10
10
  @python_app(executors=['threads'])
@@ -1,10 +1,11 @@
1
- import parsl
1
+ import logging
2
+
2
3
  import pytest
3
4
 
5
+ import parsl
4
6
  from parsl.app.app import python_app
5
7
  from parsl.tests.configs.ec2_single_node import config
6
8
 
7
- import logging
8
9
  logger = logging.getLogger(__name__)
9
10
 
10
11
  local_config = config
@@ -1,9 +1,10 @@
1
+ import logging
2
+
1
3
  import pytest
2
4
 
3
5
  from parsl import python_app
4
6
  from parsl.tests.configs.local_adhoc import fresh_config as local_config
5
7
 
6
- import logging
7
8
  logger = logging.getLogger(__name__)
8
9
 
9
10
 
@@ -1,11 +1,12 @@
1
1
  """Tests related to Parsl workers being able to access their worker ID"""
2
2
 
3
- from parsl.providers import LocalProvider
3
+ import pytest
4
+
5
+ from parsl import python_app
4
6
  from parsl.channels import LocalChannel
5
7
  from parsl.config import Config
6
8
  from parsl.executors import HighThroughputExecutor
7
- from parsl import python_app
8
- import pytest
9
+ from parsl.providers import LocalProvider
9
10
 
10
11
 
11
12
  def local_config():
@@ -6,7 +6,6 @@ import pytest
6
6
  from parsl.app.app import python_app
7
7
  from parsl.tests.configs.local_threads import config
8
8
 
9
-
10
9
  pytestmark = pytest.mark.skip('not asserting anything')
11
10
 
12
11
 
@@ -1,9 +1,9 @@
1
- import parsl
2
1
  import pytest
3
2
 
3
+ import parsl
4
4
  from parsl.app.app import bash_app
5
- from parsl.tests.configs.local_threads import config
6
5
  from parsl.app.errors import AppTimeout
6
+ from parsl.tests.configs.local_threads import config
7
7
 
8
8
 
9
9
  @bash_app
@@ -4,12 +4,9 @@ import os
4
4
  import pytest
5
5
 
6
6
  import parsl
7
- from parsl.app.app import bash_app
8
7
  import parsl.app.errors as pe
9
-
10
-
8
+ from parsl.app.app import bash_app
11
9
  from parsl.app.errors import BashExitFailure
12
-
13
10
  from parsl.tests.configs.local_threads import fresh_config as local_config
14
11
 
15
12