parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (283) hide show
  1. parsl/__init__.py +9 -10
  2. parsl/addresses.py +6 -4
  3. parsl/app/app.py +3 -6
  4. parsl/app/bash.py +4 -4
  5. parsl/app/errors.py +5 -3
  6. parsl/app/futures.py +3 -3
  7. parsl/app/python.py +2 -1
  8. parsl/benchmark/perf.py +2 -1
  9. parsl/channels/__init__.py +2 -2
  10. parsl/channels/base.py +0 -1
  11. parsl/channels/errors.py +2 -1
  12. parsl/channels/oauth_ssh/oauth_ssh.py +4 -3
  13. parsl/channels/ssh/ssh.py +9 -1
  14. parsl/channels/ssh_il/ssh_il.py +1 -0
  15. parsl/concurrent/__init__.py +2 -2
  16. parsl/config.py +5 -5
  17. parsl/configs/ASPIRE1.py +3 -3
  18. parsl/configs/Azure.py +6 -7
  19. parsl/configs/ad_hoc.py +4 -3
  20. parsl/configs/bridges.py +3 -3
  21. parsl/configs/cc_in2p3.py +2 -2
  22. parsl/configs/ec2.py +1 -1
  23. parsl/configs/expanse.py +1 -2
  24. parsl/configs/frontera.py +2 -3
  25. parsl/configs/htex_local.py +1 -2
  26. parsl/configs/illinoiscluster.py +1 -1
  27. parsl/configs/kubernetes.py +1 -2
  28. parsl/configs/midway.py +3 -3
  29. parsl/configs/osg.py +1 -1
  30. parsl/configs/polaris.py +1 -1
  31. parsl/configs/stampede2.py +4 -5
  32. parsl/configs/summit.py +1 -3
  33. parsl/configs/toss3_llnl.py +1 -2
  34. parsl/configs/vineex_local.py +3 -3
  35. parsl/configs/wqex_local.py +2 -2
  36. parsl/data_provider/data_manager.py +3 -3
  37. parsl/data_provider/file_noop.py +1 -2
  38. parsl/data_provider/files.py +3 -3
  39. parsl/data_provider/ftp.py +1 -3
  40. parsl/data_provider/globus.py +7 -6
  41. parsl/data_provider/http.py +2 -2
  42. parsl/data_provider/rsync.py +1 -1
  43. parsl/data_provider/staging.py +2 -2
  44. parsl/data_provider/zip.py +4 -5
  45. parsl/dataflow/dflow.py +21 -16
  46. parsl/dataflow/errors.py +2 -1
  47. parsl/dataflow/futures.py +1 -2
  48. parsl/dataflow/memoization.py +5 -5
  49. parsl/dataflow/rundirs.py +1 -1
  50. parsl/dataflow/taskrecord.py +4 -5
  51. parsl/executors/__init__.py +3 -3
  52. parsl/executors/base.py +1 -0
  53. parsl/executors/flux/execute_parsl_task.py +2 -2
  54. parsl/executors/flux/executor.py +11 -12
  55. parsl/executors/flux/flux_instance_manager.py +3 -3
  56. parsl/executors/high_throughput/executor.py +29 -35
  57. parsl/executors/high_throughput/interchange.py +16 -18
  58. parsl/executors/high_throughput/manager_record.py +1 -0
  59. parsl/executors/high_throughput/monitoring_info.py +2 -1
  60. parsl/executors/high_throughput/mpi_executor.py +5 -2
  61. parsl/executors/high_throughput/mpi_prefix_composer.py +1 -1
  62. parsl/executors/high_throughput/mpi_resource_management.py +1 -2
  63. parsl/executors/high_throughput/probe.py +6 -4
  64. parsl/executors/high_throughput/process_worker_pool.py +31 -20
  65. parsl/executors/high_throughput/zmq_pipes.py +28 -14
  66. parsl/executors/radical/executor.py +15 -15
  67. parsl/executors/radical/rpex_master.py +1 -2
  68. parsl/executors/radical/rpex_resources.py +1 -2
  69. parsl/executors/radical/rpex_worker.py +2 -1
  70. parsl/executors/status_handling.py +5 -4
  71. parsl/executors/taskvine/__init__.py +1 -1
  72. parsl/executors/taskvine/errors.py +1 -1
  73. parsl/executors/taskvine/exec_parsl_function.py +2 -2
  74. parsl/executors/taskvine/executor.py +23 -24
  75. parsl/executors/taskvine/factory.py +1 -1
  76. parsl/executors/taskvine/manager.py +11 -13
  77. parsl/executors/threads.py +4 -5
  78. parsl/executors/workqueue/errors.py +1 -1
  79. parsl/executors/workqueue/exec_parsl_function.py +5 -4
  80. parsl/executors/workqueue/executor.py +26 -27
  81. parsl/executors/workqueue/parsl_coprocess.py +1 -1
  82. parsl/jobs/error_handlers.py +1 -1
  83. parsl/jobs/job_status_poller.py +2 -5
  84. parsl/jobs/states.py +1 -1
  85. parsl/jobs/strategy.py +2 -2
  86. parsl/launchers/__init__.py +12 -3
  87. parsl/launchers/errors.py +1 -1
  88. parsl/log_utils.py +1 -2
  89. parsl/monitoring/db_manager.py +16 -10
  90. parsl/monitoring/monitoring.py +11 -15
  91. parsl/monitoring/queries/pandas.py +1 -2
  92. parsl/monitoring/radios.py +2 -4
  93. parsl/monitoring/remote.py +13 -8
  94. parsl/monitoring/router.py +8 -11
  95. parsl/monitoring/types.py +2 -0
  96. parsl/monitoring/visualization/app.py +4 -2
  97. parsl/monitoring/visualization/models.py +0 -1
  98. parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
  99. parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
  100. parsl/monitoring/visualization/utils.py +0 -1
  101. parsl/monitoring/visualization/views.py +16 -9
  102. parsl/multiprocessing.py +0 -1
  103. parsl/process_loggers.py +1 -2
  104. parsl/providers/__init__.py +9 -12
  105. parsl/providers/ad_hoc/ad_hoc.py +1 -1
  106. parsl/providers/aws/aws.py +2 -3
  107. parsl/providers/azure/azure.py +4 -5
  108. parsl/providers/base.py +1 -1
  109. parsl/providers/cluster_provider.py +1 -1
  110. parsl/providers/cobalt/cobalt.py +3 -3
  111. parsl/providers/condor/condor.py +4 -2
  112. parsl/providers/errors.py +2 -2
  113. parsl/providers/googlecloud/googlecloud.py +2 -1
  114. parsl/providers/grid_engine/grid_engine.py +2 -2
  115. parsl/providers/kubernetes/kube.py +5 -3
  116. parsl/providers/local/local.py +5 -1
  117. parsl/providers/lsf/lsf.py +2 -2
  118. parsl/providers/pbspro/pbspro.py +1 -1
  119. parsl/providers/slurm/slurm.py +5 -5
  120. parsl/providers/torque/torque.py +1 -1
  121. parsl/serialize/__init__.py +8 -3
  122. parsl/serialize/base.py +1 -2
  123. parsl/serialize/concretes.py +5 -4
  124. parsl/serialize/proxystore.py +3 -2
  125. parsl/tests/__init__.py +1 -1
  126. parsl/tests/configs/ad_hoc_cluster_htex.py +4 -4
  127. parsl/tests/configs/azure_single_node.py +4 -5
  128. parsl/tests/configs/bridges.py +3 -2
  129. parsl/tests/configs/cc_in2p3.py +2 -2
  130. parsl/tests/configs/comet.py +2 -1
  131. parsl/tests/configs/ec2_single_node.py +1 -2
  132. parsl/tests/configs/ec2_spot.py +1 -2
  133. parsl/tests/configs/frontera.py +3 -2
  134. parsl/tests/configs/htex_ad_hoc_cluster.py +2 -4
  135. parsl/tests/configs/htex_local.py +2 -3
  136. parsl/tests/configs/htex_local_alternate.py +8 -11
  137. parsl/tests/configs/htex_local_intask_staging.py +5 -7
  138. parsl/tests/configs/htex_local_rsync_staging.py +4 -6
  139. parsl/tests/configs/local_adhoc.py +1 -1
  140. parsl/tests/configs/local_radical.py +1 -3
  141. parsl/tests/configs/local_radical_mpi.py +2 -2
  142. parsl/tests/configs/midway.py +2 -2
  143. parsl/tests/configs/nscc_singapore.py +3 -3
  144. parsl/tests/configs/osg_htex.py +1 -1
  145. parsl/tests/configs/petrelkube.py +3 -2
  146. parsl/tests/configs/summit.py +1 -0
  147. parsl/tests/configs/swan_htex.py +2 -2
  148. parsl/tests/configs/taskvine_ex.py +3 -5
  149. parsl/tests/configs/theta.py +2 -2
  150. parsl/tests/configs/workqueue_ex.py +3 -4
  151. parsl/tests/conftest.py +4 -4
  152. parsl/tests/integration/test_channels/test_ssh_errors.py +1 -1
  153. parsl/tests/integration/test_stress/test_python_simple.py +3 -4
  154. parsl/tests/integration/test_stress/test_python_threads.py +3 -5
  155. parsl/tests/manual_tests/htex_local.py +4 -4
  156. parsl/tests/manual_tests/test_ad_hoc_htex.py +2 -1
  157. parsl/tests/manual_tests/test_basic.py +1 -0
  158. parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +4 -4
  159. parsl/tests/manual_tests/test_log_filter.py +3 -1
  160. parsl/tests/manual_tests/test_memory_limits.py +6 -6
  161. parsl/tests/manual_tests/test_regression_220.py +2 -1
  162. parsl/tests/manual_tests/test_udp_simple.py +4 -3
  163. parsl/tests/manual_tests/test_worker_count.py +3 -2
  164. parsl/tests/scaling_tests/htex_local.py +2 -2
  165. parsl/tests/scaling_tests/test_scale.py +0 -9
  166. parsl/tests/scaling_tests/vineex_condor.py +1 -2
  167. parsl/tests/scaling_tests/vineex_local.py +1 -2
  168. parsl/tests/site_tests/test_provider.py +3 -1
  169. parsl/tests/site_tests/test_site.py +2 -0
  170. parsl/tests/sites/test_affinity.py +7 -5
  171. parsl/tests/sites/test_dynamic_executor.py +3 -3
  172. parsl/tests/sites/test_ec2.py +3 -2
  173. parsl/tests/sites/test_local_adhoc.py +2 -1
  174. parsl/tests/sites/test_worker_info.py +4 -3
  175. parsl/tests/test_aalst_patterns.py +0 -1
  176. parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
  177. parsl/tests/test_bash_apps/test_error_codes.py +1 -4
  178. parsl/tests/test_bash_apps/test_memoize_ignore_args.py +1 -0
  179. parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +2 -2
  180. parsl/tests/test_bash_apps/test_pipeline.py +1 -1
  181. parsl/tests/test_bash_apps/test_std_uri.py +4 -3
  182. parsl/tests/test_callables.py +2 -2
  183. parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
  184. parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -1
  185. parsl/tests/test_checkpointing/test_regression_239.py +1 -1
  186. parsl/tests/test_checkpointing/test_task_exit.py +1 -1
  187. parsl/tests/test_docs/test_from_slides.py +2 -2
  188. parsl/tests/test_docs/test_kwargs.py +1 -1
  189. parsl/tests/test_docs/test_tutorial_1.py +1 -2
  190. parsl/tests/test_docs/test_workflow1.py +2 -2
  191. parsl/tests/test_docs/test_workflow2.py +0 -1
  192. parsl/tests/test_error_handling/test_rand_fail.py +2 -2
  193. parsl/tests/test_error_handling/test_resource_spec.py +4 -2
  194. parsl/tests/test_error_handling/test_retries.py +2 -1
  195. parsl/tests/test_error_handling/test_retry_handler.py +1 -0
  196. parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
  197. parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
  198. parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
  199. parsl/tests/test_flux.py +1 -1
  200. parsl/tests/test_htex/test_command_client_timeout.py +9 -12
  201. parsl/tests/test_htex/test_connected_blocks.py +3 -2
  202. parsl/tests/test_htex/test_cpu_affinity_explicit.py +5 -2
  203. parsl/tests/test_htex/test_disconnected_blocks.py +6 -4
  204. parsl/tests/test_htex/test_drain.py +5 -5
  205. parsl/tests/test_htex/test_htex.py +1 -2
  206. parsl/tests/test_htex/test_managers_command.py +3 -2
  207. parsl/tests/test_htex/test_multiple_disconnected_blocks.py +6 -4
  208. parsl/tests/test_monitoring/test_app_names.py +3 -2
  209. parsl/tests/test_monitoring/test_basic.py +4 -4
  210. parsl/tests/test_monitoring/test_db_locks.py +6 -3
  211. parsl/tests/test_monitoring/test_fuzz_zmq.py +6 -3
  212. parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +5 -5
  213. parsl/tests/test_monitoring/test_incomplete_futures.py +5 -3
  214. parsl/tests/test_monitoring/test_memoization_representation.py +4 -1
  215. parsl/tests/test_monitoring/test_stdouterr.py +4 -4
  216. parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
  217. parsl/tests/test_mpi_apps/test_bad_mpi_config.py +1 -1
  218. parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +2 -0
  219. parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +7 -5
  220. parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
  221. parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
  222. parsl/tests/test_mpi_apps/test_mpiex.py +4 -3
  223. parsl/tests/test_mpi_apps/test_resource_spec.py +9 -10
  224. parsl/tests/test_providers/test_cobalt_deprecation_warning.py +2 -0
  225. parsl/tests/test_providers/test_local_provider.py +2 -1
  226. parsl/tests/test_providers/test_pbspro_template.py +1 -1
  227. parsl/tests/test_providers/test_slurm_template.py +1 -1
  228. parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
  229. parsl/tests/test_python_apps/test_context_manager.py +4 -3
  230. parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
  231. parsl/tests/test_python_apps/test_futures.py +2 -1
  232. parsl/tests/test_python_apps/test_join.py +0 -1
  233. parsl/tests/test_python_apps/test_lifted.py +3 -3
  234. parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
  235. parsl/tests/test_python_apps/test_pluggable_future_resolution.py +1 -1
  236. parsl/tests/test_radical/test_mpi_funcs.py +1 -1
  237. parsl/tests/test_regression/test_1480.py +2 -1
  238. parsl/tests/test_regression/test_1653.py +2 -1
  239. parsl/tests/test_regression/test_2652.py +1 -0
  240. parsl/tests/test_regression/test_69a.py +0 -1
  241. parsl/tests/test_regression/test_854.py +4 -2
  242. parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
  243. parsl/tests/test_regression/test_98.py +0 -1
  244. parsl/tests/test_scaling/test_block_error_handler.py +9 -4
  245. parsl/tests/test_scaling/test_scale_down.py +2 -3
  246. parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +4 -5
  247. parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +3 -4
  248. parsl/tests/test_scaling/test_shutdown_scalein.py +1 -2
  249. parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
  250. parsl/tests/test_serialization/test_basic.py +2 -1
  251. parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
  252. parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
  253. parsl/tests/test_serialization/test_proxystore_configured.py +10 -5
  254. parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
  255. parsl/tests/test_shutdown/test_kill_monitoring.py +3 -2
  256. parsl/tests/test_staging/staging_provider.py +2 -2
  257. parsl/tests/test_staging/test_1316.py +3 -2
  258. parsl/tests/test_staging/test_docs_1.py +1 -1
  259. parsl/tests/test_staging/test_docs_2.py +2 -1
  260. parsl/tests/test_staging/test_elaborate_noop_file.py +2 -2
  261. parsl/tests/test_staging/test_staging_https.py +2 -2
  262. parsl/tests/test_staging/test_staging_stdout.py +4 -3
  263. parsl/tests/test_staging/test_zip_in.py +6 -8
  264. parsl/tests/test_staging/test_zip_out.py +7 -9
  265. parsl/tests/test_staging/test_zip_to_zip.py +6 -8
  266. parsl/tests/test_summary.py +2 -1
  267. parsl/tests/test_thread_parallelism.py +0 -1
  268. parsl/tests/test_threads/test_configs.py +1 -1
  269. parsl/tests/test_threads/test_lazy_errors.py +2 -1
  270. parsl/usage_tracking/api.py +2 -3
  271. parsl/usage_tracking/usage.py +8 -18
  272. parsl/utils.py +13 -2
  273. parsl/version.py +1 -1
  274. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py +5 -4
  275. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py +31 -20
  276. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/METADATA +2 -2
  277. parsl-2024.6.3.dist-info/RECORD +471 -0
  278. parsl-2024.5.27.dist-info/RECORD +0 -471
  279. {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/parsl_coprocess.py +1 -1
  280. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/LICENSE +0 -0
  281. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/WHEEL +0 -0
  282. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/entry_points.txt +0 -0
  283. {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,14 @@
1
1
  import logging
2
2
  import os
3
+ import pickle
3
4
  from unittest import mock
5
+
4
6
  import pytest
5
- import pickle
6
- from parsl.executors.high_throughput.mpi_resource_management import TaskScheduler, MPITaskScheduler
7
+
8
+ from parsl.executors.high_throughput.mpi_resource_management import (
9
+ MPITaskScheduler,
10
+ TaskScheduler,
11
+ )
7
12
  from parsl.multiprocessing import SpawnContext
8
13
  from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
9
14
 
@@ -5,11 +5,12 @@ from pathlib import Path
5
5
  import pytest
6
6
 
7
7
  import parsl
8
- from .test_mpi_mode_enabled import get_env_vars
9
- from parsl import HighThroughputExecutor, Config
8
+ from parsl import Config, HighThroughputExecutor
9
+ from parsl.executors.high_throughput.mpi_executor import MPIExecutor
10
10
  from parsl.launchers import SimpleLauncher
11
11
  from parsl.providers import LocalProvider
12
- from parsl.executors.high_throughput.mpi_executor import MPIExecutor
12
+
13
+ from .test_mpi_mode_enabled import get_env_vars
13
14
 
14
15
  cwd = Path(__file__).parent.absolute()
15
16
  pbs_nodefile = cwd.joinpath("mocks", "pbs_nodefile")
@@ -2,26 +2,25 @@ import contextlib
2
2
  import logging
3
3
  import os
4
4
  import typing
5
-
5
+ import unittest
6
+ from typing import Dict
6
7
 
7
8
  import pytest
8
- import unittest
9
9
 
10
10
  import parsl
11
11
  from parsl.app.app import python_app
12
- from parsl.tests.configs.htex_local import fresh_config
13
- from typing import Dict
12
+ from parsl.executors.high_throughput.mpi_prefix_composer import (
13
+ InvalidResourceSpecification,
14
+ MissingResourceSpecification,
15
+ validate_resource_spec,
16
+ )
14
17
  from parsl.executors.high_throughput.mpi_resource_management import (
18
+ get_nodes_in_batchjob,
15
19
  get_pbs_hosts_list,
16
20
  get_slurm_hosts_list,
17
- get_nodes_in_batchjob,
18
21
  identify_scheduler,
19
22
  )
20
- from parsl.executors.high_throughput.mpi_prefix_composer import (
21
- validate_resource_spec,
22
- InvalidResourceSpecification,
23
- MissingResourceSpecification
24
- )
23
+ from parsl.tests.configs.htex_local import fresh_config
25
24
 
26
25
  EXECUTOR_LABEL = "MPI_TEST"
27
26
 
@@ -1,5 +1,7 @@
1
1
  import warnings
2
+
2
3
  import pytest
4
+
3
5
  from parsl.providers import CobaltProvider
4
6
 
5
7
 
@@ -1,7 +1,6 @@
1
1
  import logging
2
2
  import os
3
3
  import pathlib
4
- import pytest
5
4
  import random
6
5
  import shutil
7
6
  import socket
@@ -10,6 +9,8 @@ import tempfile
10
9
  import threading
11
10
  import time
12
11
 
12
+ import pytest
13
+
13
14
  from parsl.channels import LocalChannel, SSHChannel
14
15
  from parsl.jobs.states import JobState
15
16
  from parsl.launchers import SingleNodeLauncher
@@ -1,6 +1,6 @@
1
1
  import random
2
-
3
2
  from unittest import mock
3
+
4
4
  import pytest
5
5
 
6
6
  from parsl.channels import LocalChannel
@@ -1,7 +1,7 @@
1
1
  import logging
2
2
  import random
3
-
4
3
  from unittest import mock
4
+
5
5
  import pytest
6
6
 
7
7
  from parsl.channels import LocalChannel
@@ -1,7 +1,8 @@
1
- import pytest
2
1
  import random
3
2
  import string
4
3
 
4
+ import pytest
5
+
5
6
  from parsl.providers.errors import SubmitException
6
7
 
7
8
 
@@ -1,8 +1,9 @@
1
- import parsl
2
- from parsl.tests.configs.local_threads import fresh_config
3
1
  import pytest
4
- from parsl.errors import NoDataFlowKernelError
2
+
3
+ import parsl
5
4
  from parsl.dataflow.dflow import DataFlowKernel
5
+ from parsl.errors import NoDataFlowKernelError
6
+ from parsl.tests.configs.local_threads import fresh_config
6
7
 
7
8
 
8
9
  @parsl.python_app
@@ -1,6 +1,7 @@
1
+ from concurrent.futures import Future
2
+
1
3
  import parsl
2
4
  from parsl.dataflow.errors import DependencyError
3
- from concurrent.futures import Future
4
5
 
5
6
 
6
7
  @parsl.python_app
@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
11
11
  2. done() called on 1, vs 2
12
12
 
13
13
  """
14
- import pytest
15
14
  from os.path import basename
16
15
 
16
+ import pytest
17
+
17
18
  from parsl.app.app import python_app
18
19
  from parsl.data_provider.files import File
19
20
 
@@ -3,7 +3,6 @@ import pytest
3
3
  from parsl import join_app, python_app
4
4
  from parsl.dataflow.errors import JoinError
5
5
 
6
-
7
6
  RESULT_CONSTANT = 3
8
7
 
9
8
 
@@ -1,10 +1,10 @@
1
+ from concurrent.futures import Future
2
+ from typing import TypeVar
3
+
1
4
  import pytest
2
5
 
3
- from concurrent.futures import Future
4
6
  from parsl import python_app
5
7
 
6
- from typing import TypeVar
7
-
8
8
  T = TypeVar('T')
9
9
 
10
10
 
@@ -1,4 +1,5 @@
1
1
  import pytest
2
+
2
3
  from parsl import python_app
3
4
  from parsl.dataflow.memoization import id_for_memo
4
5
 
@@ -7,8 +7,8 @@ import pytest
7
7
 
8
8
  import parsl
9
9
  from parsl.config import Config
10
- from parsl.dataflow.errors import DependencyError
11
10
  from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
11
+ from parsl.dataflow.errors import DependencyError
12
12
 
13
13
 
14
14
  def local_config():
@@ -1,6 +1,6 @@
1
- import parsl
2
1
  import pytest
3
2
 
3
+ import parsl
4
4
  from parsl.tests.configs.local_radical_mpi import fresh_config as local_config
5
5
 
6
6
 
@@ -1,5 +1,6 @@
1
- from parsl import python_app
2
1
  import pytest
2
+
3
+ from parsl import python_app
3
4
  from parsl.tests.configs.htex_local import fresh_config as local_config
4
5
 
5
6
 
@@ -1,5 +1,6 @@
1
- from parsl import python_app
2
1
  import pytest
2
+
3
+ from parsl import python_app
3
4
  from parsl.tests.configs.htex_local import fresh_config as local_config
4
5
 
5
6
 
@@ -1,4 +1,5 @@
1
1
  import pytest
2
+
2
3
  from parsl.jobs.states import JobState
3
4
 
4
5
 
@@ -6,7 +6,6 @@ import pytest
6
6
  from parsl.app.app import bash_app, python_app
7
7
  from parsl.tests.configs.local_threads import config
8
8
 
9
-
10
9
  local_config = config
11
10
 
12
11
 
@@ -1,8 +1,10 @@
1
- import time
2
1
  import multiprocessing
2
+ import random
3
+ import time
4
+
3
5
  import pytest
6
+
4
7
  from parsl.multiprocessing import MacSafeQueue
5
- import random
6
8
 
7
9
 
8
10
  def consumer(in_q, out_q, delay=0):
@@ -1,11 +1,10 @@
1
1
  import pytest
2
2
 
3
3
  import parsl
4
-
5
4
  from parsl.config import Config
6
5
  from parsl.executors import HighThroughputExecutor
7
- from parsl.providers import LocalProvider
8
6
  from parsl.launchers import SimpleLauncher
7
+ from parsl.providers import LocalProvider
9
8
 
10
9
 
11
10
  def local_config() -> Config:
@@ -6,7 +6,6 @@ import argparse
6
6
  import pytest
7
7
 
8
8
  import parsl
9
-
10
9
  from parsl.dataflow.dflow import DataFlowKernel
11
10
  from parsl.tests.configs.local_threads import config
12
11
 
@@ -1,11 +1,16 @@
1
+ from functools import partial
2
+ from unittest.mock import Mock
3
+
1
4
  import pytest
2
5
 
3
6
  from parsl.executors import HighThroughputExecutor
7
+ from parsl.jobs.error_handlers import (
8
+ noop_error_handler,
9
+ simple_error_handler,
10
+ windowed_error_handler,
11
+ )
12
+ from parsl.jobs.states import JobState, JobStatus
4
13
  from parsl.providers import LocalProvider
5
- from unittest.mock import Mock
6
- from parsl.jobs.states import JobStatus, JobState
7
- from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
8
- from functools import partial
9
14
 
10
15
 
11
16
  @pytest.mark.local
@@ -4,13 +4,12 @@ import time
4
4
  import pytest
5
5
 
6
6
  import parsl
7
-
8
7
  from parsl import File, python_app
9
- from parsl.providers import LocalProvider
10
8
  from parsl.channels import LocalChannel
11
- from parsl.launchers import SingleNodeLauncher
12
9
  from parsl.config import Config
13
10
  from parsl.executors import HighThroughputExecutor
11
+ from parsl.launchers import SingleNodeLauncher
12
+ from parsl.providers import LocalProvider
14
13
 
15
14
  logger = logging.getLogger(__name__)
16
15
 
@@ -1,15 +1,14 @@
1
+ from threading import Event
2
+
1
3
  import pytest
2
4
 
3
5
  import parsl
4
-
5
6
  from parsl import File, python_app
6
- from parsl.providers import LocalProvider
7
7
  from parsl.channels import LocalChannel
8
- from parsl.launchers import SingleNodeLauncher
9
8
  from parsl.config import Config
10
9
  from parsl.executors import HighThroughputExecutor
11
-
12
- from threading import Event
10
+ from parsl.launchers import SingleNodeLauncher
11
+ from parsl.providers import LocalProvider
13
12
 
14
13
  _max_blocks = 5
15
14
  _min_blocks = 0
@@ -4,14 +4,13 @@ import time
4
4
  import pytest
5
5
 
6
6
  import parsl
7
-
8
7
  from parsl import File, python_app
9
- from parsl.jobs.states import JobState, TERMINAL_STATES
10
- from parsl.providers import LocalProvider
11
8
  from parsl.channels import LocalChannel
12
- from parsl.launchers import SingleNodeLauncher
13
9
  from parsl.config import Config
14
10
  from parsl.executors import HighThroughputExecutor
11
+ from parsl.jobs.states import TERMINAL_STATES, JobState
12
+ from parsl.launchers import SingleNodeLauncher
13
+ from parsl.providers import LocalProvider
15
14
 
16
15
  logger = logging.getLogger(__name__)
17
16
 
@@ -1,3 +1,4 @@
1
+ import random
1
2
  import threading
2
3
 
3
4
  import pytest
@@ -9,8 +10,6 @@ from parsl.executors import HighThroughputExecutor
9
10
  from parsl.launchers import SimpleLauncher
10
11
  from parsl.providers import LocalProvider
11
12
 
12
- import random
13
-
14
13
  # we need some blocks, but it doesn't matter too much how many, as long
15
14
  # as they can all start up and get registered within the try_assert
16
15
  # timeout later on.
@@ -1,6 +1,6 @@
1
- import parsl
2
1
  import pytest
3
2
 
3
+ import parsl
4
4
  from parsl.tests.configs.htex_local import fresh_config as local_config
5
5
 
6
6
 
@@ -1,5 +1,6 @@
1
1
  import pytest
2
- from parsl.serialize import serialize, deserialize
2
+
3
+ from parsl.serialize import deserialize, serialize
3
4
  from parsl.serialize.concretes import DillSerializer, PickleSerializer
4
5
 
5
6
 
@@ -1,10 +1,9 @@
1
- import parsl
2
- import pytest
3
-
4
1
  from typing import Any
5
2
 
6
- from parsl.serialize.facade import methods_for_code
3
+ import pytest
7
4
 
5
+ import parsl
6
+ from parsl.serialize.facade import methods_for_code
8
7
  from parsl.tests.configs.htex_local import fresh_config as local_config
9
8
 
10
9
 
@@ -1,5 +1,6 @@
1
1
  import pytest
2
- from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message
2
+
3
+ from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
3
4
 
4
5
 
5
6
  def double(x: int, y: int = 2) -> int:
@@ -1,19 +1,24 @@
1
1
  import logging
2
- import pytest
3
2
  import uuid
4
3
 
4
+ import pytest
5
+
5
6
  import parsl
6
- from parsl.serialize.facade import additional_methods_for_deserialization, methods_for_data, register_method_for_data
7
+ from parsl.serialize.facade import (
8
+ additional_methods_for_deserialization,
9
+ methods_for_data,
10
+ register_method_for_data,
11
+ )
7
12
  from parsl.tests.configs.htex_local import fresh_config
8
13
 
9
-
10
14
  logger = logging.getLogger(__name__)
11
15
 
12
16
 
13
17
  def local_setup():
14
- from parsl.serialize.proxystore import ProxyStoreSerializer
15
- from proxystore.store import Store, register_store
16
18
  from proxystore.connectors.file import FileConnector
19
+ from proxystore.store import Store, register_store
20
+
21
+ from parsl.serialize.proxystore import ProxyStoreSerializer
17
22
 
18
23
  parsl.load(fresh_config())
19
24
 
@@ -1,6 +1,7 @@
1
- import pytest
2
1
  import uuid
3
2
 
3
+ import pytest
4
+
4
5
 
5
6
  def policy_example(o):
6
7
  """Example policy will proxy only lists."""
@@ -13,10 +14,11 @@ def test_proxystore_nonglobal():
13
14
  """
14
15
  # import in function, because proxystore is not importable in base parsl
15
16
  # installation.
16
- from parsl.serialize.proxystore import ProxyStoreSerializer
17
+ from proxystore.connectors.file import FileConnector
17
18
  from proxystore.proxy import Proxy
18
19
  from proxystore.store import Store, register_store
19
- from proxystore.connectors.file import FileConnector
20
+
21
+ from parsl.serialize.proxystore import ProxyStoreSerializer
20
22
 
21
23
  store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp"))
22
24
  register_store(store)
@@ -1,9 +1,10 @@
1
1
  import os
2
- import parsl
3
- import pytest
4
2
  import signal
5
3
  import time
6
4
 
5
+ import pytest
6
+
7
+ import parsl
7
8
  from parsl.tests.configs.htex_local_alternate import fresh_config
8
9
 
9
10
  # This is a very generous upper bound on expected shutdown time of target
@@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk):
62
62
 
63
63
 
64
64
  def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True):
65
- import time
66
65
  import logging
66
+ import time
67
67
  logger = logging.getLogger(__name__)
68
68
  logger.info("stage_out_noop")
69
69
  time.sleep(1)
@@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk):
75
75
 
76
76
 
77
77
  def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True):
78
- import time
79
78
  import logging
79
+ import time
80
80
  logger = logging.getLogger(__name__)
81
81
  logger.info("stage_in_noop")
82
82
  time.sleep(1)
@@ -1,8 +1,9 @@
1
+ import time
2
+
1
3
  import pytest
2
4
 
3
5
  import parsl
4
- import time
5
- from parsl import python_app, ThreadPoolExecutor
6
+ from parsl import ThreadPoolExecutor, python_app
6
7
  from parsl.config import Config
7
8
  from parsl.data_provider.files import File
8
9
  from parsl.data_provider.staging import Staging
@@ -1,6 +1,6 @@
1
1
  import pytest
2
2
 
3
- from parsl import python_app, File
3
+ from parsl import File, python_app
4
4
 
5
5
 
6
6
  @python_app
@@ -1,5 +1,6 @@
1
1
  import pytest
2
- from parsl import bash_app, File
2
+
3
+ from parsl import File, bash_app
3
4
  from parsl.tests.configs.local_threads import fresh_config as local_config
4
5
 
5
6
 
@@ -6,15 +6,15 @@
6
6
  # of the globus staging provider
7
7
 
8
8
  import logging
9
+
9
10
  import pytest
10
11
 
11
12
  import parsl
12
-
13
13
  from parsl import bash_app, python_app
14
14
  from parsl.config import Config
15
15
  from parsl.data_provider.files import File
16
16
  from parsl.executors.threads import ThreadPoolExecutor
17
- from parsl.tests.test_staging.staging_provider import NoOpTestingFileStaging, NoOpError
17
+ from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging
18
18
 
19
19
  logger = logging.getLogger(__name__)
20
20
 
@@ -1,9 +1,9 @@
1
+ import pytest
2
+
1
3
  import parsl
2
4
  from parsl.app.app import python_app
3
5
  from parsl.data_provider.files import File
4
6
 
5
- import pytest
6
-
7
7
  # This config is for the local test which will adding an executor.
8
8
  # Most tests in this file should be non-local and use the configuration
9
9
  # specificed with --config, not this one.
@@ -1,12 +1,13 @@
1
1
  import logging
2
2
  import os
3
- import parsl
4
- import pytest
5
3
  import zipfile
6
4
 
5
+ import pytest
6
+
7
+ import parsl
7
8
  from parsl.app.futures import DataFuture
8
- from parsl.tests.configs.htex_local import fresh_config as local_config
9
9
  from parsl.data_provider.files import File
10
+ from parsl.tests.configs.htex_local import fresh_config as local_config
10
11
 
11
12
 
12
13
  @parsl.bash_app
@@ -1,18 +1,16 @@
1
- import parsl
2
- import pytest
3
1
  import random
4
2
  import zipfile
5
3
 
6
- from parsl.data_provider.files import File
7
- from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
4
+ import pytest
8
5
 
9
- from parsl.providers import LocalProvider
6
+ import parsl
10
7
  from parsl.channels import LocalChannel
11
- from parsl.launchers import SimpleLauncher
12
-
13
8
  from parsl.config import Config
9
+ from parsl.data_provider.files import File
10
+ from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
14
11
  from parsl.executors import HighThroughputExecutor
15
-
12
+ from parsl.launchers import SimpleLauncher
13
+ from parsl.providers import LocalProvider
16
14
  from parsl.tests.configs.htex_local import fresh_config as local_config
17
15
 
18
16
 
@@ -1,18 +1,16 @@
1
- import parsl
2
- import pytest
3
1
  import zipfile
4
2
 
5
- from parsl.data_provider.files import File
6
- from parsl.data_provider.data_manager import default_staging
7
- from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
3
+ import pytest
8
4
 
9
- from parsl.providers import LocalProvider
5
+ import parsl
10
6
  from parsl.channels import LocalChannel
11
- from parsl.launchers import SimpleLauncher
12
-
13
7
  from parsl.config import Config
8
+ from parsl.data_provider.data_manager import default_staging
9
+ from parsl.data_provider.files import File
10
+ from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
14
11
  from parsl.executors import HighThroughputExecutor
15
-
12
+ from parsl.launchers import SimpleLauncher
13
+ from parsl.providers import LocalProvider
16
14
  from parsl.tests.configs.htex_local import fresh_config as local_config
17
15
 
18
16