parsl 2024.3.11__py3-none-any.whl → 2025.1.13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (369) hide show
  1. parsl/__init__.py +9 -10
  2. parsl/addresses.py +29 -7
  3. parsl/app/app.py +7 -8
  4. parsl/app/bash.py +15 -8
  5. parsl/app/errors.py +10 -13
  6. parsl/app/futures.py +8 -10
  7. parsl/app/python.py +2 -1
  8. parsl/benchmark/perf.py +2 -1
  9. parsl/concurrent/__init__.py +2 -2
  10. parsl/config.py +57 -10
  11. parsl/configs/ASPIRE1.py +6 -5
  12. parsl/configs/Azure.py +9 -8
  13. parsl/configs/bridges.py +6 -4
  14. parsl/configs/cc_in2p3.py +3 -3
  15. parsl/configs/ec2.py +3 -1
  16. parsl/configs/expanse.py +4 -3
  17. parsl/configs/frontera.py +3 -4
  18. parsl/configs/htex_local.py +3 -4
  19. parsl/configs/illinoiscluster.py +3 -1
  20. parsl/configs/improv.py +34 -0
  21. parsl/configs/kubernetes.py +4 -3
  22. parsl/configs/local_threads.py +5 -1
  23. parsl/configs/midway.py +5 -3
  24. parsl/configs/osg.py +4 -2
  25. parsl/configs/polaris.py +4 -2
  26. parsl/configs/stampede2.py +6 -5
  27. parsl/configs/summit.py +3 -3
  28. parsl/configs/toss3_llnl.py +4 -3
  29. parsl/configs/vineex_local.py +6 -4
  30. parsl/configs/wqex_local.py +5 -3
  31. parsl/curvezmq.py +4 -0
  32. parsl/data_provider/data_manager.py +4 -3
  33. parsl/data_provider/file_noop.py +1 -2
  34. parsl/data_provider/files.py +3 -3
  35. parsl/data_provider/ftp.py +1 -3
  36. parsl/data_provider/globus.py +7 -6
  37. parsl/data_provider/http.py +2 -2
  38. parsl/data_provider/rsync.py +1 -1
  39. parsl/data_provider/staging.py +2 -2
  40. parsl/data_provider/zip.py +135 -0
  41. parsl/dataflow/dependency_resolvers.py +115 -0
  42. parsl/dataflow/dflow.py +262 -224
  43. parsl/dataflow/errors.py +3 -5
  44. parsl/dataflow/futures.py +27 -14
  45. parsl/dataflow/memoization.py +5 -5
  46. parsl/dataflow/rundirs.py +5 -6
  47. parsl/dataflow/taskrecord.py +4 -5
  48. parsl/executors/__init__.py +4 -2
  49. parsl/executors/base.py +45 -15
  50. parsl/executors/errors.py +13 -0
  51. parsl/executors/execute_task.py +37 -0
  52. parsl/executors/flux/execute_parsl_task.py +3 -3
  53. parsl/executors/flux/executor.py +18 -19
  54. parsl/executors/flux/flux_instance_manager.py +26 -27
  55. parsl/executors/high_throughput/errors.py +43 -3
  56. parsl/executors/high_throughput/executor.py +316 -282
  57. parsl/executors/high_throughput/interchange.py +158 -167
  58. parsl/executors/high_throughput/manager_record.py +5 -0
  59. parsl/executors/high_throughput/manager_selector.py +55 -0
  60. parsl/executors/high_throughput/monitoring_info.py +2 -1
  61. parsl/executors/high_throughput/mpi_executor.py +113 -0
  62. parsl/executors/high_throughput/mpi_prefix_composer.py +10 -11
  63. parsl/executors/high_throughput/mpi_resource_management.py +6 -17
  64. parsl/executors/high_throughput/probe.py +9 -7
  65. parsl/executors/high_throughput/process_worker_pool.py +115 -77
  66. parsl/executors/high_throughput/zmq_pipes.py +81 -23
  67. parsl/executors/radical/executor.py +130 -79
  68. parsl/executors/radical/rpex_resources.py +17 -15
  69. parsl/executors/radical/rpex_worker.py +4 -3
  70. parsl/executors/status_handling.py +157 -51
  71. parsl/executors/taskvine/__init__.py +1 -1
  72. parsl/executors/taskvine/errors.py +1 -1
  73. parsl/executors/taskvine/exec_parsl_function.py +2 -2
  74. parsl/executors/taskvine/executor.py +41 -57
  75. parsl/executors/taskvine/factory.py +1 -1
  76. parsl/executors/taskvine/factory_config.py +1 -1
  77. parsl/executors/taskvine/manager.py +18 -13
  78. parsl/executors/taskvine/manager_config.py +9 -5
  79. parsl/executors/threads.py +6 -6
  80. parsl/executors/workqueue/errors.py +1 -1
  81. parsl/executors/workqueue/exec_parsl_function.py +6 -5
  82. parsl/executors/workqueue/executor.py +64 -63
  83. parsl/executors/workqueue/parsl_coprocess.py +1 -1
  84. parsl/jobs/error_handlers.py +2 -2
  85. parsl/jobs/job_status_poller.py +30 -113
  86. parsl/jobs/states.py +7 -2
  87. parsl/jobs/strategy.py +43 -31
  88. parsl/launchers/__init__.py +12 -3
  89. parsl/launchers/errors.py +1 -1
  90. parsl/launchers/launchers.py +6 -12
  91. parsl/log_utils.py +9 -6
  92. parsl/monitoring/db_manager.py +59 -95
  93. parsl/monitoring/errors.py +6 -0
  94. parsl/monitoring/monitoring.py +87 -356
  95. parsl/monitoring/queries/pandas.py +1 -2
  96. parsl/monitoring/radios/base.py +13 -0
  97. parsl/monitoring/radios/filesystem.py +52 -0
  98. parsl/monitoring/radios/htex.py +57 -0
  99. parsl/monitoring/radios/multiprocessing.py +17 -0
  100. parsl/monitoring/radios/udp.py +56 -0
  101. parsl/monitoring/radios/zmq.py +17 -0
  102. parsl/monitoring/remote.py +33 -37
  103. parsl/monitoring/router.py +212 -0
  104. parsl/monitoring/types.py +5 -6
  105. parsl/monitoring/visualization/app.py +4 -2
  106. parsl/monitoring/visualization/models.py +0 -1
  107. parsl/monitoring/visualization/plots/default/workflow_plots.py +11 -4
  108. parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
  109. parsl/monitoring/visualization/utils.py +0 -1
  110. parsl/monitoring/visualization/views.py +16 -8
  111. parsl/multiprocessing.py +0 -1
  112. parsl/process_loggers.py +1 -2
  113. parsl/providers/__init__.py +8 -17
  114. parsl/providers/aws/aws.py +2 -3
  115. parsl/providers/azure/azure.py +4 -5
  116. parsl/providers/base.py +2 -18
  117. parsl/providers/cluster_provider.py +4 -12
  118. parsl/providers/condor/condor.py +7 -17
  119. parsl/providers/errors.py +2 -2
  120. parsl/providers/googlecloud/googlecloud.py +2 -1
  121. parsl/providers/grid_engine/grid_engine.py +5 -14
  122. parsl/providers/kubernetes/kube.py +80 -40
  123. parsl/providers/local/local.py +13 -26
  124. parsl/providers/lsf/lsf.py +5 -23
  125. parsl/providers/pbspro/pbspro.py +5 -17
  126. parsl/providers/slurm/slurm.py +81 -39
  127. parsl/providers/torque/torque.py +3 -14
  128. parsl/serialize/__init__.py +8 -3
  129. parsl/serialize/base.py +1 -2
  130. parsl/serialize/concretes.py +5 -4
  131. parsl/serialize/facade.py +3 -3
  132. parsl/serialize/proxystore.py +3 -2
  133. parsl/tests/__init__.py +1 -1
  134. parsl/tests/configs/azure_single_node.py +4 -5
  135. parsl/tests/configs/bridges.py +3 -2
  136. parsl/tests/configs/cc_in2p3.py +1 -3
  137. parsl/tests/configs/comet.py +2 -1
  138. parsl/tests/configs/ec2_single_node.py +1 -2
  139. parsl/tests/configs/ec2_spot.py +1 -2
  140. parsl/tests/configs/flux_local.py +11 -0
  141. parsl/tests/configs/frontera.py +2 -3
  142. parsl/tests/configs/htex_local.py +3 -5
  143. parsl/tests/configs/htex_local_alternate.py +11 -15
  144. parsl/tests/configs/htex_local_intask_staging.py +5 -9
  145. parsl/tests/configs/htex_local_rsync_staging.py +4 -8
  146. parsl/tests/configs/local_radical.py +1 -3
  147. parsl/tests/configs/local_radical_mpi.py +2 -2
  148. parsl/tests/configs/local_threads_checkpoint_periodic.py +8 -10
  149. parsl/tests/configs/local_threads_monitoring.py +0 -1
  150. parsl/tests/configs/midway.py +2 -2
  151. parsl/tests/configs/nscc_singapore.py +3 -3
  152. parsl/tests/configs/osg_htex.py +1 -1
  153. parsl/tests/configs/petrelkube.py +3 -2
  154. parsl/tests/configs/slurm_local.py +24 -0
  155. parsl/tests/configs/summit.py +1 -0
  156. parsl/tests/configs/taskvine_ex.py +4 -7
  157. parsl/tests/configs/user_opts.py +2 -8
  158. parsl/tests/configs/workqueue_ex.py +4 -6
  159. parsl/tests/conftest.py +27 -13
  160. parsl/tests/integration/test_stress/test_python_simple.py +3 -4
  161. parsl/tests/integration/test_stress/test_python_threads.py +3 -5
  162. parsl/tests/manual_tests/htex_local.py +4 -6
  163. parsl/tests/manual_tests/test_basic.py +1 -0
  164. parsl/tests/manual_tests/test_log_filter.py +3 -1
  165. parsl/tests/manual_tests/test_memory_limits.py +6 -8
  166. parsl/tests/manual_tests/test_regression_220.py +2 -1
  167. parsl/tests/manual_tests/test_udp_simple.py +4 -4
  168. parsl/tests/manual_tests/test_worker_count.py +3 -2
  169. parsl/tests/scaling_tests/htex_local.py +2 -4
  170. parsl/tests/scaling_tests/test_scale.py +0 -9
  171. parsl/tests/scaling_tests/vineex_condor.py +1 -2
  172. parsl/tests/scaling_tests/vineex_local.py +1 -2
  173. parsl/tests/site_tests/site_config_selector.py +1 -6
  174. parsl/tests/site_tests/test_provider.py +4 -2
  175. parsl/tests/site_tests/test_site.py +2 -0
  176. parsl/tests/sites/test_affinity.py +7 -7
  177. parsl/tests/sites/test_dynamic_executor.py +3 -4
  178. parsl/tests/sites/test_ec2.py +3 -2
  179. parsl/tests/sites/test_worker_info.py +4 -5
  180. parsl/tests/test_aalst_patterns.py +0 -1
  181. parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
  182. parsl/tests/test_bash_apps/test_basic.py +10 -4
  183. parsl/tests/test_bash_apps/test_error_codes.py +5 -7
  184. parsl/tests/test_bash_apps/test_inputs_default.py +25 -0
  185. parsl/tests/test_bash_apps/test_kwarg_storage.py +1 -1
  186. parsl/tests/test_bash_apps/test_memoize.py +2 -8
  187. parsl/tests/test_bash_apps/test_memoize_ignore_args.py +9 -14
  188. parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +9 -14
  189. parsl/tests/test_bash_apps/test_multiline.py +1 -1
  190. parsl/tests/test_bash_apps/test_pipeline.py +1 -1
  191. parsl/tests/test_bash_apps/test_std_uri.py +123 -0
  192. parsl/tests/test_bash_apps/test_stdout.py +33 -8
  193. parsl/tests/test_callables.py +2 -2
  194. parsl/tests/test_checkpointing/test_periodic.py +21 -39
  195. parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
  196. parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -2
  197. parsl/tests/test_checkpointing/test_python_checkpoint_3.py +0 -1
  198. parsl/tests/test_checkpointing/test_regression_239.py +1 -1
  199. parsl/tests/test_checkpointing/test_task_exit.py +2 -3
  200. parsl/tests/test_docs/test_from_slides.py +5 -2
  201. parsl/tests/test_docs/test_kwargs.py +4 -1
  202. parsl/tests/test_docs/test_tutorial_1.py +1 -2
  203. parsl/tests/test_docs/test_workflow1.py +2 -2
  204. parsl/tests/test_docs/test_workflow2.py +0 -1
  205. parsl/tests/test_error_handling/test_rand_fail.py +2 -2
  206. parsl/tests/test_error_handling/test_resource_spec.py +10 -12
  207. parsl/tests/test_error_handling/test_retries.py +6 -16
  208. parsl/tests/test_error_handling/test_retry_handler.py +1 -0
  209. parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
  210. parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
  211. parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
  212. parsl/tests/test_execute_task.py +29 -0
  213. parsl/tests/test_flux.py +1 -1
  214. parsl/tests/test_htex/test_basic.py +2 -3
  215. parsl/tests/test_htex/test_block_manager_selector_unit.py +20 -0
  216. parsl/tests/test_htex/test_command_client_timeout.py +66 -0
  217. parsl/tests/test_htex/test_connected_blocks.py +3 -2
  218. parsl/tests/test_htex/test_cpu_affinity_explicit.py +6 -10
  219. parsl/tests/test_htex/test_disconnected_blocks.py +6 -5
  220. parsl/tests/test_htex/test_disconnected_blocks_failing_provider.py +71 -0
  221. parsl/tests/test_htex/test_drain.py +79 -0
  222. parsl/tests/test_htex/test_htex.py +51 -25
  223. parsl/tests/test_htex/test_manager_failure.py +0 -1
  224. parsl/tests/test_htex/test_manager_selector_by_block.py +51 -0
  225. parsl/tests/test_htex/test_managers_command.py +36 -0
  226. parsl/tests/test_htex/test_missing_worker.py +2 -12
  227. parsl/tests/test_htex/test_multiple_disconnected_blocks.py +9 -9
  228. parsl/tests/test_htex/test_resource_spec_validation.py +45 -0
  229. parsl/tests/test_htex/test_zmq_binding.py +29 -8
  230. parsl/tests/test_monitoring/test_app_names.py +86 -0
  231. parsl/tests/test_monitoring/test_basic.py +73 -25
  232. parsl/tests/test_monitoring/test_db_locks.py +6 -4
  233. parsl/tests/test_monitoring/test_fuzz_zmq.py +19 -8
  234. parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +80 -0
  235. parsl/tests/test_monitoring/test_incomplete_futures.py +5 -4
  236. parsl/tests/test_monitoring/test_memoization_representation.py +4 -2
  237. parsl/tests/test_monitoring/test_stdouterr.py +134 -0
  238. parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
  239. parsl/tests/test_mpi_apps/test_bad_mpi_config.py +33 -26
  240. parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +28 -11
  241. parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
  242. parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
  243. parsl/tests/test_mpi_apps/test_mpiex.py +64 -0
  244. parsl/tests/test_mpi_apps/test_resource_spec.py +42 -49
  245. parsl/tests/test_providers/test_kubernetes_provider.py +102 -0
  246. parsl/tests/test_providers/test_local_provider.py +3 -132
  247. parsl/tests/test_providers/test_pbspro_template.py +2 -3
  248. parsl/tests/test_providers/test_slurm_template.py +2 -3
  249. parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
  250. parsl/tests/test_python_apps/test_context_manager.py +128 -0
  251. parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
  252. parsl/tests/test_python_apps/test_dependencies_deep.py +59 -0
  253. parsl/tests/test_python_apps/test_fail.py +0 -25
  254. parsl/tests/test_python_apps/test_futures.py +2 -1
  255. parsl/tests/test_python_apps/test_inputs_default.py +22 -0
  256. parsl/tests/test_python_apps/test_join.py +0 -1
  257. parsl/tests/test_python_apps/test_lifted.py +11 -7
  258. parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
  259. parsl/tests/test_python_apps/test_outputs.py +1 -1
  260. parsl/tests/test_python_apps/test_pluggable_future_resolution.py +161 -0
  261. parsl/tests/test_radical/test_mpi_funcs.py +1 -2
  262. parsl/tests/test_regression/test_1480.py +2 -1
  263. parsl/tests/test_regression/test_1653.py +2 -1
  264. parsl/tests/test_regression/test_226.py +1 -0
  265. parsl/tests/test_regression/test_2652.py +1 -0
  266. parsl/tests/test_regression/test_69a.py +0 -1
  267. parsl/tests/test_regression/test_854.py +4 -2
  268. parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
  269. parsl/tests/test_regression/test_98.py +0 -1
  270. parsl/tests/test_scaling/test_block_error_handler.py +9 -4
  271. parsl/tests/test_scaling/test_regression_1621.py +11 -15
  272. parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py +84 -0
  273. parsl/tests/test_scaling/test_regression_3696_oscillation.py +103 -0
  274. parsl/tests/test_scaling/test_scale_down.py +2 -5
  275. parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +6 -18
  276. parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +71 -0
  277. parsl/tests/test_scaling/test_shutdown_scalein.py +73 -0
  278. parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py +90 -0
  279. parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
  280. parsl/tests/test_serialization/test_3495_deserialize_managerlost.py +47 -0
  281. parsl/tests/test_serialization/test_basic.py +2 -1
  282. parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
  283. parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
  284. parsl/tests/test_serialization/test_proxystore_configured.py +10 -6
  285. parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
  286. parsl/tests/test_shutdown/test_kill_monitoring.py +64 -0
  287. parsl/tests/test_staging/staging_provider.py +2 -2
  288. parsl/tests/test_staging/test_1316.py +3 -4
  289. parsl/tests/test_staging/test_docs_1.py +2 -1
  290. parsl/tests/test_staging/test_docs_2.py +2 -1
  291. parsl/tests/test_staging/test_elaborate_noop_file.py +2 -3
  292. parsl/tests/{test_data → test_staging}/test_file.py +6 -6
  293. parsl/tests/{test_data → test_staging}/test_output_chain_filenames.py +3 -0
  294. parsl/tests/test_staging/test_staging_ftp.py +1 -0
  295. parsl/tests/test_staging/test_staging_https.py +5 -2
  296. parsl/tests/test_staging/test_staging_stdout.py +64 -0
  297. parsl/tests/test_staging/test_zip_in.py +39 -0
  298. parsl/tests/test_staging/test_zip_out.py +110 -0
  299. parsl/tests/test_staging/test_zip_to_zip.py +41 -0
  300. parsl/tests/test_summary.py +2 -2
  301. parsl/tests/test_thread_parallelism.py +0 -1
  302. parsl/tests/test_threads/test_configs.py +1 -2
  303. parsl/tests/test_threads/test_lazy_errors.py +2 -2
  304. parsl/tests/test_utils/test_execute_wait.py +35 -0
  305. parsl/tests/test_utils/test_sanitize_dns.py +76 -0
  306. parsl/tests/unit/test_address.py +20 -0
  307. parsl/tests/unit/test_file.py +99 -0
  308. parsl/tests/unit/test_usage_tracking.py +66 -0
  309. parsl/usage_tracking/api.py +65 -0
  310. parsl/usage_tracking/levels.py +6 -0
  311. parsl/usage_tracking/usage.py +104 -62
  312. parsl/utils.py +139 -6
  313. parsl/version.py +1 -1
  314. {parsl-2024.3.11.data → parsl-2025.1.13.data}/scripts/exec_parsl_function.py +6 -5
  315. parsl-2025.1.13.data/scripts/interchange.py +649 -0
  316. {parsl-2024.3.11.data → parsl-2025.1.13.data}/scripts/process_worker_pool.py +115 -77
  317. parsl-2025.1.13.dist-info/METADATA +96 -0
  318. parsl-2025.1.13.dist-info/RECORD +462 -0
  319. {parsl-2024.3.11.dist-info → parsl-2025.1.13.dist-info}/WHEEL +1 -1
  320. parsl/channels/__init__.py +0 -7
  321. parsl/channels/base.py +0 -141
  322. parsl/channels/errors.py +0 -113
  323. parsl/channels/local/local.py +0 -164
  324. parsl/channels/oauth_ssh/oauth_ssh.py +0 -110
  325. parsl/channels/ssh/ssh.py +0 -276
  326. parsl/channels/ssh_il/__init__.py +0 -0
  327. parsl/channels/ssh_il/ssh_il.py +0 -74
  328. parsl/configs/ad_hoc.py +0 -35
  329. parsl/executors/radical/rpex_master.py +0 -42
  330. parsl/monitoring/radios.py +0 -175
  331. parsl/providers/ad_hoc/__init__.py +0 -0
  332. parsl/providers/ad_hoc/ad_hoc.py +0 -248
  333. parsl/providers/cobalt/__init__.py +0 -0
  334. parsl/providers/cobalt/cobalt.py +0 -236
  335. parsl/providers/cobalt/template.py +0 -17
  336. parsl/tests/configs/ad_hoc_cluster_htex.py +0 -35
  337. parsl/tests/configs/cooley_htex.py +0 -37
  338. parsl/tests/configs/htex_ad_hoc_cluster.py +0 -28
  339. parsl/tests/configs/local_adhoc.py +0 -18
  340. parsl/tests/configs/swan_htex.py +0 -43
  341. parsl/tests/configs/theta.py +0 -37
  342. parsl/tests/integration/test_channels/__init__.py +0 -0
  343. parsl/tests/integration/test_channels/test_channels.py +0 -17
  344. parsl/tests/integration/test_channels/test_local_channel.py +0 -42
  345. parsl/tests/integration/test_channels/test_scp_1.py +0 -45
  346. parsl/tests/integration/test_channels/test_ssh_1.py +0 -40
  347. parsl/tests/integration/test_channels/test_ssh_errors.py +0 -46
  348. parsl/tests/integration/test_channels/test_ssh_file_transport.py +0 -41
  349. parsl/tests/integration/test_channels/test_ssh_interactive.py +0 -24
  350. parsl/tests/manual_tests/test_ad_hoc_htex.py +0 -48
  351. parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +0 -88
  352. parsl/tests/manual_tests/test_oauth_ssh.py +0 -13
  353. parsl/tests/sites/test_local_adhoc.py +0 -61
  354. parsl/tests/test_channels/__init__.py +0 -0
  355. parsl/tests/test_channels/test_large_output.py +0 -22
  356. parsl/tests/test_data/__init__.py +0 -0
  357. parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +0 -51
  358. parsl/tests/test_providers/test_cobalt_deprecation_warning.py +0 -16
  359. parsl-2024.3.11.dist-info/METADATA +0 -98
  360. parsl-2024.3.11.dist-info/RECORD +0 -447
  361. parsl/{channels/local → monitoring/radios}/__init__.py +0 -0
  362. parsl/{channels/oauth_ssh → tests/test_shutdown}/__init__.py +0 -0
  363. parsl/tests/{test_data → test_staging}/test_file_apps.py +0 -0
  364. parsl/tests/{test_data → test_staging}/test_file_staging.py +0 -0
  365. parsl/{channels/ssh → tests/unit}/__init__.py +0 -0
  366. {parsl-2024.3.11.data → parsl-2025.1.13.data}/scripts/parsl_coprocess.py +1 -1
  367. {parsl-2024.3.11.dist-info → parsl-2025.1.13.dist-info}/LICENSE +0 -0
  368. {parsl-2024.3.11.dist-info → parsl-2025.1.13.dist-info}/entry_points.txt +0 -0
  369. {parsl-2024.3.11.dist-info → parsl-2025.1.13.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,57 @@
1
+ import logging
2
+ import pickle
3
+
4
+ from parsl.monitoring.radios.base import MonitoringRadioSender
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+
9
+ class HTEXRadioSender(MonitoringRadioSender):
10
+
11
+ def __init__(self, monitoring_url: str, timeout: int = 10):
12
+ """
13
+ Parameters
14
+ ----------
15
+
16
+ monitoring_url : str
17
+ URL of the form <scheme>://<IP>:<PORT>
18
+ timeout : int
19
+ timeout, default=10s
20
+ """
21
+ logger.info("htex-based monitoring channel initialising")
22
+
23
+ def send(self, message: object) -> None:
24
+ """ Sends a message to the UDP receiver
25
+
26
+ Parameter
27
+ ---------
28
+
29
+ message: object
30
+ Arbitrary pickle-able object that is to be sent
31
+
32
+ Returns:
33
+ None
34
+ """
35
+
36
+ import parsl.executors.high_throughput.monitoring_info
37
+
38
+ result_queue = parsl.executors.high_throughput.monitoring_info.result_queue
39
+
40
+ # this message needs to go in the result queue tagged so that it is treated
41
+ # i) as a monitoring message by the interchange, and then further more treated
42
+ # as a RESOURCE_INFO message when received by monitoring (rather than a NODE_INFO
43
+ # which is the implicit default for messages from the interchange)
44
+
45
+ # for the interchange, the outer wrapper, this needs to be a dict:
46
+
47
+ interchange_msg = {
48
+ 'type': 'monitoring',
49
+ 'payload': message
50
+ }
51
+
52
+ if result_queue:
53
+ result_queue.put(pickle.dumps(interchange_msg))
54
+ else:
55
+ logger.error("result_queue is uninitialized - cannot put monitoring message")
56
+
57
+ return
@@ -0,0 +1,17 @@
1
+ from multiprocessing.queues import Queue
2
+
3
+ from parsl.monitoring.radios.base import MonitoringRadioSender
4
+
5
+
6
+ class MultiprocessingQueueRadioSender(MonitoringRadioSender):
7
+ """A monitoring radio which connects over a multiprocessing Queue.
8
+ This radio is intended to be used on the submit side, where components
9
+ in the submit process, or processes launched by multiprocessing, will have
10
+ access to a Queue shared with the monitoring database code (bypassing the
11
+ monitoring router).
12
+ """
13
+ def __init__(self, queue: Queue) -> None:
14
+ self.queue = queue
15
+
16
+ def send(self, message: object) -> None:
17
+ self.queue.put(message)
@@ -0,0 +1,56 @@
1
+ import logging
2
+ import pickle
3
+ import socket
4
+
5
+ from parsl.monitoring.radios.base import MonitoringRadioSender
6
+
7
+
8
+ class UDPRadioSender(MonitoringRadioSender):
9
+
10
+ def __init__(self, monitoring_url: str, timeout: int = 10):
11
+ """
12
+ Parameters
13
+ ----------
14
+
15
+ monitoring_url : str
16
+ URL of the form <scheme>://<IP>:<PORT>
17
+ timeout : int
18
+ timeout, default=10s
19
+ """
20
+ self.monitoring_url = monitoring_url
21
+ self.sock_timeout = timeout
22
+ try:
23
+ self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
24
+ self.port = int(port)
25
+ except Exception:
26
+ raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
27
+
28
+ self.sock = socket.socket(socket.AF_INET,
29
+ socket.SOCK_DGRAM,
30
+ socket.IPPROTO_UDP) # UDP
31
+ self.sock.settimeout(self.sock_timeout)
32
+
33
+ def send(self, message: object) -> None:
34
+ """ Sends a message to the UDP receiver
35
+
36
+ Parameter
37
+ ---------
38
+
39
+ message: object
40
+ Arbitrary pickle-able object that is to be sent
41
+
42
+ Returns:
43
+ None
44
+ """
45
+ try:
46
+ buffer = pickle.dumps(message)
47
+ except Exception:
48
+ logging.exception("Exception during pickling", exc_info=True)
49
+ return
50
+
51
+ try:
52
+ self.sock.sendto(buffer, (self.ip, self.port))
53
+ except socket.timeout:
54
+ logging.error("Could not send message within timeout limit")
55
+ return
56
+ return
@@ -0,0 +1,17 @@
1
+ import zmq
2
+
3
+ from parsl.monitoring.radios.base import MonitoringRadioSender
4
+
5
+
6
+ class ZMQRadioSender(MonitoringRadioSender):
7
+ """A monitoring radio which connects over ZMQ. This radio is not
8
+ thread-safe, because its use of ZMQ is not thread-safe.
9
+ """
10
+
11
+ def __init__(self, hub_address: str, hub_zmq_port: int) -> None:
12
+ self._hub_channel = zmq.Context().socket(zmq.DEALER)
13
+ self._hub_channel.set_hwm(0)
14
+ self._hub_channel.connect(f"tcp://{hub_address}:{hub_zmq_port}")
15
+
16
+ def send(self, message: object) -> None:
17
+ self._hub_channel.send_pyobj(message)
@@ -1,21 +1,24 @@
1
+ import datetime
2
+ import logging
1
3
  import os
2
4
  import time
3
- import logging
4
- import datetime
5
5
  from functools import wraps
6
-
7
- from parsl.multiprocessing import ForkProcess
8
6
  from multiprocessing import Event
9
- from parsl.process_loggers import wrap_with_logs
7
+ from typing import Any, Callable, Dict, List, Sequence, Tuple
10
8
 
11
9
  from parsl.monitoring.message_type import MessageType
12
- from parsl.monitoring.radios import MonitoringRadio, UDPRadio, HTEXRadio, FilesystemRadio
13
- from typing import Any, Callable, Dict, List, Sequence, Tuple
10
+ from parsl.monitoring.radios.base import MonitoringRadioSender
11
+ from parsl.monitoring.radios.filesystem import FilesystemRadioSender
12
+ from parsl.monitoring.radios.htex import HTEXRadioSender
13
+ from parsl.monitoring.radios.udp import UDPRadioSender
14
+ from parsl.multiprocessing import ForkProcess
15
+ from parsl.process_loggers import wrap_with_logs
14
16
 
15
17
  logger = logging.getLogger(__name__)
16
18
 
17
19
 
18
- def monitor_wrapper(f: Any, # per app
20
+ def monitor_wrapper(*,
21
+ f: Any, # per app
19
22
  args: Sequence, # per invocation
20
23
  kwargs: Dict, # per invocation
21
24
  x_try_id: int, # per invocation
@@ -95,6 +98,20 @@ def monitor_wrapper(f: Any, # per app
95
98
  return (wrapped, args, new_kwargs)
96
99
 
97
100
 
101
+ def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadioSender:
102
+ radio: MonitoringRadioSender
103
+ if radio_mode == "udp":
104
+ radio = UDPRadioSender(monitoring_hub_url)
105
+ elif radio_mode == "htex":
106
+ radio = HTEXRadioSender(monitoring_hub_url)
107
+ elif radio_mode == "filesystem":
108
+ radio = FilesystemRadioSender(monitoring_url=monitoring_hub_url,
109
+ run_dir=run_dir)
110
+ else:
111
+ raise RuntimeError(f"Unknown radio mode: {radio_mode}")
112
+ return radio
113
+
114
+
98
115
  @wrap_with_logs
99
116
  def send_first_message(try_id: int,
100
117
  task_id: int,
@@ -118,21 +135,10 @@ def send_first_last_message(try_id: int,
118
135
  monitoring_hub_url: str,
119
136
  run_id: str, radio_mode: str, run_dir: str,
120
137
  is_last: bool) -> None:
121
- import platform
122
138
  import os
139
+ import platform
123
140
 
124
- radio: MonitoringRadio
125
- if radio_mode == "udp":
126
- radio = UDPRadio(monitoring_hub_url,
127
- source_id=task_id)
128
- elif radio_mode == "htex":
129
- radio = HTEXRadio(monitoring_hub_url,
130
- source_id=task_id)
131
- elif radio_mode == "filesystem":
132
- radio = FilesystemRadio(monitoring_url=monitoring_hub_url,
133
- source_id=task_id, run_dir=run_dir)
134
- else:
135
- raise RuntimeError(f"Unknown radio mode: {radio_mode}")
141
+ radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
136
142
 
137
143
  msg = (MessageType.RESOURCE_INFO,
138
144
  {'run_id': run_id,
@@ -171,24 +177,14 @@ def monitor(pid: int,
171
177
  """
172
178
  import logging
173
179
  import platform
180
+
174
181
  import psutil
175
182
 
176
183
  from parsl.utils import setproctitle
177
184
 
178
185
  setproctitle("parsl: task resource monitor")
179
186
 
180
- radio: MonitoringRadio
181
- if radio_mode == "udp":
182
- radio = UDPRadio(monitoring_hub_url,
183
- source_id=task_id)
184
- elif radio_mode == "htex":
185
- radio = HTEXRadio(monitoring_hub_url,
186
- source_id=task_id)
187
- elif radio_mode == "filesystem":
188
- radio = FilesystemRadio(monitoring_url=monitoring_hub_url,
189
- source_id=task_id, run_dir=run_dir)
190
- else:
191
- raise RuntimeError(f"Unknown radio mode: {radio_mode}")
187
+ radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
192
188
 
193
189
  logging.debug("start of monitor")
194
190
 
@@ -199,10 +195,10 @@ def monitor(pid: int,
199
195
 
200
196
  pm = psutil.Process(pid)
201
197
 
202
- children_user_time = {} # type: Dict[int, float]
203
- children_system_time = {} # type: Dict[int, float]
204
- children_num_ctx_switches_voluntary = {} # type: Dict[int, float]
205
- children_num_ctx_switches_involuntary = {} # type: Dict[int, float]
198
+ children_user_time: Dict[int, float] = {}
199
+ children_system_time: Dict[int, float] = {}
200
+ children_num_ctx_switches_voluntary: Dict[int, float] = {}
201
+ children_num_ctx_switches_involuntary: Dict[int, float] = {}
206
202
 
207
203
  def accumulate_and_prepare() -> Dict[str, Any]:
208
204
  d = {"psutil_process_" + str(k): v for k, v in pm.as_dict().items() if k in simple}
@@ -0,0 +1,212 @@
1
+ from __future__ import annotations
2
+
3
+ import logging
4
+ import multiprocessing.queues as mpq
5
+ import os
6
+ import pickle
7
+ import socket
8
+ import threading
9
+ import time
10
+ from multiprocessing.synchronize import Event
11
+ from typing import Optional, Tuple
12
+
13
+ import typeguard
14
+ import zmq
15
+
16
+ from parsl.log_utils import set_file_logger
17
+ from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
18
+ from parsl.monitoring.types import TaggedMonitoringMessage
19
+ from parsl.process_loggers import wrap_with_logs
20
+ from parsl.utils import setproctitle
21
+
22
+ logger = logging.getLogger(__name__)
23
+
24
+
25
+ class MonitoringRouter:
26
+
27
+ def __init__(self,
28
+ *,
29
+ hub_address: str,
30
+ udp_port: Optional[int] = None,
31
+ zmq_port_range: Tuple[int, int] = (55050, 56000),
32
+
33
+ monitoring_hub_address: str = "127.0.0.1",
34
+ run_dir: str = ".",
35
+ logging_level: int = logging.INFO,
36
+ atexit_timeout: int = 3, # in seconds
37
+ resource_msgs: mpq.Queue,
38
+ exit_event: Event,
39
+ ):
40
+ """ Initializes a monitoring configuration class.
41
+
42
+ Parameters
43
+ ----------
44
+ hub_address : str
45
+ The ip address at which the workers will be able to reach the Hub.
46
+ udp_port : int
47
+ The specific port at which workers will be able to reach the Hub via UDP. Default: None
48
+ zmq_port_range : tuple(int, int)
49
+ The MonitoringHub picks ports at random from the range which will be used by Hub.
50
+ Default: (55050, 56000)
51
+ run_dir : str
52
+ Parsl log directory paths. Logs and temp files go here. Default: '.'
53
+ logging_level : int
54
+ Logging level as defined in the logging module. Default: logging.INFO
55
+ atexit_timeout : float, optional
56
+ The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
57
+ resource_msgs : multiprocessing.Queue
58
+ A multiprocessing queue to receive messages to be routed onwards to the database process
59
+ exit_event : Event
60
+ An event that the main Parsl process will set to signal that the monitoring router should shut down.
61
+ """
62
+ os.makedirs(run_dir, exist_ok=True)
63
+ self.logger = set_file_logger(f"{run_dir}/monitoring_router.log",
64
+ name="monitoring_router",
65
+ level=logging_level)
66
+ self.logger.debug("Monitoring router starting")
67
+
68
+ self.hub_address = hub_address
69
+ self.atexit_timeout = atexit_timeout
70
+
71
+ self.loop_freq = 10.0 # milliseconds
72
+
73
+ # Initialize the UDP socket
74
+ self.udp_sock = socket.socket(socket.AF_INET,
75
+ socket.SOCK_DGRAM,
76
+ socket.IPPROTO_UDP)
77
+
78
+ # We are trying to bind to all interfaces with 0.0.0.0
79
+ if not udp_port:
80
+ self.udp_sock.bind(('0.0.0.0', 0))
81
+ self.udp_port = self.udp_sock.getsockname()[1]
82
+ else:
83
+ self.udp_port = udp_port
84
+ try:
85
+ self.udp_sock.bind(('0.0.0.0', self.udp_port))
86
+ except Exception as e:
87
+ raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
88
+ self.udp_sock.settimeout(self.loop_freq / 1000)
89
+ self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
90
+
91
+ self._context = zmq.Context()
92
+ self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
93
+ self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
94
+ self.zmq_receiver_channel.set_hwm(0)
95
+ self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq) # in milliseconds
96
+ self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
97
+ self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
98
+ min_port=zmq_port_range[0],
99
+ max_port=zmq_port_range[1])
100
+
101
+ self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
102
+ self.exit_event = exit_event
103
+
104
+ @wrap_with_logs(target="monitoring_router")
105
+ def start(self) -> None:
106
+ self.logger.info("Starting UDP listener thread")
107
+ udp_radio_receiver_thread = threading.Thread(target=self.start_udp_listener, daemon=True)
108
+ udp_radio_receiver_thread.start()
109
+
110
+ self.logger.info("Starting ZMQ listener thread")
111
+ zmq_radio_receiver_thread = threading.Thread(target=self.start_zmq_listener, daemon=True)
112
+ zmq_radio_receiver_thread.start()
113
+
114
+ self.logger.info("Joining on ZMQ listener thread")
115
+ zmq_radio_receiver_thread.join()
116
+ self.logger.info("Joining on UDP listener thread")
117
+ udp_radio_receiver_thread.join()
118
+ self.logger.info("Joined on both ZMQ and UDP listener threads")
119
+
120
+ @wrap_with_logs(target="monitoring_router")
121
+ def start_udp_listener(self) -> None:
122
+ try:
123
+ while not self.exit_event.is_set():
124
+ try:
125
+ data, addr = self.udp_sock.recvfrom(2048)
126
+ resource_msg = pickle.loads(data)
127
+ self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
128
+ self.target_radio.send(resource_msg)
129
+ except socket.timeout:
130
+ pass
131
+
132
+ self.logger.info("UDP listener draining")
133
+ last_msg_received_time = time.time()
134
+ while time.time() - last_msg_received_time < self.atexit_timeout:
135
+ try:
136
+ data, addr = self.udp_sock.recvfrom(2048)
137
+ msg = pickle.loads(data)
138
+ self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
139
+ self.target_radio.send(msg)
140
+ last_msg_received_time = time.time()
141
+ except socket.timeout:
142
+ pass
143
+
144
+ self.logger.info("UDP listener finishing normally")
145
+ finally:
146
+ self.logger.info("UDP listener finished")
147
+
148
+ @wrap_with_logs(target="monitoring_router")
149
+ def start_zmq_listener(self) -> None:
150
+ try:
151
+ while not self.exit_event.is_set():
152
+ try:
153
+ dfk_loop_start = time.time()
154
+ while time.time() - dfk_loop_start < 1.0: # TODO make configurable
155
+ # note that nothing checks that msg really is of the annotated type
156
+ msg: TaggedMonitoringMessage
157
+ msg = self.zmq_receiver_channel.recv_pyobj()
158
+
159
+ assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
160
+ assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
161
+ assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
162
+
163
+ self.target_radio.send(msg)
164
+ except zmq.Again:
165
+ pass
166
+ except Exception:
167
+ # This will catch malformed messages. What happens if the
168
+ # channel is broken in such a way that it always raises
169
+ # an exception? Looping on this would maybe be the wrong
170
+ # thing to do.
171
+ self.logger.warning("Failure processing a ZMQ message", exc_info=True)
172
+
173
+ self.logger.info("ZMQ listener finishing normally")
174
+ finally:
175
+ self.logger.info("ZMQ listener finished")
176
+
177
+
178
+ @wrap_with_logs
179
+ @typeguard.typechecked
180
+ def router_starter(*,
181
+ comm_q: mpq.Queue,
182
+ exception_q: mpq.Queue,
183
+ resource_msgs: mpq.Queue,
184
+ exit_event: Event,
185
+
186
+ hub_address: str,
187
+ udp_port: Optional[int],
188
+ zmq_port_range: Tuple[int, int],
189
+
190
+ run_dir: str,
191
+ logging_level: int) -> None:
192
+ setproctitle("parsl: monitoring router")
193
+ try:
194
+ router = MonitoringRouter(hub_address=hub_address,
195
+ udp_port=udp_port,
196
+ zmq_port_range=zmq_port_range,
197
+ run_dir=run_dir,
198
+ logging_level=logging_level,
199
+ resource_msgs=resource_msgs,
200
+ exit_event=exit_event)
201
+ except Exception as e:
202
+ logger.error("MonitoringRouter construction failed.", exc_info=True)
203
+ comm_q.put(f"Monitoring router construction failed: {e}")
204
+ else:
205
+ comm_q.put((router.udp_port, router.zmq_receiver_port))
206
+
207
+ router.logger.info("Starting MonitoringRouter in router_starter")
208
+ try:
209
+ router.start()
210
+ except Exception as e:
211
+ router.logger.exception("router.start exception")
212
+ exception_q.put(('Hub', str(e)))
parsl/monitoring/types.py CHANGED
@@ -1,12 +1,11 @@
1
- from typing import Any, Dict, Tuple, Union
1
+ from typing import Any, Dict, Tuple
2
+
2
3
  from typing_extensions import TypeAlias
4
+
3
5
  from parsl.monitoring.message_type import MessageType
4
6
 
5
- # A basic parsl monitoring message is wrapped by up to two wrappers:
6
- # The basic monitoring message dictionary can first be tagged, giving
7
- # a TaggedMonitoringMessage, and then that can be further tagged with
8
- # an often unused sender address, giving an AddressedMonitoringMessage.
7
+ # A MonitoringMessage dictionary can be tagged, giving a
8
+ # TaggedMonitoringMessage.
9
9
 
10
10
  MonitoringMessage: TypeAlias = Dict[str, Any]
11
11
  TaggedMonitoringMessage: TypeAlias = Tuple[MessageType, MonitoringMessage]
12
- AddressedMonitoringMessage: TypeAlias = Tuple[TaggedMonitoringMessage, Union[str, int]]
@@ -1,8 +1,10 @@
1
- from flask import Flask
2
- from parsl.monitoring.visualization.models import db
3
1
  import argparse
4
2
  import os
5
3
 
4
+ from flask import Flask
5
+
6
+ from parsl.monitoring.visualization.models import db
7
+
6
8
 
7
9
  def cli_run():
8
10
  """ Instantiates the Monitoring viz server
@@ -1,6 +1,5 @@
1
1
  from flask_sqlalchemy import SQLAlchemy
2
2
 
3
-
4
3
  WORKFLOW = 'workflow' # Workflow table includes workflow metadata
5
4
  TASK = 'task' # Task table includes task metadata
6
5
  STATUS = 'status' # Status table includes task status
@@ -1,13 +1,17 @@
1
1
  import math
2
+
3
+ import networkx as nx
2
4
  import numpy as np
3
5
  import pandas as pd
4
- import plotly.graph_objs as go
5
6
  import plotly.figure_factory as ff
7
+ import plotly.graph_objs as go
6
8
  from plotly.offline import plot
7
- import networkx as nx
8
-
9
- from parsl.monitoring.visualization.utils import timestamp_to_int, num_to_timestamp, DB_DATE_FORMAT
10
9
 
10
+ from parsl.monitoring.visualization.utils import (
11
+ DB_DATE_FORMAT,
12
+ num_to_timestamp,
13
+ timestamp_to_int,
14
+ )
11
15
 
12
16
  # gantt_colors must assign a color value for every state name defined
13
17
  # in parsl/dataflow/states.py
@@ -27,6 +31,9 @@ gantt_colors = {'unsched': 'rgb(240, 240, 240)',
27
31
 
28
32
  def task_gantt_plot(df_task, df_status, time_completed=None):
29
33
 
34
+ if df_task.empty:
35
+ return None
36
+
30
37
  # if the workflow is not recorded as completed, then assume
31
38
  # that tasks should continue in their last state until now,
32
39
  # rather than the workflow end time.
@@ -1,4 +1,5 @@
1
1
  import math
2
+
2
3
  import numpy as np
3
4
  import pandas as pd
4
5
  import plotly.graph_objs as go
@@ -1,6 +1,5 @@
1
1
  from datetime import datetime
2
2
 
3
-
4
3
  DB_DATE_FORMAT = '%Y-%m-%d %H:%M:%S'
5
4
 
6
5
 
@@ -1,14 +1,22 @@
1
- from flask import render_template
2
- from flask import current_app as app
3
1
  import pandas as pd
2
+ from flask import current_app as app
3
+ from flask import render_template
4
4
 
5
5
  import parsl.monitoring.queries.pandas as queries
6
-
7
- from parsl.monitoring.visualization.models import Workflow, Task, Status, db
8
-
9
- from parsl.monitoring.visualization.plots.default.workflow_plots import task_gantt_plot, task_per_app_plot, workflow_dag_plot
10
- from parsl.monitoring.visualization.plots.default.task_plots import time_series_memory_per_task_plot
11
- from parsl.monitoring.visualization.plots.default.workflow_resource_plots import resource_distribution_plot, resource_efficiency, worker_efficiency
6
+ from parsl.monitoring.visualization.models import Status, Task, Workflow, db
7
+ from parsl.monitoring.visualization.plots.default.task_plots import (
8
+ time_series_memory_per_task_plot,
9
+ )
10
+ from parsl.monitoring.visualization.plots.default.workflow_plots import (
11
+ task_gantt_plot,
12
+ task_per_app_plot,
13
+ workflow_dag_plot,
14
+ )
15
+ from parsl.monitoring.visualization.plots.default.workflow_resource_plots import (
16
+ resource_distribution_plot,
17
+ resource_efficiency,
18
+ worker_efficiency,
19
+ )
12
20
 
13
21
  dummy = True
14
22
 
parsl/multiprocessing.py CHANGED
@@ -6,7 +6,6 @@ import multiprocessing
6
6
  import multiprocessing.queues
7
7
  import platform
8
8
  from multiprocessing.context import ForkProcess as ForkProcessType
9
-
10
9
  from typing import Callable
11
10
 
12
11
  logger = logging.getLogger(__name__)
parsl/process_loggers.py CHANGED
@@ -1,7 +1,6 @@
1
+ import functools
1
2
  import logging
2
3
  import threading
3
- import functools
4
-
5
4
  from typing import Callable, Optional
6
5
 
7
6
 
@@ -1,33 +1,24 @@
1
- # Workstation Provider
2
- from parsl.providers.local.local import LocalProvider
3
-
4
- # Cluster Providers
5
-
6
- from parsl.providers.cobalt.cobalt import CobaltProvider
7
- from parsl.providers.condor.condor import CondorProvider
8
- from parsl.providers.grid_engine.grid_engine import GridEngineProvider
9
- from parsl.providers.slurm.slurm import SlurmProvider
10
- from parsl.providers.torque.torque import TorqueProvider
11
- from parsl.providers.pbspro.pbspro import PBSProProvider
12
- from parsl.providers.lsf.lsf import LSFProvider
13
- from parsl.providers.ad_hoc.ad_hoc import AdHocProvider
14
-
15
1
  # Cloud Providers
16
2
  from parsl.providers.aws.aws import AWSProvider
17
- from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider
18
3
  from parsl.providers.azure.azure import AzureProvider
4
+ from parsl.providers.condor.condor import CondorProvider
5
+ from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider
6
+ from parsl.providers.grid_engine.grid_engine import GridEngineProvider
19
7
 
20
8
  # Kubernetes
21
9
  from parsl.providers.kubernetes.kube import KubernetesProvider
10
+ from parsl.providers.local.local import LocalProvider
11
+ from parsl.providers.lsf.lsf import LSFProvider
12
+ from parsl.providers.pbspro.pbspro import PBSProProvider
13
+ from parsl.providers.slurm.slurm import SlurmProvider
14
+ from parsl.providers.torque.torque import TorqueProvider
22
15
 
23
16
  __all__ = ['LocalProvider',
24
- 'CobaltProvider',
25
17
  'CondorProvider',
26
18
  'GridEngineProvider',
27
19
  'SlurmProvider',
28
20
  'TorqueProvider',
29
21
  'LSFProvider',
30
- 'AdHocProvider',
31
22
  'PBSProProvider',
32
23
  'AWSProvider',
33
24
  'GoogleCloudProvider',
@@ -4,13 +4,12 @@ import os
4
4
  import time
5
5
  from string import Template
6
6
 
7
- from parsl.errors import ConfigurationError
7
+ from parsl.errors import ConfigurationError, OptionalModuleMissing
8
8
  from parsl.jobs.states import JobState, JobStatus
9
+ from parsl.launchers import SingleNodeLauncher
9
10
  from parsl.providers.aws.template import template_string
10
11
  from parsl.providers.base import ExecutionProvider
11
- from parsl.errors import OptionalModuleMissing
12
12
  from parsl.utils import RepresentationMixin
13
- from parsl.launchers import SingleNodeLauncher
14
13
 
15
14
  logger = logging.getLogger(__name__)
16
15