parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsl/__init__.py +9 -10
- parsl/addresses.py +26 -6
- parsl/app/app.py +7 -8
- parsl/app/bash.py +15 -8
- parsl/app/errors.py +10 -13
- parsl/app/futures.py +8 -10
- parsl/app/python.py +2 -1
- parsl/benchmark/perf.py +2 -1
- parsl/concurrent/__init__.py +2 -2
- parsl/config.py +53 -10
- parsl/configs/ASPIRE1.py +6 -5
- parsl/configs/Azure.py +9 -8
- parsl/configs/bridges.py +6 -4
- parsl/configs/cc_in2p3.py +3 -3
- parsl/configs/ec2.py +3 -1
- parsl/configs/expanse.py +4 -3
- parsl/configs/frontera.py +3 -4
- parsl/configs/htex_local.py +3 -4
- parsl/configs/illinoiscluster.py +3 -1
- parsl/configs/improv.py +34 -0
- parsl/configs/kubernetes.py +4 -3
- parsl/configs/local_threads.py +5 -1
- parsl/configs/midway.py +5 -3
- parsl/configs/osg.py +4 -2
- parsl/configs/polaris.py +4 -2
- parsl/configs/stampede2.py +6 -5
- parsl/configs/summit.py +3 -3
- parsl/configs/toss3_llnl.py +4 -3
- parsl/configs/vineex_local.py +6 -4
- parsl/configs/wqex_local.py +5 -3
- parsl/curvezmq.py +4 -0
- parsl/data_provider/data_manager.py +4 -3
- parsl/data_provider/file_noop.py +1 -2
- parsl/data_provider/files.py +3 -3
- parsl/data_provider/ftp.py +1 -3
- parsl/data_provider/globus.py +7 -6
- parsl/data_provider/http.py +2 -2
- parsl/data_provider/rsync.py +1 -1
- parsl/data_provider/staging.py +2 -2
- parsl/data_provider/zip.py +135 -0
- parsl/dataflow/dependency_resolvers.py +115 -0
- parsl/dataflow/dflow.py +259 -223
- parsl/dataflow/errors.py +3 -5
- parsl/dataflow/futures.py +27 -14
- parsl/dataflow/memoization.py +5 -5
- parsl/dataflow/rundirs.py +5 -6
- parsl/dataflow/taskrecord.py +4 -5
- parsl/executors/__init__.py +4 -2
- parsl/executors/base.py +45 -15
- parsl/executors/errors.py +13 -0
- parsl/executors/execute_task.py +37 -0
- parsl/executors/flux/execute_parsl_task.py +3 -3
- parsl/executors/flux/executor.py +18 -19
- parsl/executors/flux/flux_instance_manager.py +26 -27
- parsl/executors/high_throughput/errors.py +43 -3
- parsl/executors/high_throughput/executor.py +307 -285
- parsl/executors/high_throughput/interchange.py +137 -168
- parsl/executors/high_throughput/manager_record.py +4 -0
- parsl/executors/high_throughput/manager_selector.py +55 -0
- parsl/executors/high_throughput/monitoring_info.py +2 -1
- parsl/executors/high_throughput/mpi_executor.py +113 -0
- parsl/executors/high_throughput/mpi_prefix_composer.py +10 -11
- parsl/executors/high_throughput/mpi_resource_management.py +6 -17
- parsl/executors/high_throughput/probe.py +9 -7
- parsl/executors/high_throughput/process_worker_pool.py +77 -75
- parsl/executors/high_throughput/zmq_pipes.py +81 -23
- parsl/executors/radical/executor.py +130 -79
- parsl/executors/radical/rpex_resources.py +17 -15
- parsl/executors/radical/rpex_worker.py +4 -3
- parsl/executors/status_handling.py +157 -51
- parsl/executors/taskvine/__init__.py +1 -1
- parsl/executors/taskvine/errors.py +1 -1
- parsl/executors/taskvine/exec_parsl_function.py +2 -2
- parsl/executors/taskvine/executor.py +38 -55
- parsl/executors/taskvine/factory.py +1 -1
- parsl/executors/taskvine/factory_config.py +1 -1
- parsl/executors/taskvine/manager.py +17 -13
- parsl/executors/taskvine/manager_config.py +7 -2
- parsl/executors/threads.py +6 -6
- parsl/executors/workqueue/errors.py +1 -1
- parsl/executors/workqueue/exec_parsl_function.py +6 -5
- parsl/executors/workqueue/executor.py +64 -63
- parsl/executors/workqueue/parsl_coprocess.py +1 -1
- parsl/jobs/error_handlers.py +2 -2
- parsl/jobs/job_status_poller.py +28 -112
- parsl/jobs/states.py +7 -2
- parsl/jobs/strategy.py +43 -31
- parsl/launchers/__init__.py +12 -3
- parsl/launchers/errors.py +1 -1
- parsl/launchers/launchers.py +0 -6
- parsl/log_utils.py +1 -2
- parsl/monitoring/db_manager.py +55 -93
- parsl/monitoring/errors.py +6 -0
- parsl/monitoring/monitoring.py +85 -311
- parsl/monitoring/queries/pandas.py +1 -2
- parsl/monitoring/radios/base.py +13 -0
- parsl/monitoring/radios/filesystem.py +52 -0
- parsl/monitoring/radios/htex.py +57 -0
- parsl/monitoring/radios/multiprocessing.py +17 -0
- parsl/monitoring/radios/udp.py +56 -0
- parsl/monitoring/radios/zmq.py +17 -0
- parsl/monitoring/remote.py +33 -37
- parsl/monitoring/router.py +212 -0
- parsl/monitoring/types.py +5 -6
- parsl/monitoring/visualization/app.py +4 -2
- parsl/monitoring/visualization/models.py +0 -1
- parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
- parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
- parsl/monitoring/visualization/utils.py +0 -1
- parsl/monitoring/visualization/views.py +16 -9
- parsl/multiprocessing.py +0 -1
- parsl/process_loggers.py +1 -2
- parsl/providers/__init__.py +8 -17
- parsl/providers/aws/aws.py +2 -3
- parsl/providers/azure/azure.py +4 -5
- parsl/providers/base.py +2 -18
- parsl/providers/cluster_provider.py +3 -9
- parsl/providers/condor/condor.py +7 -17
- parsl/providers/errors.py +2 -2
- parsl/providers/googlecloud/googlecloud.py +2 -1
- parsl/providers/grid_engine/grid_engine.py +5 -14
- parsl/providers/kubernetes/kube.py +80 -40
- parsl/providers/local/local.py +13 -26
- parsl/providers/lsf/lsf.py +5 -23
- parsl/providers/pbspro/pbspro.py +5 -17
- parsl/providers/slurm/slurm.py +81 -39
- parsl/providers/torque/torque.py +3 -14
- parsl/serialize/__init__.py +8 -3
- parsl/serialize/base.py +1 -2
- parsl/serialize/concretes.py +5 -4
- parsl/serialize/facade.py +3 -3
- parsl/serialize/proxystore.py +3 -2
- parsl/tests/__init__.py +1 -1
- parsl/tests/configs/azure_single_node.py +4 -5
- parsl/tests/configs/bridges.py +3 -2
- parsl/tests/configs/cc_in2p3.py +1 -3
- parsl/tests/configs/comet.py +2 -1
- parsl/tests/configs/ec2_single_node.py +1 -2
- parsl/tests/configs/ec2_spot.py +1 -2
- parsl/tests/configs/flux_local.py +11 -0
- parsl/tests/configs/frontera.py +2 -3
- parsl/tests/configs/htex_local.py +3 -5
- parsl/tests/configs/htex_local_alternate.py +11 -15
- parsl/tests/configs/htex_local_intask_staging.py +5 -9
- parsl/tests/configs/htex_local_rsync_staging.py +4 -8
- parsl/tests/configs/local_radical.py +1 -3
- parsl/tests/configs/local_radical_mpi.py +2 -2
- parsl/tests/configs/local_threads_checkpoint_periodic.py +8 -10
- parsl/tests/configs/local_threads_monitoring.py +0 -1
- parsl/tests/configs/midway.py +2 -2
- parsl/tests/configs/nscc_singapore.py +3 -3
- parsl/tests/configs/osg_htex.py +1 -1
- parsl/tests/configs/petrelkube.py +3 -2
- parsl/tests/configs/slurm_local.py +24 -0
- parsl/tests/configs/summit.py +1 -0
- parsl/tests/configs/taskvine_ex.py +4 -7
- parsl/tests/configs/user_opts.py +0 -7
- parsl/tests/configs/workqueue_ex.py +4 -6
- parsl/tests/conftest.py +27 -13
- parsl/tests/integration/test_stress/test_python_simple.py +3 -4
- parsl/tests/integration/test_stress/test_python_threads.py +3 -5
- parsl/tests/manual_tests/htex_local.py +4 -6
- parsl/tests/manual_tests/test_basic.py +1 -0
- parsl/tests/manual_tests/test_log_filter.py +3 -1
- parsl/tests/manual_tests/test_memory_limits.py +6 -8
- parsl/tests/manual_tests/test_regression_220.py +2 -1
- parsl/tests/manual_tests/test_udp_simple.py +4 -4
- parsl/tests/manual_tests/test_worker_count.py +3 -2
- parsl/tests/scaling_tests/htex_local.py +2 -4
- parsl/tests/scaling_tests/test_scale.py +0 -9
- parsl/tests/scaling_tests/vineex_condor.py +1 -2
- parsl/tests/scaling_tests/vineex_local.py +1 -2
- parsl/tests/site_tests/site_config_selector.py +1 -6
- parsl/tests/site_tests/test_provider.py +4 -2
- parsl/tests/site_tests/test_site.py +2 -0
- parsl/tests/sites/test_affinity.py +7 -7
- parsl/tests/sites/test_dynamic_executor.py +3 -4
- parsl/tests/sites/test_ec2.py +3 -2
- parsl/tests/sites/test_worker_info.py +4 -5
- parsl/tests/test_aalst_patterns.py +0 -1
- parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
- parsl/tests/test_bash_apps/test_basic.py +10 -4
- parsl/tests/test_bash_apps/test_error_codes.py +5 -7
- parsl/tests/test_bash_apps/test_inputs_default.py +25 -0
- parsl/tests/test_bash_apps/test_kwarg_storage.py +1 -1
- parsl/tests/test_bash_apps/test_memoize.py +2 -8
- parsl/tests/test_bash_apps/test_memoize_ignore_args.py +9 -14
- parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +9 -14
- parsl/tests/test_bash_apps/test_multiline.py +1 -1
- parsl/tests/test_bash_apps/test_pipeline.py +1 -1
- parsl/tests/test_bash_apps/test_std_uri.py +123 -0
- parsl/tests/test_bash_apps/test_stdout.py +33 -8
- parsl/tests/test_callables.py +2 -2
- parsl/tests/test_checkpointing/test_periodic.py +21 -39
- parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
- parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -2
- parsl/tests/test_checkpointing/test_python_checkpoint_3.py +0 -1
- parsl/tests/test_checkpointing/test_regression_239.py +1 -1
- parsl/tests/test_checkpointing/test_task_exit.py +2 -3
- parsl/tests/test_docs/test_from_slides.py +5 -2
- parsl/tests/test_docs/test_kwargs.py +4 -1
- parsl/tests/test_docs/test_tutorial_1.py +1 -2
- parsl/tests/test_docs/test_workflow1.py +2 -2
- parsl/tests/test_docs/test_workflow2.py +0 -1
- parsl/tests/test_error_handling/test_rand_fail.py +2 -2
- parsl/tests/test_error_handling/test_resource_spec.py +10 -12
- parsl/tests/test_error_handling/test_retries.py +6 -16
- parsl/tests/test_error_handling/test_retry_handler.py +1 -0
- parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
- parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
- parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
- parsl/tests/test_execute_task.py +29 -0
- parsl/tests/test_flux.py +1 -1
- parsl/tests/test_htex/test_basic.py +2 -3
- parsl/tests/test_htex/test_block_manager_selector_unit.py +20 -0
- parsl/tests/test_htex/test_command_client_timeout.py +66 -0
- parsl/tests/test_htex/test_connected_blocks.py +3 -2
- parsl/tests/test_htex/test_cpu_affinity_explicit.py +6 -10
- parsl/tests/test_htex/test_disconnected_blocks.py +6 -5
- parsl/tests/test_htex/test_disconnected_blocks_failing_provider.py +71 -0
- parsl/tests/test_htex/test_drain.py +11 -10
- parsl/tests/test_htex/test_htex.py +51 -25
- parsl/tests/test_htex/test_manager_failure.py +0 -1
- parsl/tests/test_htex/test_manager_selector_by_block.py +51 -0
- parsl/tests/test_htex/test_managers_command.py +36 -0
- parsl/tests/test_htex/test_missing_worker.py +2 -12
- parsl/tests/test_htex/test_multiple_disconnected_blocks.py +9 -9
- parsl/tests/test_htex/test_resource_spec_validation.py +45 -0
- parsl/tests/test_htex/test_zmq_binding.py +29 -8
- parsl/tests/test_monitoring/test_app_names.py +5 -5
- parsl/tests/test_monitoring/test_basic.py +73 -25
- parsl/tests/test_monitoring/test_db_locks.py +6 -4
- parsl/tests/test_monitoring/test_fuzz_zmq.py +19 -8
- parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +80 -0
- parsl/tests/test_monitoring/test_incomplete_futures.py +5 -4
- parsl/tests/test_monitoring/test_memoization_representation.py +4 -2
- parsl/tests/test_monitoring/test_stdouterr.py +134 -0
- parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
- parsl/tests/test_mpi_apps/test_bad_mpi_config.py +33 -26
- parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +28 -11
- parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
- parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
- parsl/tests/test_mpi_apps/test_mpiex.py +64 -0
- parsl/tests/test_mpi_apps/test_resource_spec.py +42 -49
- parsl/tests/test_providers/test_kubernetes_provider.py +102 -0
- parsl/tests/test_providers/test_local_provider.py +3 -132
- parsl/tests/test_providers/test_pbspro_template.py +2 -3
- parsl/tests/test_providers/test_slurm_template.py +2 -3
- parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
- parsl/tests/test_python_apps/test_context_manager.py +128 -0
- parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
- parsl/tests/test_python_apps/test_dependencies_deep.py +59 -0
- parsl/tests/test_python_apps/test_fail.py +0 -25
- parsl/tests/test_python_apps/test_futures.py +2 -1
- parsl/tests/test_python_apps/test_inputs_default.py +22 -0
- parsl/tests/test_python_apps/test_join.py +0 -1
- parsl/tests/test_python_apps/test_lifted.py +11 -7
- parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
- parsl/tests/test_python_apps/test_outputs.py +1 -1
- parsl/tests/test_python_apps/test_pluggable_future_resolution.py +161 -0
- parsl/tests/test_radical/test_mpi_funcs.py +1 -2
- parsl/tests/test_regression/test_1480.py +2 -1
- parsl/tests/test_regression/test_1653.py +2 -1
- parsl/tests/test_regression/test_226.py +1 -0
- parsl/tests/test_regression/test_2652.py +1 -0
- parsl/tests/test_regression/test_69a.py +0 -1
- parsl/tests/test_regression/test_854.py +4 -2
- parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
- parsl/tests/test_regression/test_98.py +0 -1
- parsl/tests/test_scaling/test_block_error_handler.py +9 -4
- parsl/tests/test_scaling/test_regression_1621.py +11 -15
- parsl/tests/test_scaling/test_regression_3568_scaledown_vs_MISSING.py +84 -0
- parsl/tests/test_scaling/test_regression_3696_oscillation.py +103 -0
- parsl/tests/test_scaling/test_scale_down.py +2 -5
- parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +5 -8
- parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +71 -0
- parsl/tests/test_scaling/test_shutdown_scalein.py +73 -0
- parsl/tests/test_scaling/test_worker_interchange_bad_messages_3262.py +90 -0
- parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
- parsl/tests/test_serialization/test_3495_deserialize_managerlost.py +47 -0
- parsl/tests/test_serialization/test_basic.py +2 -1
- parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
- parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
- parsl/tests/test_serialization/test_proxystore_configured.py +10 -6
- parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
- parsl/tests/test_shutdown/test_kill_monitoring.py +64 -0
- parsl/tests/test_staging/staging_provider.py +2 -2
- parsl/tests/test_staging/test_1316.py +3 -4
- parsl/tests/test_staging/test_docs_1.py +2 -1
- parsl/tests/test_staging/test_docs_2.py +2 -1
- parsl/tests/test_staging/test_elaborate_noop_file.py +2 -3
- parsl/tests/{test_data → test_staging}/test_file.py +6 -6
- parsl/tests/{test_data → test_staging}/test_output_chain_filenames.py +3 -0
- parsl/tests/test_staging/test_staging_ftp.py +1 -0
- parsl/tests/test_staging/test_staging_https.py +5 -2
- parsl/tests/test_staging/test_staging_stdout.py +64 -0
- parsl/tests/test_staging/test_zip_in.py +39 -0
- parsl/tests/test_staging/test_zip_out.py +110 -0
- parsl/tests/test_staging/test_zip_to_zip.py +41 -0
- parsl/tests/test_summary.py +2 -2
- parsl/tests/test_thread_parallelism.py +0 -1
- parsl/tests/test_threads/test_configs.py +1 -2
- parsl/tests/test_threads/test_lazy_errors.py +2 -2
- parsl/tests/test_utils/test_execute_wait.py +35 -0
- parsl/tests/test_utils/test_sanitize_dns.py +76 -0
- parsl/tests/unit/test_address.py +20 -0
- parsl/tests/unit/test_file.py +99 -0
- parsl/tests/unit/test_usage_tracking.py +66 -0
- parsl/usage_tracking/api.py +65 -0
- parsl/usage_tracking/levels.py +6 -0
- parsl/usage_tracking/usage.py +104 -62
- parsl/utils.py +137 -4
- parsl/version.py +1 -1
- {parsl-2024.3.18.data → parsl-2025.1.13.data}/scripts/exec_parsl_function.py +6 -5
- parsl-2025.1.13.data/scripts/interchange.py +649 -0
- {parsl-2024.3.18.data → parsl-2025.1.13.data}/scripts/process_worker_pool.py +77 -75
- parsl-2025.1.13.dist-info/METADATA +96 -0
- parsl-2025.1.13.dist-info/RECORD +462 -0
- {parsl-2024.3.18.dist-info → parsl-2025.1.13.dist-info}/WHEEL +1 -1
- parsl/channels/__init__.py +0 -7
- parsl/channels/base.py +0 -141
- parsl/channels/errors.py +0 -113
- parsl/channels/local/local.py +0 -164
- parsl/channels/oauth_ssh/oauth_ssh.py +0 -110
- parsl/channels/ssh/ssh.py +0 -276
- parsl/channels/ssh_il/__init__.py +0 -0
- parsl/channels/ssh_il/ssh_il.py +0 -74
- parsl/configs/ad_hoc.py +0 -35
- parsl/executors/radical/rpex_master.py +0 -42
- parsl/monitoring/radios.py +0 -175
- parsl/providers/ad_hoc/__init__.py +0 -0
- parsl/providers/ad_hoc/ad_hoc.py +0 -248
- parsl/providers/cobalt/__init__.py +0 -0
- parsl/providers/cobalt/cobalt.py +0 -236
- parsl/providers/cobalt/template.py +0 -17
- parsl/tests/configs/ad_hoc_cluster_htex.py +0 -35
- parsl/tests/configs/cooley_htex.py +0 -37
- parsl/tests/configs/htex_ad_hoc_cluster.py +0 -28
- parsl/tests/configs/local_adhoc.py +0 -18
- parsl/tests/configs/swan_htex.py +0 -43
- parsl/tests/configs/theta.py +0 -37
- parsl/tests/integration/test_channels/__init__.py +0 -0
- parsl/tests/integration/test_channels/test_channels.py +0 -17
- parsl/tests/integration/test_channels/test_local_channel.py +0 -42
- parsl/tests/integration/test_channels/test_scp_1.py +0 -45
- parsl/tests/integration/test_channels/test_ssh_1.py +0 -40
- parsl/tests/integration/test_channels/test_ssh_errors.py +0 -46
- parsl/tests/integration/test_channels/test_ssh_file_transport.py +0 -41
- parsl/tests/integration/test_channels/test_ssh_interactive.py +0 -24
- parsl/tests/manual_tests/test_ad_hoc_htex.py +0 -48
- parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +0 -88
- parsl/tests/manual_tests/test_oauth_ssh.py +0 -13
- parsl/tests/sites/test_local_adhoc.py +0 -61
- parsl/tests/test_channels/__init__.py +0 -0
- parsl/tests/test_channels/test_large_output.py +0 -22
- parsl/tests/test_data/__init__.py +0 -0
- parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +0 -51
- parsl/tests/test_providers/test_cobalt_deprecation_warning.py +0 -16
- parsl-2024.3.18.dist-info/METADATA +0 -98
- parsl-2024.3.18.dist-info/RECORD +0 -449
- parsl/{channels/local → monitoring/radios}/__init__.py +0 -0
- parsl/{channels/oauth_ssh → tests/test_shutdown}/__init__.py +0 -0
- parsl/tests/{test_data → test_staging}/test_file_apps.py +0 -0
- parsl/tests/{test_data → test_staging}/test_file_staging.py +0 -0
- parsl/{channels/ssh → tests/unit}/__init__.py +0 -0
- {parsl-2024.3.18.data → parsl-2025.1.13.data}/scripts/parsl_coprocess.py +1 -1
- {parsl-2024.3.18.dist-info → parsl-2025.1.13.dist-info}/LICENSE +0 -0
- {parsl-2024.3.18.dist-info → parsl-2025.1.13.dist-info}/entry_points.txt +0 -0
- {parsl-2024.3.18.dist-info → parsl-2025.1.13.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,17 @@
|
|
1
|
+
from multiprocessing.queues import Queue
|
2
|
+
|
3
|
+
from parsl.monitoring.radios.base import MonitoringRadioSender
|
4
|
+
|
5
|
+
|
6
|
+
class MultiprocessingQueueRadioSender(MonitoringRadioSender):
|
7
|
+
"""A monitoring radio which connects over a multiprocessing Queue.
|
8
|
+
This radio is intended to be used on the submit side, where components
|
9
|
+
in the submit process, or processes launched by multiprocessing, will have
|
10
|
+
access to a Queue shared with the monitoring database code (bypassing the
|
11
|
+
monitoring router).
|
12
|
+
"""
|
13
|
+
def __init__(self, queue: Queue) -> None:
|
14
|
+
self.queue = queue
|
15
|
+
|
16
|
+
def send(self, message: object) -> None:
|
17
|
+
self.queue.put(message)
|
@@ -0,0 +1,56 @@
|
|
1
|
+
import logging
|
2
|
+
import pickle
|
3
|
+
import socket
|
4
|
+
|
5
|
+
from parsl.monitoring.radios.base import MonitoringRadioSender
|
6
|
+
|
7
|
+
|
8
|
+
class UDPRadioSender(MonitoringRadioSender):
|
9
|
+
|
10
|
+
def __init__(self, monitoring_url: str, timeout: int = 10):
|
11
|
+
"""
|
12
|
+
Parameters
|
13
|
+
----------
|
14
|
+
|
15
|
+
monitoring_url : str
|
16
|
+
URL of the form <scheme>://<IP>:<PORT>
|
17
|
+
timeout : int
|
18
|
+
timeout, default=10s
|
19
|
+
"""
|
20
|
+
self.monitoring_url = monitoring_url
|
21
|
+
self.sock_timeout = timeout
|
22
|
+
try:
|
23
|
+
self.scheme, self.ip, port = (x.strip('/') for x in monitoring_url.split(':'))
|
24
|
+
self.port = int(port)
|
25
|
+
except Exception:
|
26
|
+
raise Exception("Failed to parse monitoring url: {}".format(monitoring_url))
|
27
|
+
|
28
|
+
self.sock = socket.socket(socket.AF_INET,
|
29
|
+
socket.SOCK_DGRAM,
|
30
|
+
socket.IPPROTO_UDP) # UDP
|
31
|
+
self.sock.settimeout(self.sock_timeout)
|
32
|
+
|
33
|
+
def send(self, message: object) -> None:
|
34
|
+
""" Sends a message to the UDP receiver
|
35
|
+
|
36
|
+
Parameter
|
37
|
+
---------
|
38
|
+
|
39
|
+
message: object
|
40
|
+
Arbitrary pickle-able object that is to be sent
|
41
|
+
|
42
|
+
Returns:
|
43
|
+
None
|
44
|
+
"""
|
45
|
+
try:
|
46
|
+
buffer = pickle.dumps(message)
|
47
|
+
except Exception:
|
48
|
+
logging.exception("Exception during pickling", exc_info=True)
|
49
|
+
return
|
50
|
+
|
51
|
+
try:
|
52
|
+
self.sock.sendto(buffer, (self.ip, self.port))
|
53
|
+
except socket.timeout:
|
54
|
+
logging.error("Could not send message within timeout limit")
|
55
|
+
return
|
56
|
+
return
|
@@ -0,0 +1,17 @@
|
|
1
|
+
import zmq
|
2
|
+
|
3
|
+
from parsl.monitoring.radios.base import MonitoringRadioSender
|
4
|
+
|
5
|
+
|
6
|
+
class ZMQRadioSender(MonitoringRadioSender):
|
7
|
+
"""A monitoring radio which connects over ZMQ. This radio is not
|
8
|
+
thread-safe, because its use of ZMQ is not thread-safe.
|
9
|
+
"""
|
10
|
+
|
11
|
+
def __init__(self, hub_address: str, hub_zmq_port: int) -> None:
|
12
|
+
self._hub_channel = zmq.Context().socket(zmq.DEALER)
|
13
|
+
self._hub_channel.set_hwm(0)
|
14
|
+
self._hub_channel.connect(f"tcp://{hub_address}:{hub_zmq_port}")
|
15
|
+
|
16
|
+
def send(self, message: object) -> None:
|
17
|
+
self._hub_channel.send_pyobj(message)
|
parsl/monitoring/remote.py
CHANGED
@@ -1,21 +1,24 @@
|
|
1
|
+
import datetime
|
2
|
+
import logging
|
1
3
|
import os
|
2
4
|
import time
|
3
|
-
import logging
|
4
|
-
import datetime
|
5
5
|
from functools import wraps
|
6
|
-
|
7
|
-
from parsl.multiprocessing import ForkProcess
|
8
6
|
from multiprocessing import Event
|
9
|
-
from
|
7
|
+
from typing import Any, Callable, Dict, List, Sequence, Tuple
|
10
8
|
|
11
9
|
from parsl.monitoring.message_type import MessageType
|
12
|
-
from parsl.monitoring.radios import
|
13
|
-
from
|
10
|
+
from parsl.monitoring.radios.base import MonitoringRadioSender
|
11
|
+
from parsl.monitoring.radios.filesystem import FilesystemRadioSender
|
12
|
+
from parsl.monitoring.radios.htex import HTEXRadioSender
|
13
|
+
from parsl.monitoring.radios.udp import UDPRadioSender
|
14
|
+
from parsl.multiprocessing import ForkProcess
|
15
|
+
from parsl.process_loggers import wrap_with_logs
|
14
16
|
|
15
17
|
logger = logging.getLogger(__name__)
|
16
18
|
|
17
19
|
|
18
|
-
def monitor_wrapper(
|
20
|
+
def monitor_wrapper(*,
|
21
|
+
f: Any, # per app
|
19
22
|
args: Sequence, # per invocation
|
20
23
|
kwargs: Dict, # per invocation
|
21
24
|
x_try_id: int, # per invocation
|
@@ -95,6 +98,20 @@ def monitor_wrapper(f: Any, # per app
|
|
95
98
|
return (wrapped, args, new_kwargs)
|
96
99
|
|
97
100
|
|
101
|
+
def get_radio(radio_mode: str, monitoring_hub_url: str, task_id: int, run_dir: str) -> MonitoringRadioSender:
|
102
|
+
radio: MonitoringRadioSender
|
103
|
+
if radio_mode == "udp":
|
104
|
+
radio = UDPRadioSender(monitoring_hub_url)
|
105
|
+
elif radio_mode == "htex":
|
106
|
+
radio = HTEXRadioSender(monitoring_hub_url)
|
107
|
+
elif radio_mode == "filesystem":
|
108
|
+
radio = FilesystemRadioSender(monitoring_url=monitoring_hub_url,
|
109
|
+
run_dir=run_dir)
|
110
|
+
else:
|
111
|
+
raise RuntimeError(f"Unknown radio mode: {radio_mode}")
|
112
|
+
return radio
|
113
|
+
|
114
|
+
|
98
115
|
@wrap_with_logs
|
99
116
|
def send_first_message(try_id: int,
|
100
117
|
task_id: int,
|
@@ -118,21 +135,10 @@ def send_first_last_message(try_id: int,
|
|
118
135
|
monitoring_hub_url: str,
|
119
136
|
run_id: str, radio_mode: str, run_dir: str,
|
120
137
|
is_last: bool) -> None:
|
121
|
-
import platform
|
122
138
|
import os
|
139
|
+
import platform
|
123
140
|
|
124
|
-
radio
|
125
|
-
if radio_mode == "udp":
|
126
|
-
radio = UDPRadio(monitoring_hub_url,
|
127
|
-
source_id=task_id)
|
128
|
-
elif radio_mode == "htex":
|
129
|
-
radio = HTEXRadio(monitoring_hub_url,
|
130
|
-
source_id=task_id)
|
131
|
-
elif radio_mode == "filesystem":
|
132
|
-
radio = FilesystemRadio(monitoring_url=monitoring_hub_url,
|
133
|
-
source_id=task_id, run_dir=run_dir)
|
134
|
-
else:
|
135
|
-
raise RuntimeError(f"Unknown radio mode: {radio_mode}")
|
141
|
+
radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
|
136
142
|
|
137
143
|
msg = (MessageType.RESOURCE_INFO,
|
138
144
|
{'run_id': run_id,
|
@@ -171,24 +177,14 @@ def monitor(pid: int,
|
|
171
177
|
"""
|
172
178
|
import logging
|
173
179
|
import platform
|
180
|
+
|
174
181
|
import psutil
|
175
182
|
|
176
183
|
from parsl.utils import setproctitle
|
177
184
|
|
178
185
|
setproctitle("parsl: task resource monitor")
|
179
186
|
|
180
|
-
radio
|
181
|
-
if radio_mode == "udp":
|
182
|
-
radio = UDPRadio(monitoring_hub_url,
|
183
|
-
source_id=task_id)
|
184
|
-
elif radio_mode == "htex":
|
185
|
-
radio = HTEXRadio(monitoring_hub_url,
|
186
|
-
source_id=task_id)
|
187
|
-
elif radio_mode == "filesystem":
|
188
|
-
radio = FilesystemRadio(monitoring_url=monitoring_hub_url,
|
189
|
-
source_id=task_id, run_dir=run_dir)
|
190
|
-
else:
|
191
|
-
raise RuntimeError(f"Unknown radio mode: {radio_mode}")
|
187
|
+
radio = get_radio(radio_mode, monitoring_hub_url, task_id, run_dir)
|
192
188
|
|
193
189
|
logging.debug("start of monitor")
|
194
190
|
|
@@ -199,10 +195,10 @@ def monitor(pid: int,
|
|
199
195
|
|
200
196
|
pm = psutil.Process(pid)
|
201
197
|
|
202
|
-
children_user_time
|
203
|
-
children_system_time
|
204
|
-
children_num_ctx_switches_voluntary
|
205
|
-
children_num_ctx_switches_involuntary
|
198
|
+
children_user_time: Dict[int, float] = {}
|
199
|
+
children_system_time: Dict[int, float] = {}
|
200
|
+
children_num_ctx_switches_voluntary: Dict[int, float] = {}
|
201
|
+
children_num_ctx_switches_involuntary: Dict[int, float] = {}
|
206
202
|
|
207
203
|
def accumulate_and_prepare() -> Dict[str, Any]:
|
208
204
|
d = {"psutil_process_" + str(k): v for k, v in pm.as_dict().items() if k in simple}
|
@@ -0,0 +1,212 @@
|
|
1
|
+
from __future__ import annotations
|
2
|
+
|
3
|
+
import logging
|
4
|
+
import multiprocessing.queues as mpq
|
5
|
+
import os
|
6
|
+
import pickle
|
7
|
+
import socket
|
8
|
+
import threading
|
9
|
+
import time
|
10
|
+
from multiprocessing.synchronize import Event
|
11
|
+
from typing import Optional, Tuple
|
12
|
+
|
13
|
+
import typeguard
|
14
|
+
import zmq
|
15
|
+
|
16
|
+
from parsl.log_utils import set_file_logger
|
17
|
+
from parsl.monitoring.radios.multiprocessing import MultiprocessingQueueRadioSender
|
18
|
+
from parsl.monitoring.types import TaggedMonitoringMessage
|
19
|
+
from parsl.process_loggers import wrap_with_logs
|
20
|
+
from parsl.utils import setproctitle
|
21
|
+
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
|
24
|
+
|
25
|
+
class MonitoringRouter:
|
26
|
+
|
27
|
+
def __init__(self,
|
28
|
+
*,
|
29
|
+
hub_address: str,
|
30
|
+
udp_port: Optional[int] = None,
|
31
|
+
zmq_port_range: Tuple[int, int] = (55050, 56000),
|
32
|
+
|
33
|
+
monitoring_hub_address: str = "127.0.0.1",
|
34
|
+
run_dir: str = ".",
|
35
|
+
logging_level: int = logging.INFO,
|
36
|
+
atexit_timeout: int = 3, # in seconds
|
37
|
+
resource_msgs: mpq.Queue,
|
38
|
+
exit_event: Event,
|
39
|
+
):
|
40
|
+
""" Initializes a monitoring configuration class.
|
41
|
+
|
42
|
+
Parameters
|
43
|
+
----------
|
44
|
+
hub_address : str
|
45
|
+
The ip address at which the workers will be able to reach the Hub.
|
46
|
+
udp_port : int
|
47
|
+
The specific port at which workers will be able to reach the Hub via UDP. Default: None
|
48
|
+
zmq_port_range : tuple(int, int)
|
49
|
+
The MonitoringHub picks ports at random from the range which will be used by Hub.
|
50
|
+
Default: (55050, 56000)
|
51
|
+
run_dir : str
|
52
|
+
Parsl log directory paths. Logs and temp files go here. Default: '.'
|
53
|
+
logging_level : int
|
54
|
+
Logging level as defined in the logging module. Default: logging.INFO
|
55
|
+
atexit_timeout : float, optional
|
56
|
+
The amount of time in seconds to terminate the hub without receiving any messages, after the last dfk workflow message is received.
|
57
|
+
resource_msgs : multiprocessing.Queue
|
58
|
+
A multiprocessing queue to receive messages to be routed onwards to the database process
|
59
|
+
exit_event : Event
|
60
|
+
An event that the main Parsl process will set to signal that the monitoring router should shut down.
|
61
|
+
"""
|
62
|
+
os.makedirs(run_dir, exist_ok=True)
|
63
|
+
self.logger = set_file_logger(f"{run_dir}/monitoring_router.log",
|
64
|
+
name="monitoring_router",
|
65
|
+
level=logging_level)
|
66
|
+
self.logger.debug("Monitoring router starting")
|
67
|
+
|
68
|
+
self.hub_address = hub_address
|
69
|
+
self.atexit_timeout = atexit_timeout
|
70
|
+
|
71
|
+
self.loop_freq = 10.0 # milliseconds
|
72
|
+
|
73
|
+
# Initialize the UDP socket
|
74
|
+
self.udp_sock = socket.socket(socket.AF_INET,
|
75
|
+
socket.SOCK_DGRAM,
|
76
|
+
socket.IPPROTO_UDP)
|
77
|
+
|
78
|
+
# We are trying to bind to all interfaces with 0.0.0.0
|
79
|
+
if not udp_port:
|
80
|
+
self.udp_sock.bind(('0.0.0.0', 0))
|
81
|
+
self.udp_port = self.udp_sock.getsockname()[1]
|
82
|
+
else:
|
83
|
+
self.udp_port = udp_port
|
84
|
+
try:
|
85
|
+
self.udp_sock.bind(('0.0.0.0', self.udp_port))
|
86
|
+
except Exception as e:
|
87
|
+
raise RuntimeError(f"Could not bind to udp_port {udp_port} because: {e}")
|
88
|
+
self.udp_sock.settimeout(self.loop_freq / 1000)
|
89
|
+
self.logger.info("Initialized the UDP socket on 0.0.0.0:{}".format(self.udp_port))
|
90
|
+
|
91
|
+
self._context = zmq.Context()
|
92
|
+
self.zmq_receiver_channel = self._context.socket(zmq.DEALER)
|
93
|
+
self.zmq_receiver_channel.setsockopt(zmq.LINGER, 0)
|
94
|
+
self.zmq_receiver_channel.set_hwm(0)
|
95
|
+
self.zmq_receiver_channel.RCVTIMEO = int(self.loop_freq) # in milliseconds
|
96
|
+
self.logger.debug("hub_address: {}. zmq_port_range {}".format(hub_address, zmq_port_range))
|
97
|
+
self.zmq_receiver_port = self.zmq_receiver_channel.bind_to_random_port("tcp://*",
|
98
|
+
min_port=zmq_port_range[0],
|
99
|
+
max_port=zmq_port_range[1])
|
100
|
+
|
101
|
+
self.target_radio = MultiprocessingQueueRadioSender(resource_msgs)
|
102
|
+
self.exit_event = exit_event
|
103
|
+
|
104
|
+
@wrap_with_logs(target="monitoring_router")
|
105
|
+
def start(self) -> None:
|
106
|
+
self.logger.info("Starting UDP listener thread")
|
107
|
+
udp_radio_receiver_thread = threading.Thread(target=self.start_udp_listener, daemon=True)
|
108
|
+
udp_radio_receiver_thread.start()
|
109
|
+
|
110
|
+
self.logger.info("Starting ZMQ listener thread")
|
111
|
+
zmq_radio_receiver_thread = threading.Thread(target=self.start_zmq_listener, daemon=True)
|
112
|
+
zmq_radio_receiver_thread.start()
|
113
|
+
|
114
|
+
self.logger.info("Joining on ZMQ listener thread")
|
115
|
+
zmq_radio_receiver_thread.join()
|
116
|
+
self.logger.info("Joining on UDP listener thread")
|
117
|
+
udp_radio_receiver_thread.join()
|
118
|
+
self.logger.info("Joined on both ZMQ and UDP listener threads")
|
119
|
+
|
120
|
+
@wrap_with_logs(target="monitoring_router")
|
121
|
+
def start_udp_listener(self) -> None:
|
122
|
+
try:
|
123
|
+
while not self.exit_event.is_set():
|
124
|
+
try:
|
125
|
+
data, addr = self.udp_sock.recvfrom(2048)
|
126
|
+
resource_msg = pickle.loads(data)
|
127
|
+
self.logger.debug("Got UDP Message from {}: {}".format(addr, resource_msg))
|
128
|
+
self.target_radio.send(resource_msg)
|
129
|
+
except socket.timeout:
|
130
|
+
pass
|
131
|
+
|
132
|
+
self.logger.info("UDP listener draining")
|
133
|
+
last_msg_received_time = time.time()
|
134
|
+
while time.time() - last_msg_received_time < self.atexit_timeout:
|
135
|
+
try:
|
136
|
+
data, addr = self.udp_sock.recvfrom(2048)
|
137
|
+
msg = pickle.loads(data)
|
138
|
+
self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
|
139
|
+
self.target_radio.send(msg)
|
140
|
+
last_msg_received_time = time.time()
|
141
|
+
except socket.timeout:
|
142
|
+
pass
|
143
|
+
|
144
|
+
self.logger.info("UDP listener finishing normally")
|
145
|
+
finally:
|
146
|
+
self.logger.info("UDP listener finished")
|
147
|
+
|
148
|
+
@wrap_with_logs(target="monitoring_router")
|
149
|
+
def start_zmq_listener(self) -> None:
|
150
|
+
try:
|
151
|
+
while not self.exit_event.is_set():
|
152
|
+
try:
|
153
|
+
dfk_loop_start = time.time()
|
154
|
+
while time.time() - dfk_loop_start < 1.0: # TODO make configurable
|
155
|
+
# note that nothing checks that msg really is of the annotated type
|
156
|
+
msg: TaggedMonitoringMessage
|
157
|
+
msg = self.zmq_receiver_channel.recv_pyobj()
|
158
|
+
|
159
|
+
assert isinstance(msg, tuple), "ZMQ Receiver expects only tuples, got {}".format(msg)
|
160
|
+
assert len(msg) >= 1, "ZMQ Receiver expects tuples of length at least 1, got {}".format(msg)
|
161
|
+
assert len(msg) == 2, "ZMQ Receiver expects message tuples of exactly length 2, got {}".format(msg)
|
162
|
+
|
163
|
+
self.target_radio.send(msg)
|
164
|
+
except zmq.Again:
|
165
|
+
pass
|
166
|
+
except Exception:
|
167
|
+
# This will catch malformed messages. What happens if the
|
168
|
+
# channel is broken in such a way that it always raises
|
169
|
+
# an exception? Looping on this would maybe be the wrong
|
170
|
+
# thing to do.
|
171
|
+
self.logger.warning("Failure processing a ZMQ message", exc_info=True)
|
172
|
+
|
173
|
+
self.logger.info("ZMQ listener finishing normally")
|
174
|
+
finally:
|
175
|
+
self.logger.info("ZMQ listener finished")
|
176
|
+
|
177
|
+
|
178
|
+
@wrap_with_logs
|
179
|
+
@typeguard.typechecked
|
180
|
+
def router_starter(*,
|
181
|
+
comm_q: mpq.Queue,
|
182
|
+
exception_q: mpq.Queue,
|
183
|
+
resource_msgs: mpq.Queue,
|
184
|
+
exit_event: Event,
|
185
|
+
|
186
|
+
hub_address: str,
|
187
|
+
udp_port: Optional[int],
|
188
|
+
zmq_port_range: Tuple[int, int],
|
189
|
+
|
190
|
+
run_dir: str,
|
191
|
+
logging_level: int) -> None:
|
192
|
+
setproctitle("parsl: monitoring router")
|
193
|
+
try:
|
194
|
+
router = MonitoringRouter(hub_address=hub_address,
|
195
|
+
udp_port=udp_port,
|
196
|
+
zmq_port_range=zmq_port_range,
|
197
|
+
run_dir=run_dir,
|
198
|
+
logging_level=logging_level,
|
199
|
+
resource_msgs=resource_msgs,
|
200
|
+
exit_event=exit_event)
|
201
|
+
except Exception as e:
|
202
|
+
logger.error("MonitoringRouter construction failed.", exc_info=True)
|
203
|
+
comm_q.put(f"Monitoring router construction failed: {e}")
|
204
|
+
else:
|
205
|
+
comm_q.put((router.udp_port, router.zmq_receiver_port))
|
206
|
+
|
207
|
+
router.logger.info("Starting MonitoringRouter in router_starter")
|
208
|
+
try:
|
209
|
+
router.start()
|
210
|
+
except Exception as e:
|
211
|
+
router.logger.exception("router.start exception")
|
212
|
+
exception_q.put(('Hub', str(e)))
|
parsl/monitoring/types.py
CHANGED
@@ -1,12 +1,11 @@
|
|
1
|
-
from typing import Any, Dict, Tuple
|
1
|
+
from typing import Any, Dict, Tuple
|
2
|
+
|
2
3
|
from typing_extensions import TypeAlias
|
4
|
+
|
3
5
|
from parsl.monitoring.message_type import MessageType
|
4
6
|
|
5
|
-
# A
|
6
|
-
#
|
7
|
-
# a TaggedMonitoringMessage, and then that can be further tagged with
|
8
|
-
# an often unused sender address, giving an AddressedMonitoringMessage.
|
7
|
+
# A MonitoringMessage dictionary can be tagged, giving a
|
8
|
+
# TaggedMonitoringMessage.
|
9
9
|
|
10
10
|
MonitoringMessage: TypeAlias = Dict[str, Any]
|
11
11
|
TaggedMonitoringMessage: TypeAlias = Tuple[MessageType, MonitoringMessage]
|
12
|
-
AddressedMonitoringMessage: TypeAlias = Tuple[TaggedMonitoringMessage, Union[str, int]]
|
@@ -1,8 +1,10 @@
|
|
1
|
-
from flask import Flask
|
2
|
-
from parsl.monitoring.visualization.models import db
|
3
1
|
import argparse
|
4
2
|
import os
|
5
3
|
|
4
|
+
from flask import Flask
|
5
|
+
|
6
|
+
from parsl.monitoring.visualization.models import db
|
7
|
+
|
6
8
|
|
7
9
|
def cli_run():
|
8
10
|
""" Instantiates the Monitoring viz server
|
@@ -1,13 +1,17 @@
|
|
1
1
|
import math
|
2
|
+
|
3
|
+
import networkx as nx
|
2
4
|
import numpy as np
|
3
5
|
import pandas as pd
|
4
|
-
import plotly.graph_objs as go
|
5
6
|
import plotly.figure_factory as ff
|
7
|
+
import plotly.graph_objs as go
|
6
8
|
from plotly.offline import plot
|
7
|
-
import networkx as nx
|
8
|
-
|
9
|
-
from parsl.monitoring.visualization.utils import timestamp_to_int, num_to_timestamp, DB_DATE_FORMAT
|
10
9
|
|
10
|
+
from parsl.monitoring.visualization.utils import (
|
11
|
+
DB_DATE_FORMAT,
|
12
|
+
num_to_timestamp,
|
13
|
+
timestamp_to_int,
|
14
|
+
)
|
11
15
|
|
12
16
|
# gantt_colors must assign a color value for every state name defined
|
13
17
|
# in parsl/dataflow/states.py
|
@@ -1,15 +1,22 @@
|
|
1
|
-
from flask import render_template
|
2
|
-
from flask import current_app as app
|
3
1
|
import pandas as pd
|
2
|
+
from flask import current_app as app
|
3
|
+
from flask import render_template
|
4
4
|
|
5
5
|
import parsl.monitoring.queries.pandas as queries
|
6
|
-
|
7
|
-
from parsl.monitoring.visualization.
|
8
|
-
|
9
|
-
|
10
|
-
from parsl.monitoring.visualization.plots.default.
|
11
|
-
|
12
|
-
|
6
|
+
from parsl.monitoring.visualization.models import Status, Task, Workflow, db
|
7
|
+
from parsl.monitoring.visualization.plots.default.task_plots import (
|
8
|
+
time_series_memory_per_task_plot,
|
9
|
+
)
|
10
|
+
from parsl.monitoring.visualization.plots.default.workflow_plots import (
|
11
|
+
task_gantt_plot,
|
12
|
+
task_per_app_plot,
|
13
|
+
workflow_dag_plot,
|
14
|
+
)
|
15
|
+
from parsl.monitoring.visualization.plots.default.workflow_resource_plots import (
|
16
|
+
resource_distribution_plot,
|
17
|
+
resource_efficiency,
|
18
|
+
worker_efficiency,
|
19
|
+
)
|
13
20
|
|
14
21
|
dummy = True
|
15
22
|
|
parsl/multiprocessing.py
CHANGED
parsl/process_loggers.py
CHANGED
parsl/providers/__init__.py
CHANGED
@@ -1,33 +1,24 @@
|
|
1
|
-
# Workstation Provider
|
2
|
-
from parsl.providers.local.local import LocalProvider
|
3
|
-
|
4
|
-
# Cluster Providers
|
5
|
-
|
6
|
-
from parsl.providers.cobalt.cobalt import CobaltProvider
|
7
|
-
from parsl.providers.condor.condor import CondorProvider
|
8
|
-
from parsl.providers.grid_engine.grid_engine import GridEngineProvider
|
9
|
-
from parsl.providers.slurm.slurm import SlurmProvider
|
10
|
-
from parsl.providers.torque.torque import TorqueProvider
|
11
|
-
from parsl.providers.pbspro.pbspro import PBSProProvider
|
12
|
-
from parsl.providers.lsf.lsf import LSFProvider
|
13
|
-
from parsl.providers.ad_hoc.ad_hoc import AdHocProvider
|
14
|
-
|
15
1
|
# Cloud Providers
|
16
2
|
from parsl.providers.aws.aws import AWSProvider
|
17
|
-
from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider
|
18
3
|
from parsl.providers.azure.azure import AzureProvider
|
4
|
+
from parsl.providers.condor.condor import CondorProvider
|
5
|
+
from parsl.providers.googlecloud.googlecloud import GoogleCloudProvider
|
6
|
+
from parsl.providers.grid_engine.grid_engine import GridEngineProvider
|
19
7
|
|
20
8
|
# Kubernetes
|
21
9
|
from parsl.providers.kubernetes.kube import KubernetesProvider
|
10
|
+
from parsl.providers.local.local import LocalProvider
|
11
|
+
from parsl.providers.lsf.lsf import LSFProvider
|
12
|
+
from parsl.providers.pbspro.pbspro import PBSProProvider
|
13
|
+
from parsl.providers.slurm.slurm import SlurmProvider
|
14
|
+
from parsl.providers.torque.torque import TorqueProvider
|
22
15
|
|
23
16
|
__all__ = ['LocalProvider',
|
24
|
-
'CobaltProvider',
|
25
17
|
'CondorProvider',
|
26
18
|
'GridEngineProvider',
|
27
19
|
'SlurmProvider',
|
28
20
|
'TorqueProvider',
|
29
21
|
'LSFProvider',
|
30
|
-
'AdHocProvider',
|
31
22
|
'PBSProProvider',
|
32
23
|
'AWSProvider',
|
33
24
|
'GoogleCloudProvider',
|
parsl/providers/aws/aws.py
CHANGED
@@ -4,13 +4,12 @@ import os
|
|
4
4
|
import time
|
5
5
|
from string import Template
|
6
6
|
|
7
|
-
from parsl.errors import ConfigurationError
|
7
|
+
from parsl.errors import ConfigurationError, OptionalModuleMissing
|
8
8
|
from parsl.jobs.states import JobState, JobStatus
|
9
|
+
from parsl.launchers import SingleNodeLauncher
|
9
10
|
from parsl.providers.aws.template import template_string
|
10
11
|
from parsl.providers.base import ExecutionProvider
|
11
|
-
from parsl.errors import OptionalModuleMissing
|
12
12
|
from parsl.utils import RepresentationMixin
|
13
|
-
from parsl.launchers import SingleNodeLauncher
|
14
13
|
|
15
14
|
logger = logging.getLogger(__name__)
|
16
15
|
|
parsl/providers/azure/azure.py
CHANGED
@@ -4,22 +4,21 @@ import os
|
|
4
4
|
import time
|
5
5
|
from string import Template
|
6
6
|
|
7
|
-
from parsl.errors import ConfigurationError
|
7
|
+
from parsl.errors import ConfigurationError, OptionalModuleMissing
|
8
8
|
from parsl.jobs.states import JobState, JobStatus
|
9
|
+
from parsl.launchers import SingleNodeLauncher
|
9
10
|
from parsl.providers.azure.template import template_string
|
10
11
|
from parsl.providers.base import ExecutionProvider
|
11
|
-
from parsl.errors import OptionalModuleMissing
|
12
12
|
from parsl.utils import RepresentationMixin
|
13
|
-
from parsl.launchers import SingleNodeLauncher
|
14
13
|
|
15
14
|
logger = logging.getLogger(__name__)
|
16
15
|
|
17
16
|
try:
|
18
17
|
from azure.common.credentials import ServicePrincipalCredentials
|
19
|
-
from azure.mgmt.resource import ResourceManagementClient
|
20
|
-
from azure.mgmt.network import NetworkManagementClient
|
21
18
|
from azure.mgmt.compute import ComputeManagementClient
|
22
19
|
from azure.mgmt.compute.models import DiskCreateOption
|
20
|
+
from azure.mgmt.network import NetworkManagementClient
|
21
|
+
from azure.mgmt.resource import ResourceManagementClient
|
23
22
|
from msrestazure.azure_exceptions import CloudError
|
24
23
|
|
25
24
|
_api_enabled = True
|
parsl/providers/base.py
CHANGED
@@ -1,8 +1,7 @@
|
|
1
|
-
from abc import ABCMeta, abstractmethod, abstractproperty
|
2
1
|
import logging
|
2
|
+
from abc import ABCMeta, abstractmethod, abstractproperty
|
3
3
|
from typing import Any, Dict, List, Optional
|
4
4
|
|
5
|
-
from parsl.channels.base import Channel
|
6
5
|
from parsl.jobs.states import JobStatus
|
7
6
|
|
8
7
|
logger = logging.getLogger(__name__)
|
@@ -12,7 +11,7 @@ class ExecutionProvider(metaclass=ABCMeta):
|
|
12
11
|
"""Execution providers are responsible for managing execution resources
|
13
12
|
that have a Local Resource Manager (LRM). For instance, campus clusters
|
14
13
|
and supercomputers generally have LRMs (schedulers) such as Slurm,
|
15
|
-
Torque/PBS,
|
14
|
+
Torque/PBS, and Condor. Clouds, on the other hand, have API
|
16
15
|
interfaces that allow much more fine-grained composition of an execution
|
17
16
|
environment. An execution provider abstracts these types of resources and
|
18
17
|
provides a single uniform interface to them.
|
@@ -154,18 +153,3 @@ class ExecutionProvider(metaclass=ABCMeta):
|
|
154
153
|
:return: the number of seconds to wait between calls to status()
|
155
154
|
"""
|
156
155
|
pass
|
157
|
-
|
158
|
-
|
159
|
-
class Channeled():
|
160
|
-
"""A marker type to indicate that parsl should manage a Channel for this provider"""
|
161
|
-
def __init__(self) -> None:
|
162
|
-
self.channel: Channel
|
163
|
-
pass
|
164
|
-
|
165
|
-
|
166
|
-
class MultiChanneled():
|
167
|
-
"""A marker type to indicate that parsl should manage multiple Channels for this provider"""
|
168
|
-
|
169
|
-
def __init__(self) -> None:
|
170
|
-
self.channels: List[Channel]
|
171
|
-
pass
|