parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsl/__init__.py +9 -10
- parsl/addresses.py +6 -4
- parsl/app/app.py +3 -6
- parsl/app/bash.py +4 -4
- parsl/app/errors.py +5 -3
- parsl/app/futures.py +3 -3
- parsl/app/python.py +2 -1
- parsl/benchmark/perf.py +2 -1
- parsl/channels/__init__.py +2 -2
- parsl/channels/base.py +0 -1
- parsl/channels/errors.py +2 -1
- parsl/channels/oauth_ssh/oauth_ssh.py +4 -3
- parsl/channels/ssh/ssh.py +9 -1
- parsl/channels/ssh_il/ssh_il.py +1 -0
- parsl/concurrent/__init__.py +2 -2
- parsl/config.py +5 -5
- parsl/configs/ASPIRE1.py +3 -3
- parsl/configs/Azure.py +6 -7
- parsl/configs/ad_hoc.py +4 -3
- parsl/configs/bridges.py +3 -3
- parsl/configs/cc_in2p3.py +2 -2
- parsl/configs/ec2.py +1 -1
- parsl/configs/expanse.py +1 -2
- parsl/configs/frontera.py +2 -3
- parsl/configs/htex_local.py +1 -2
- parsl/configs/illinoiscluster.py +1 -1
- parsl/configs/kubernetes.py +1 -2
- parsl/configs/midway.py +3 -3
- parsl/configs/osg.py +1 -1
- parsl/configs/polaris.py +1 -1
- parsl/configs/stampede2.py +4 -5
- parsl/configs/summit.py +1 -3
- parsl/configs/toss3_llnl.py +1 -2
- parsl/configs/vineex_local.py +3 -3
- parsl/configs/wqex_local.py +2 -2
- parsl/data_provider/data_manager.py +3 -3
- parsl/data_provider/file_noop.py +1 -2
- parsl/data_provider/files.py +3 -3
- parsl/data_provider/ftp.py +1 -3
- parsl/data_provider/globus.py +7 -6
- parsl/data_provider/http.py +2 -2
- parsl/data_provider/rsync.py +1 -1
- parsl/data_provider/staging.py +2 -2
- parsl/data_provider/zip.py +4 -5
- parsl/dataflow/dflow.py +21 -16
- parsl/dataflow/errors.py +2 -1
- parsl/dataflow/futures.py +1 -2
- parsl/dataflow/memoization.py +5 -5
- parsl/dataflow/rundirs.py +1 -1
- parsl/dataflow/taskrecord.py +4 -5
- parsl/executors/__init__.py +3 -3
- parsl/executors/base.py +1 -0
- parsl/executors/flux/execute_parsl_task.py +2 -2
- parsl/executors/flux/executor.py +11 -12
- parsl/executors/flux/flux_instance_manager.py +3 -3
- parsl/executors/high_throughput/executor.py +29 -35
- parsl/executors/high_throughput/interchange.py +16 -18
- parsl/executors/high_throughput/manager_record.py +1 -0
- parsl/executors/high_throughput/monitoring_info.py +2 -1
- parsl/executors/high_throughput/mpi_executor.py +5 -2
- parsl/executors/high_throughput/mpi_prefix_composer.py +1 -1
- parsl/executors/high_throughput/mpi_resource_management.py +1 -2
- parsl/executors/high_throughput/probe.py +6 -4
- parsl/executors/high_throughput/process_worker_pool.py +31 -20
- parsl/executors/high_throughput/zmq_pipes.py +28 -14
- parsl/executors/radical/executor.py +15 -15
- parsl/executors/radical/rpex_master.py +1 -2
- parsl/executors/radical/rpex_resources.py +1 -2
- parsl/executors/radical/rpex_worker.py +2 -1
- parsl/executors/status_handling.py +5 -4
- parsl/executors/taskvine/__init__.py +1 -1
- parsl/executors/taskvine/errors.py +1 -1
- parsl/executors/taskvine/exec_parsl_function.py +2 -2
- parsl/executors/taskvine/executor.py +23 -24
- parsl/executors/taskvine/factory.py +1 -1
- parsl/executors/taskvine/manager.py +11 -13
- parsl/executors/threads.py +4 -5
- parsl/executors/workqueue/errors.py +1 -1
- parsl/executors/workqueue/exec_parsl_function.py +5 -4
- parsl/executors/workqueue/executor.py +26 -27
- parsl/executors/workqueue/parsl_coprocess.py +1 -1
- parsl/jobs/error_handlers.py +1 -1
- parsl/jobs/job_status_poller.py +2 -5
- parsl/jobs/states.py +1 -1
- parsl/jobs/strategy.py +2 -2
- parsl/launchers/__init__.py +12 -3
- parsl/launchers/errors.py +1 -1
- parsl/log_utils.py +1 -2
- parsl/monitoring/db_manager.py +16 -10
- parsl/monitoring/monitoring.py +11 -15
- parsl/monitoring/queries/pandas.py +1 -2
- parsl/monitoring/radios.py +2 -4
- parsl/monitoring/remote.py +13 -8
- parsl/monitoring/router.py +8 -11
- parsl/monitoring/types.py +2 -0
- parsl/monitoring/visualization/app.py +4 -2
- parsl/monitoring/visualization/models.py +0 -1
- parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
- parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
- parsl/monitoring/visualization/utils.py +0 -1
- parsl/monitoring/visualization/views.py +16 -9
- parsl/multiprocessing.py +0 -1
- parsl/process_loggers.py +1 -2
- parsl/providers/__init__.py +9 -12
- parsl/providers/ad_hoc/ad_hoc.py +1 -1
- parsl/providers/aws/aws.py +2 -3
- parsl/providers/azure/azure.py +4 -5
- parsl/providers/base.py +1 -1
- parsl/providers/cluster_provider.py +1 -1
- parsl/providers/cobalt/cobalt.py +3 -3
- parsl/providers/condor/condor.py +4 -2
- parsl/providers/errors.py +2 -2
- parsl/providers/googlecloud/googlecloud.py +2 -1
- parsl/providers/grid_engine/grid_engine.py +2 -2
- parsl/providers/kubernetes/kube.py +5 -3
- parsl/providers/local/local.py +5 -1
- parsl/providers/lsf/lsf.py +2 -2
- parsl/providers/pbspro/pbspro.py +1 -1
- parsl/providers/slurm/slurm.py +5 -5
- parsl/providers/torque/torque.py +1 -1
- parsl/serialize/__init__.py +8 -3
- parsl/serialize/base.py +1 -2
- parsl/serialize/concretes.py +5 -4
- parsl/serialize/proxystore.py +3 -2
- parsl/tests/__init__.py +1 -1
- parsl/tests/configs/ad_hoc_cluster_htex.py +4 -4
- parsl/tests/configs/azure_single_node.py +4 -5
- parsl/tests/configs/bridges.py +3 -2
- parsl/tests/configs/cc_in2p3.py +2 -2
- parsl/tests/configs/comet.py +2 -1
- parsl/tests/configs/ec2_single_node.py +1 -2
- parsl/tests/configs/ec2_spot.py +1 -2
- parsl/tests/configs/frontera.py +3 -2
- parsl/tests/configs/htex_ad_hoc_cluster.py +2 -4
- parsl/tests/configs/htex_local.py +2 -3
- parsl/tests/configs/htex_local_alternate.py +8 -11
- parsl/tests/configs/htex_local_intask_staging.py +5 -7
- parsl/tests/configs/htex_local_rsync_staging.py +4 -6
- parsl/tests/configs/local_adhoc.py +1 -1
- parsl/tests/configs/local_radical.py +1 -3
- parsl/tests/configs/local_radical_mpi.py +2 -2
- parsl/tests/configs/midway.py +2 -2
- parsl/tests/configs/nscc_singapore.py +3 -3
- parsl/tests/configs/osg_htex.py +1 -1
- parsl/tests/configs/petrelkube.py +3 -2
- parsl/tests/configs/summit.py +1 -0
- parsl/tests/configs/swan_htex.py +2 -2
- parsl/tests/configs/taskvine_ex.py +3 -5
- parsl/tests/configs/theta.py +2 -2
- parsl/tests/configs/workqueue_ex.py +3 -4
- parsl/tests/conftest.py +4 -4
- parsl/tests/integration/test_channels/test_ssh_errors.py +1 -1
- parsl/tests/integration/test_stress/test_python_simple.py +3 -4
- parsl/tests/integration/test_stress/test_python_threads.py +3 -5
- parsl/tests/manual_tests/htex_local.py +4 -4
- parsl/tests/manual_tests/test_ad_hoc_htex.py +2 -1
- parsl/tests/manual_tests/test_basic.py +1 -0
- parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +4 -4
- parsl/tests/manual_tests/test_log_filter.py +3 -1
- parsl/tests/manual_tests/test_memory_limits.py +6 -6
- parsl/tests/manual_tests/test_regression_220.py +2 -1
- parsl/tests/manual_tests/test_udp_simple.py +4 -3
- parsl/tests/manual_tests/test_worker_count.py +3 -2
- parsl/tests/scaling_tests/htex_local.py +2 -2
- parsl/tests/scaling_tests/test_scale.py +0 -9
- parsl/tests/scaling_tests/vineex_condor.py +1 -2
- parsl/tests/scaling_tests/vineex_local.py +1 -2
- parsl/tests/site_tests/test_provider.py +3 -1
- parsl/tests/site_tests/test_site.py +2 -0
- parsl/tests/sites/test_affinity.py +7 -5
- parsl/tests/sites/test_dynamic_executor.py +3 -3
- parsl/tests/sites/test_ec2.py +3 -2
- parsl/tests/sites/test_local_adhoc.py +2 -1
- parsl/tests/sites/test_worker_info.py +4 -3
- parsl/tests/test_aalst_patterns.py +0 -1
- parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
- parsl/tests/test_bash_apps/test_error_codes.py +1 -4
- parsl/tests/test_bash_apps/test_memoize_ignore_args.py +1 -0
- parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +2 -2
- parsl/tests/test_bash_apps/test_pipeline.py +1 -1
- parsl/tests/test_bash_apps/test_std_uri.py +4 -3
- parsl/tests/test_callables.py +2 -2
- parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
- parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -1
- parsl/tests/test_checkpointing/test_regression_239.py +1 -1
- parsl/tests/test_checkpointing/test_task_exit.py +1 -1
- parsl/tests/test_docs/test_from_slides.py +2 -2
- parsl/tests/test_docs/test_kwargs.py +1 -1
- parsl/tests/test_docs/test_tutorial_1.py +1 -2
- parsl/tests/test_docs/test_workflow1.py +2 -2
- parsl/tests/test_docs/test_workflow2.py +0 -1
- parsl/tests/test_error_handling/test_rand_fail.py +2 -2
- parsl/tests/test_error_handling/test_resource_spec.py +4 -2
- parsl/tests/test_error_handling/test_retries.py +2 -1
- parsl/tests/test_error_handling/test_retry_handler.py +1 -0
- parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
- parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
- parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
- parsl/tests/test_flux.py +1 -1
- parsl/tests/test_htex/test_command_client_timeout.py +9 -12
- parsl/tests/test_htex/test_connected_blocks.py +3 -2
- parsl/tests/test_htex/test_cpu_affinity_explicit.py +5 -2
- parsl/tests/test_htex/test_disconnected_blocks.py +6 -4
- parsl/tests/test_htex/test_drain.py +5 -5
- parsl/tests/test_htex/test_htex.py +1 -2
- parsl/tests/test_htex/test_managers_command.py +3 -2
- parsl/tests/test_htex/test_multiple_disconnected_blocks.py +6 -4
- parsl/tests/test_monitoring/test_app_names.py +3 -2
- parsl/tests/test_monitoring/test_basic.py +4 -4
- parsl/tests/test_monitoring/test_db_locks.py +6 -3
- parsl/tests/test_monitoring/test_fuzz_zmq.py +6 -3
- parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +5 -5
- parsl/tests/test_monitoring/test_incomplete_futures.py +5 -3
- parsl/tests/test_monitoring/test_memoization_representation.py +4 -1
- parsl/tests/test_monitoring/test_stdouterr.py +4 -4
- parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
- parsl/tests/test_mpi_apps/test_bad_mpi_config.py +1 -1
- parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +2 -0
- parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +7 -5
- parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
- parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
- parsl/tests/test_mpi_apps/test_mpiex.py +4 -3
- parsl/tests/test_mpi_apps/test_resource_spec.py +9 -10
- parsl/tests/test_providers/test_cobalt_deprecation_warning.py +2 -0
- parsl/tests/test_providers/test_local_provider.py +2 -1
- parsl/tests/test_providers/test_pbspro_template.py +1 -1
- parsl/tests/test_providers/test_slurm_template.py +1 -1
- parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
- parsl/tests/test_python_apps/test_context_manager.py +4 -3
- parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
- parsl/tests/test_python_apps/test_futures.py +2 -1
- parsl/tests/test_python_apps/test_join.py +0 -1
- parsl/tests/test_python_apps/test_lifted.py +3 -3
- parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
- parsl/tests/test_python_apps/test_pluggable_future_resolution.py +1 -1
- parsl/tests/test_radical/test_mpi_funcs.py +1 -1
- parsl/tests/test_regression/test_1480.py +2 -1
- parsl/tests/test_regression/test_1653.py +2 -1
- parsl/tests/test_regression/test_2652.py +1 -0
- parsl/tests/test_regression/test_69a.py +0 -1
- parsl/tests/test_regression/test_854.py +4 -2
- parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
- parsl/tests/test_regression/test_98.py +0 -1
- parsl/tests/test_scaling/test_block_error_handler.py +9 -4
- parsl/tests/test_scaling/test_scale_down.py +2 -3
- parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +4 -5
- parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +3 -4
- parsl/tests/test_scaling/test_shutdown_scalein.py +1 -2
- parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
- parsl/tests/test_serialization/test_basic.py +2 -1
- parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
- parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
- parsl/tests/test_serialization/test_proxystore_configured.py +10 -5
- parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
- parsl/tests/test_shutdown/test_kill_monitoring.py +3 -2
- parsl/tests/test_staging/staging_provider.py +2 -2
- parsl/tests/test_staging/test_1316.py +3 -2
- parsl/tests/test_staging/test_docs_1.py +1 -1
- parsl/tests/test_staging/test_docs_2.py +2 -1
- parsl/tests/test_staging/test_elaborate_noop_file.py +2 -2
- parsl/tests/test_staging/test_staging_https.py +2 -2
- parsl/tests/test_staging/test_staging_stdout.py +4 -3
- parsl/tests/test_staging/test_zip_in.py +6 -8
- parsl/tests/test_staging/test_zip_out.py +7 -9
- parsl/tests/test_staging/test_zip_to_zip.py +6 -8
- parsl/tests/test_summary.py +2 -1
- parsl/tests/test_thread_parallelism.py +0 -1
- parsl/tests/test_threads/test_configs.py +1 -1
- parsl/tests/test_threads/test_lazy_errors.py +2 -1
- parsl/usage_tracking/api.py +2 -3
- parsl/usage_tracking/usage.py +8 -18
- parsl/utils.py +13 -2
- parsl/version.py +1 -1
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py +5 -4
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py +31 -20
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/METADATA +2 -2
- parsl-2024.6.3.dist-info/RECORD +471 -0
- parsl-2024.5.27.dist-info/RECORD +0 -471
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/parsl_coprocess.py +1 -1
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/LICENSE +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/WHEEL +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/entry_points.txt +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/top_level.txt +0 -0
parsl/tests/configs/frontera.py
CHANGED
@@ -1,10 +1,11 @@
|
|
1
|
-
from parsl.config import Config
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
from parsl.
|
2
|
+
from parsl.config import Config
|
4
3
|
from parsl.executors import HighThroughputExecutor
|
5
4
|
from parsl.launchers import SrunLauncher
|
5
|
+
from parsl.providers import SlurmProvider
|
6
6
|
|
7
7
|
from .user_opts import user_opts
|
8
|
+
|
8
9
|
""" This config assumes that it is used to launch parsl tasks from the login nodes
|
9
10
|
of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
|
10
11
|
"""
|
@@ -1,9 +1,7 @@
|
|
1
|
-
from parsl.providers import AdHocProvider
|
2
1
|
from parsl.channels import SSHChannel
|
3
|
-
from parsl.executors import HighThroughputExecutor
|
4
|
-
|
5
2
|
from parsl.config import Config
|
6
|
-
|
3
|
+
from parsl.executors import HighThroughputExecutor
|
4
|
+
from parsl.providers import AdHocProvider
|
7
5
|
from parsl.tests.configs.user_opts import user_opts
|
8
6
|
|
9
7
|
config = Config(
|
@@ -1,9 +1,8 @@
|
|
1
|
-
from parsl.providers import LocalProvider
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
from parsl.launchers import SimpleLauncher
|
4
|
-
|
5
2
|
from parsl.config import Config
|
6
3
|
from parsl.executors import HighThroughputExecutor
|
4
|
+
from parsl.launchers import SimpleLauncher
|
5
|
+
from parsl.providers import LocalProvider
|
7
6
|
|
8
7
|
|
9
8
|
def fresh_config():
|
@@ -15,23 +15,20 @@ will cause substantially different behaviour on whatever
|
|
15
15
|
those timing parameters control.
|
16
16
|
"""
|
17
17
|
|
18
|
-
# imports for monitoring:
|
19
|
-
from parsl.monitoring import MonitoringHub
|
20
|
-
|
21
18
|
import os
|
22
19
|
|
23
|
-
from parsl.providers import LocalProvider
|
24
20
|
from parsl.channels import LocalChannel
|
25
|
-
from parsl.launchers import SingleNodeLauncher
|
26
|
-
|
27
21
|
from parsl.config import Config
|
28
|
-
from parsl.executors import HighThroughputExecutor
|
29
|
-
|
30
|
-
|
31
|
-
from parsl.data_provider.http import HTTPInTaskStaging
|
32
|
-
from parsl.data_provider.ftp import FTPInTaskStaging
|
33
22
|
from parsl.data_provider.file_noop import NoOpFileStaging
|
23
|
+
from parsl.data_provider.ftp import FTPInTaskStaging
|
24
|
+
from parsl.data_provider.http import HTTPInTaskStaging
|
34
25
|
from parsl.data_provider.zip import ZipFileStaging
|
26
|
+
from parsl.executors import HighThroughputExecutor
|
27
|
+
from parsl.launchers import SingleNodeLauncher
|
28
|
+
|
29
|
+
# imports for monitoring:
|
30
|
+
from parsl.monitoring import MonitoringHub
|
31
|
+
from parsl.providers import LocalProvider
|
35
32
|
|
36
33
|
working_dir = os.getcwd() + "/" + "test_htex_alternate"
|
37
34
|
|
@@ -1,13 +1,11 @@
|
|
1
|
-
from parsl.providers import LocalProvider
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
from parsl.launchers import SimpleLauncher
|
4
|
-
|
5
|
-
from parsl.data_provider.http import HTTPInTaskStaging
|
6
|
-
from parsl.data_provider.ftp import FTPInTaskStaging
|
7
|
-
from parsl.data_provider.file_noop import NoOpFileStaging
|
8
|
-
|
9
2
|
from parsl.config import Config
|
3
|
+
from parsl.data_provider.file_noop import NoOpFileStaging
|
4
|
+
from parsl.data_provider.ftp import FTPInTaskStaging
|
5
|
+
from parsl.data_provider.http import HTTPInTaskStaging
|
10
6
|
from parsl.executors import HighThroughputExecutor
|
7
|
+
from parsl.launchers import SimpleLauncher
|
8
|
+
from parsl.providers import LocalProvider
|
11
9
|
|
12
10
|
config = Config(
|
13
11
|
executors=[
|
@@ -1,13 +1,11 @@
|
|
1
|
-
from parsl.providers import LocalProvider
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
from parsl.
|
4
|
-
|
5
|
-
from parsl.data_provider.http import HTTPInTaskStaging
|
2
|
+
from parsl.config import Config
|
6
3
|
from parsl.data_provider.ftp import FTPInTaskStaging
|
4
|
+
from parsl.data_provider.http import HTTPInTaskStaging
|
7
5
|
from parsl.data_provider.rsync import RSyncStaging
|
8
|
-
|
9
|
-
from parsl.config import Config
|
10
6
|
from parsl.executors import HighThroughputExecutor
|
7
|
+
from parsl.launchers import SimpleLauncher
|
8
|
+
from parsl.providers import LocalProvider
|
11
9
|
|
12
10
|
config = Config(
|
13
11
|
executors=[
|
@@ -1,9 +1,7 @@
|
|
1
1
|
import os
|
2
2
|
|
3
3
|
from parsl.config import Config
|
4
|
-
from parsl.executors.radical import RadicalPilotExecutor
|
5
|
-
from parsl.executors.radical import ResourceConfig
|
6
|
-
|
4
|
+
from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
|
7
5
|
|
8
6
|
rpex_cfg = ResourceConfig()
|
9
7
|
|
@@ -1,10 +1,10 @@
|
|
1
1
|
import os
|
2
|
+
|
2
3
|
from parsl.config import Config
|
3
4
|
|
4
5
|
|
5
6
|
def fresh_config():
|
6
|
-
from parsl.executors.radical import ResourceConfig
|
7
|
-
from parsl.executors.radical import RadicalPilotExecutor
|
7
|
+
from parsl.executors.radical import RadicalPilotExecutor, ResourceConfig
|
8
8
|
|
9
9
|
rpex_cfg = ResourceConfig()
|
10
10
|
rpex_cfg.worker_type = "MPI"
|
parsl/tests/configs/midway.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.providers import SlurmProvider
|
3
|
-
from parsl.launchers import SrunLauncher
|
4
2
|
from parsl.executors import HighThroughputExecutor
|
3
|
+
from parsl.launchers import SrunLauncher
|
4
|
+
from parsl.providers import SlurmProvider
|
5
5
|
|
6
6
|
from .user_opts import user_opts
|
7
7
|
|
@@ -1,8 +1,8 @@
|
|
1
|
-
from parsl.providers import PBSProProvider
|
2
|
-
from parsl.executors import HighThroughputExecutor
|
3
|
-
from parsl.launchers import MpiRunLauncher
|
4
1
|
from parsl.addresses import address_by_interface
|
5
2
|
from parsl.config import Config
|
3
|
+
from parsl.executors import HighThroughputExecutor
|
4
|
+
from parsl.launchers import MpiRunLauncher
|
5
|
+
from parsl.providers import PBSProProvider
|
6
6
|
|
7
7
|
from .user_opts import user_opts
|
8
8
|
|
parsl/tests/configs/osg_htex.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.providers import CondorProvider
|
3
2
|
from parsl.executors import HighThroughputExecutor
|
3
|
+
from parsl.providers import CondorProvider
|
4
4
|
|
5
5
|
# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
|
6
6
|
# If you are a user copying-and-pasting this as an example, make sure to either
|
@@ -1,10 +1,11 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
from parsl.addresses import address_by_route
|
1
4
|
from parsl.config import Config
|
2
5
|
from parsl.executors import HighThroughputExecutor
|
3
6
|
from parsl.providers import KubernetesProvider
|
4
|
-
from parsl.addresses import address_by_route
|
5
7
|
|
6
8
|
from .user_opts import user_opts
|
7
|
-
import os
|
8
9
|
|
9
10
|
|
10
11
|
def fresh_config():
|
parsl/tests/configs/summit.py
CHANGED
@@ -4,6 +4,7 @@ from parsl.launchers import JsrunLauncher
|
|
4
4
|
from parsl.providers import LSFProvider
|
5
5
|
|
6
6
|
from .user_opts import user_opts
|
7
|
+
|
7
8
|
""" This config assumes that it is used to launch parsl tasks from the login nodes
|
8
9
|
of Frontera at TACC. Each job submitted to the scheduler will request 2 nodes for 10 minutes.
|
9
10
|
"""
|
parsl/tests/configs/swan_htex.py
CHANGED
@@ -8,10 +8,10 @@
|
|
8
8
|
==================
|
9
9
|
"""
|
10
10
|
from parsl.channels import SSHChannel
|
11
|
-
from parsl.launchers import AprunLauncher
|
12
|
-
from parsl.providers import TorqueProvider
|
13
11
|
from parsl.config import Config
|
14
12
|
from parsl.executors import HighThroughputExecutor
|
13
|
+
from parsl.launchers import AprunLauncher
|
14
|
+
from parsl.providers import TorqueProvider
|
15
15
|
|
16
16
|
# If you are a developer running tests, make sure to update parsl/tests/configs/user_opts.py
|
17
17
|
# If you are a user copying-and-pasting this as an example, make sure to either
|
@@ -1,10 +1,8 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.executors.taskvine import TaskVineExecutor
|
3
|
-
from parsl.executors.taskvine import TaskVineManagerConfig
|
4
|
-
|
5
|
-
from parsl.data_provider.http import HTTPInTaskStaging
|
6
|
-
from parsl.data_provider.ftp import FTPInTaskStaging
|
7
2
|
from parsl.data_provider.file_noop import NoOpFileStaging
|
3
|
+
from parsl.data_provider.ftp import FTPInTaskStaging
|
4
|
+
from parsl.data_provider.http import HTTPInTaskStaging
|
5
|
+
from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
|
8
6
|
|
9
7
|
|
10
8
|
def fresh_config():
|
parsl/tests/configs/theta.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.providers import CobaltProvider
|
3
|
-
from parsl.launchers import AprunLauncher
|
4
2
|
from parsl.executors import HighThroughputExecutor
|
3
|
+
from parsl.launchers import AprunLauncher
|
4
|
+
from parsl.providers import CobaltProvider
|
5
5
|
|
6
6
|
from .user_opts import user_opts
|
7
7
|
|
@@ -1,9 +1,8 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.executors import WorkQueueExecutor
|
3
|
-
|
4
|
-
from parsl.data_provider.http import HTTPInTaskStaging
|
5
|
-
from parsl.data_provider.ftp import FTPInTaskStaging
|
6
2
|
from parsl.data_provider.file_noop import NoOpFileStaging
|
3
|
+
from parsl.data_provider.ftp import FTPInTaskStaging
|
4
|
+
from parsl.data_provider.http import HTTPInTaskStaging
|
5
|
+
from parsl.executors import WorkQueueExecutor
|
7
6
|
|
8
7
|
|
9
8
|
def fresh_config():
|
parsl/tests/conftest.py
CHANGED
@@ -6,21 +6,21 @@ import pathlib
|
|
6
6
|
import random
|
7
7
|
import re
|
8
8
|
import shutil
|
9
|
-
import string
|
10
|
-
import time
|
11
|
-
import types
|
12
9
|
import signal
|
10
|
+
import string
|
13
11
|
import sys
|
14
12
|
import tempfile
|
15
13
|
import threading
|
14
|
+
import time
|
16
15
|
import traceback
|
16
|
+
import types
|
17
17
|
import typing as t
|
18
18
|
from datetime import datetime
|
19
19
|
from glob import glob
|
20
20
|
from itertools import chain
|
21
21
|
|
22
|
-
import pytest
|
23
22
|
import _pytest.runner as runner
|
23
|
+
import pytest
|
24
24
|
|
25
25
|
import parsl
|
26
26
|
from parsl.dataflow.dflow import DataFlowKernelLoader
|
@@ -1,14 +1,12 @@
|
|
1
1
|
''' Testing bash apps
|
2
2
|
'''
|
3
|
-
import parsl
|
4
|
-
from parsl import python_app
|
5
|
-
|
6
|
-
import time
|
7
3
|
import argparse
|
4
|
+
import time
|
8
5
|
|
6
|
+
import parsl
|
7
|
+
from parsl import python_app
|
9
8
|
from parsl.tests.configs.local_threads import config
|
10
9
|
|
11
|
-
|
12
10
|
local_config = config
|
13
11
|
|
14
12
|
|
@@ -1,11 +1,11 @@
|
|
1
|
-
from parsl.providers import LocalProvider
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
# from parsl.launchers import SimpleLauncher
|
4
|
-
from parsl.launchers import SingleNodeLauncher
|
5
|
-
|
6
2
|
from parsl.config import Config
|
7
3
|
from parsl.executors import HighThroughputExecutor
|
8
4
|
|
5
|
+
# from parsl.launchers import SimpleLauncher
|
6
|
+
from parsl.launchers import SingleNodeLauncher
|
7
|
+
from parsl.providers import LocalProvider
|
8
|
+
|
9
9
|
config = Config(
|
10
10
|
executors=[
|
11
11
|
HighThroughputExecutor(
|
@@ -1,11 +1,12 @@
|
|
1
1
|
import parsl
|
2
2
|
from parsl import python_app
|
3
|
+
|
3
4
|
parsl.set_stream_logger()
|
4
5
|
|
5
|
-
from parsl.providers import AdHocProvider
|
6
6
|
from parsl.channels import SSHChannel
|
7
7
|
from parsl.config import Config
|
8
8
|
from parsl.executors import HighThroughputExecutor
|
9
|
+
from parsl.providers import AdHocProvider
|
9
10
|
|
10
11
|
remotes = ['midway2-login2.rcc.uchicago.edu', 'midway2-login1.rcc.uchicago.edu']
|
11
12
|
|
@@ -1,13 +1,13 @@
|
|
1
|
+
import logging
|
2
|
+
|
1
3
|
import parsl
|
2
|
-
from parsl.
|
4
|
+
from parsl.app.app import python_app
|
3
5
|
from parsl.config import Config
|
4
6
|
from parsl.executors import HighThroughputExecutor
|
5
7
|
from parsl.launchers import AprunLauncher
|
8
|
+
from parsl.monitoring.monitoring import MonitoringHub
|
6
9
|
from parsl.providers import CobaltProvider
|
7
10
|
|
8
|
-
import logging
|
9
|
-
from parsl.app.app import python_app
|
10
|
-
|
11
11
|
|
12
12
|
def local_setup():
|
13
13
|
threads_config = Config(
|
@@ -1,15 +1,15 @@
|
|
1
1
|
import argparse
|
2
|
-
import parsl
|
3
|
-
import psutil
|
4
2
|
import multiprocessing
|
5
3
|
|
6
|
-
|
4
|
+
import psutil
|
5
|
+
|
6
|
+
import parsl
|
7
|
+
from parsl.app.app import python_app # , bash_app
|
7
8
|
from parsl.channels import LocalChannel
|
8
|
-
from parsl.launchers import SingleNodeLauncher
|
9
9
|
from parsl.config import Config
|
10
10
|
from parsl.executors import HighThroughputExecutor
|
11
|
-
|
12
|
-
from parsl.
|
11
|
+
from parsl.launchers import SingleNodeLauncher
|
12
|
+
from parsl.providers import LocalProvider
|
13
13
|
|
14
14
|
|
15
15
|
@python_app
|
@@ -1,9 +1,10 @@
|
|
1
|
+
import logging
|
2
|
+
|
3
|
+
import parsl
|
1
4
|
from parsl import python_app
|
2
|
-
from parsl.monitoring.monitoring import MonitoringHub
|
3
5
|
from parsl.config import Config
|
4
6
|
from parsl.executors import ThreadPoolExecutor
|
5
|
-
import
|
6
|
-
import logging
|
7
|
+
from parsl.monitoring.monitoring import MonitoringHub
|
7
8
|
|
8
9
|
|
9
10
|
def local_setup():
|
@@ -1,7 +1,7 @@
|
|
1
1
|
import argparse
|
2
|
-
import time
|
3
2
|
import math
|
4
3
|
import multiprocessing
|
4
|
+
import time
|
5
5
|
|
6
6
|
import parsl
|
7
7
|
|
@@ -10,10 +10,11 @@ CORES_PER_WORKER = 1
|
|
10
10
|
EXPECTED_WORKERS = math.floor(CORES / CORES_PER_WORKER)
|
11
11
|
|
12
12
|
|
13
|
+
from parsl.executors import HighThroughputExecutor
|
14
|
+
|
13
15
|
# from parsl.tests.configs.htex_local import config
|
14
16
|
from parsl.tests.manual_tests.htex_local import config
|
15
17
|
|
16
|
-
from parsl.executors import HighThroughputExecutor
|
17
18
|
assert isinstance(config.executors[0], HighThroughputExecutor)
|
18
19
|
config.executors[0].cores_per_worker = CORES_PER_WORKER
|
19
20
|
config.executors[0].provider.init_blocks = 1
|
@@ -1,8 +1,8 @@
|
|
1
|
-
from parsl.providers import LocalProvider
|
2
1
|
from parsl.channels import LocalChannel
|
3
|
-
|
4
2
|
from parsl.config import Config
|
5
3
|
from parsl.executors import HighThroughputExecutor
|
4
|
+
from parsl.providers import LocalProvider
|
5
|
+
|
6
6
|
# import os
|
7
7
|
config = Config(
|
8
8
|
executors=[
|
@@ -4,15 +4,6 @@ import argparse
|
|
4
4
|
import time
|
5
5
|
|
6
6
|
import parsl
|
7
|
-
|
8
|
-
# from parsl.tests.configs.htex_local import config
|
9
|
-
# from htex_local import config
|
10
|
-
# from parsl.configs.local_threads import config
|
11
|
-
# from parsl.configs.local_ipp import config
|
12
|
-
|
13
|
-
# parsl.set_stream_logger()
|
14
|
-
# config.executors[0].provider.tasks_per_node = 4
|
15
|
-
# parsl.load(config)
|
16
7
|
from parsl.app.app import python_app # , bash_app
|
17
8
|
|
18
9
|
|
@@ -1,6 +1,5 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.executors.taskvine import TaskVineExecutor
|
3
|
-
from parsl.executors.taskvine import TaskVineManagerConfig
|
2
|
+
from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
|
4
3
|
from parsl.providers import CondorProvider
|
5
4
|
|
6
5
|
config = Config(
|
@@ -1,6 +1,5 @@
|
|
1
1
|
from parsl.config import Config
|
2
|
-
from parsl.executors.taskvine import TaskVineExecutor
|
3
|
-
from parsl.executors.taskvine import TaskVineManagerConfig
|
2
|
+
from parsl.executors.taskvine import TaskVineExecutor, TaskVineManagerConfig
|
4
3
|
from parsl.providers import LocalProvider
|
5
4
|
|
6
5
|
config = Config(
|
@@ -1,8 +1,10 @@
|
|
1
1
|
import argparse
|
2
2
|
import logging
|
3
|
+
import time
|
4
|
+
|
3
5
|
import pytest
|
6
|
+
|
4
7
|
import parsl
|
5
|
-
import time
|
6
8
|
from parsl.app.app import python_app # , bash_app
|
7
9
|
from parsl.jobs.states import JobState
|
8
10
|
from parsl.tests.site_tests.site_config_selector import fresh_config
|
@@ -1,12 +1,14 @@
|
|
1
1
|
"""Tests related to assigning workers to specific compute units"""
|
2
2
|
|
3
|
-
|
3
|
+
import os
|
4
|
+
|
5
|
+
import pytest
|
6
|
+
|
7
|
+
from parsl import python_app
|
4
8
|
from parsl.channels import LocalChannel
|
5
9
|
from parsl.config import Config
|
6
10
|
from parsl.executors import HighThroughputExecutor
|
7
|
-
from parsl import
|
8
|
-
import pytest
|
9
|
-
import os
|
11
|
+
from parsl.providers import LocalProvider
|
10
12
|
|
11
13
|
|
12
14
|
def local_config():
|
@@ -32,8 +34,8 @@ def local_config():
|
|
32
34
|
|
33
35
|
@python_app
|
34
36
|
def get_worker_info():
|
35
|
-
from time import sleep
|
36
37
|
import os
|
38
|
+
from time import sleep
|
37
39
|
rank = int(os.environ['PARSL_WORKER_RANK'])
|
38
40
|
aff = os.sched_getaffinity(0)
|
39
41
|
device = os.environ.get('CUDA_VISIBLE_DEVICES')
|
@@ -1,10 +1,10 @@
|
|
1
|
+
import pytest
|
2
|
+
|
1
3
|
import parsl
|
4
|
+
from parsl.app.app import python_app
|
2
5
|
from parsl.executors import HighThroughputExecutor
|
3
6
|
from parsl.executors.threads import ThreadPoolExecutor
|
4
7
|
from parsl.providers import LocalProvider
|
5
|
-
from parsl.app.app import python_app
|
6
|
-
|
7
|
-
import pytest
|
8
8
|
|
9
9
|
|
10
10
|
@python_app(executors=['threads'])
|
parsl/tests/sites/test_ec2.py
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
"""Tests related to Parsl workers being able to access their worker ID"""
|
2
2
|
|
3
|
-
|
3
|
+
import pytest
|
4
|
+
|
5
|
+
from parsl import python_app
|
4
6
|
from parsl.channels import LocalChannel
|
5
7
|
from parsl.config import Config
|
6
8
|
from parsl.executors import HighThroughputExecutor
|
7
|
-
from parsl import
|
8
|
-
import pytest
|
9
|
+
from parsl.providers import LocalProvider
|
9
10
|
|
10
11
|
|
11
12
|
def local_config():
|
@@ -4,12 +4,9 @@ import os
|
|
4
4
|
import pytest
|
5
5
|
|
6
6
|
import parsl
|
7
|
-
from parsl.app.app import bash_app
|
8
7
|
import parsl.app.errors as pe
|
9
|
-
|
10
|
-
|
8
|
+
from parsl.app.app import bash_app
|
11
9
|
from parsl.app.errors import BashExitFailure
|
12
|
-
|
13
10
|
from parsl.tests.configs.local_threads import fresh_config as local_config
|
14
11
|
|
15
12
|
|