parsl 2024.5.27__py3-none-any.whl → 2024.6.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parsl/__init__.py +9 -10
- parsl/addresses.py +6 -4
- parsl/app/app.py +3 -6
- parsl/app/bash.py +4 -4
- parsl/app/errors.py +5 -3
- parsl/app/futures.py +3 -3
- parsl/app/python.py +2 -1
- parsl/benchmark/perf.py +2 -1
- parsl/channels/__init__.py +2 -2
- parsl/channels/base.py +0 -1
- parsl/channels/errors.py +2 -1
- parsl/channels/oauth_ssh/oauth_ssh.py +4 -3
- parsl/channels/ssh/ssh.py +9 -1
- parsl/channels/ssh_il/ssh_il.py +1 -0
- parsl/concurrent/__init__.py +2 -2
- parsl/config.py +5 -5
- parsl/configs/ASPIRE1.py +3 -3
- parsl/configs/Azure.py +6 -7
- parsl/configs/ad_hoc.py +4 -3
- parsl/configs/bridges.py +3 -3
- parsl/configs/cc_in2p3.py +2 -2
- parsl/configs/ec2.py +1 -1
- parsl/configs/expanse.py +1 -2
- parsl/configs/frontera.py +2 -3
- parsl/configs/htex_local.py +1 -2
- parsl/configs/illinoiscluster.py +1 -1
- parsl/configs/kubernetes.py +1 -2
- parsl/configs/midway.py +3 -3
- parsl/configs/osg.py +1 -1
- parsl/configs/polaris.py +1 -1
- parsl/configs/stampede2.py +4 -5
- parsl/configs/summit.py +1 -3
- parsl/configs/toss3_llnl.py +1 -2
- parsl/configs/vineex_local.py +3 -3
- parsl/configs/wqex_local.py +2 -2
- parsl/data_provider/data_manager.py +3 -3
- parsl/data_provider/file_noop.py +1 -2
- parsl/data_provider/files.py +3 -3
- parsl/data_provider/ftp.py +1 -3
- parsl/data_provider/globus.py +7 -6
- parsl/data_provider/http.py +2 -2
- parsl/data_provider/rsync.py +1 -1
- parsl/data_provider/staging.py +2 -2
- parsl/data_provider/zip.py +4 -5
- parsl/dataflow/dflow.py +21 -16
- parsl/dataflow/errors.py +2 -1
- parsl/dataflow/futures.py +1 -2
- parsl/dataflow/memoization.py +5 -5
- parsl/dataflow/rundirs.py +1 -1
- parsl/dataflow/taskrecord.py +4 -5
- parsl/executors/__init__.py +3 -3
- parsl/executors/base.py +1 -0
- parsl/executors/flux/execute_parsl_task.py +2 -2
- parsl/executors/flux/executor.py +11 -12
- parsl/executors/flux/flux_instance_manager.py +3 -3
- parsl/executors/high_throughput/executor.py +29 -35
- parsl/executors/high_throughput/interchange.py +16 -18
- parsl/executors/high_throughput/manager_record.py +1 -0
- parsl/executors/high_throughput/monitoring_info.py +2 -1
- parsl/executors/high_throughput/mpi_executor.py +5 -2
- parsl/executors/high_throughput/mpi_prefix_composer.py +1 -1
- parsl/executors/high_throughput/mpi_resource_management.py +1 -2
- parsl/executors/high_throughput/probe.py +6 -4
- parsl/executors/high_throughput/process_worker_pool.py +31 -20
- parsl/executors/high_throughput/zmq_pipes.py +28 -14
- parsl/executors/radical/executor.py +15 -15
- parsl/executors/radical/rpex_master.py +1 -2
- parsl/executors/radical/rpex_resources.py +1 -2
- parsl/executors/radical/rpex_worker.py +2 -1
- parsl/executors/status_handling.py +5 -4
- parsl/executors/taskvine/__init__.py +1 -1
- parsl/executors/taskvine/errors.py +1 -1
- parsl/executors/taskvine/exec_parsl_function.py +2 -2
- parsl/executors/taskvine/executor.py +23 -24
- parsl/executors/taskvine/factory.py +1 -1
- parsl/executors/taskvine/manager.py +11 -13
- parsl/executors/threads.py +4 -5
- parsl/executors/workqueue/errors.py +1 -1
- parsl/executors/workqueue/exec_parsl_function.py +5 -4
- parsl/executors/workqueue/executor.py +26 -27
- parsl/executors/workqueue/parsl_coprocess.py +1 -1
- parsl/jobs/error_handlers.py +1 -1
- parsl/jobs/job_status_poller.py +2 -5
- parsl/jobs/states.py +1 -1
- parsl/jobs/strategy.py +2 -2
- parsl/launchers/__init__.py +12 -3
- parsl/launchers/errors.py +1 -1
- parsl/log_utils.py +1 -2
- parsl/monitoring/db_manager.py +16 -10
- parsl/monitoring/monitoring.py +11 -15
- parsl/monitoring/queries/pandas.py +1 -2
- parsl/monitoring/radios.py +2 -4
- parsl/monitoring/remote.py +13 -8
- parsl/monitoring/router.py +8 -11
- parsl/monitoring/types.py +2 -0
- parsl/monitoring/visualization/app.py +4 -2
- parsl/monitoring/visualization/models.py +0 -1
- parsl/monitoring/visualization/plots/default/workflow_plots.py +8 -4
- parsl/monitoring/visualization/plots/default/workflow_resource_plots.py +1 -0
- parsl/monitoring/visualization/utils.py +0 -1
- parsl/monitoring/visualization/views.py +16 -9
- parsl/multiprocessing.py +0 -1
- parsl/process_loggers.py +1 -2
- parsl/providers/__init__.py +9 -12
- parsl/providers/ad_hoc/ad_hoc.py +1 -1
- parsl/providers/aws/aws.py +2 -3
- parsl/providers/azure/azure.py +4 -5
- parsl/providers/base.py +1 -1
- parsl/providers/cluster_provider.py +1 -1
- parsl/providers/cobalt/cobalt.py +3 -3
- parsl/providers/condor/condor.py +4 -2
- parsl/providers/errors.py +2 -2
- parsl/providers/googlecloud/googlecloud.py +2 -1
- parsl/providers/grid_engine/grid_engine.py +2 -2
- parsl/providers/kubernetes/kube.py +5 -3
- parsl/providers/local/local.py +5 -1
- parsl/providers/lsf/lsf.py +2 -2
- parsl/providers/pbspro/pbspro.py +1 -1
- parsl/providers/slurm/slurm.py +5 -5
- parsl/providers/torque/torque.py +1 -1
- parsl/serialize/__init__.py +8 -3
- parsl/serialize/base.py +1 -2
- parsl/serialize/concretes.py +5 -4
- parsl/serialize/proxystore.py +3 -2
- parsl/tests/__init__.py +1 -1
- parsl/tests/configs/ad_hoc_cluster_htex.py +4 -4
- parsl/tests/configs/azure_single_node.py +4 -5
- parsl/tests/configs/bridges.py +3 -2
- parsl/tests/configs/cc_in2p3.py +2 -2
- parsl/tests/configs/comet.py +2 -1
- parsl/tests/configs/ec2_single_node.py +1 -2
- parsl/tests/configs/ec2_spot.py +1 -2
- parsl/tests/configs/frontera.py +3 -2
- parsl/tests/configs/htex_ad_hoc_cluster.py +2 -4
- parsl/tests/configs/htex_local.py +2 -3
- parsl/tests/configs/htex_local_alternate.py +8 -11
- parsl/tests/configs/htex_local_intask_staging.py +5 -7
- parsl/tests/configs/htex_local_rsync_staging.py +4 -6
- parsl/tests/configs/local_adhoc.py +1 -1
- parsl/tests/configs/local_radical.py +1 -3
- parsl/tests/configs/local_radical_mpi.py +2 -2
- parsl/tests/configs/midway.py +2 -2
- parsl/tests/configs/nscc_singapore.py +3 -3
- parsl/tests/configs/osg_htex.py +1 -1
- parsl/tests/configs/petrelkube.py +3 -2
- parsl/tests/configs/summit.py +1 -0
- parsl/tests/configs/swan_htex.py +2 -2
- parsl/tests/configs/taskvine_ex.py +3 -5
- parsl/tests/configs/theta.py +2 -2
- parsl/tests/configs/workqueue_ex.py +3 -4
- parsl/tests/conftest.py +4 -4
- parsl/tests/integration/test_channels/test_ssh_errors.py +1 -1
- parsl/tests/integration/test_stress/test_python_simple.py +3 -4
- parsl/tests/integration/test_stress/test_python_threads.py +3 -5
- parsl/tests/manual_tests/htex_local.py +4 -4
- parsl/tests/manual_tests/test_ad_hoc_htex.py +2 -1
- parsl/tests/manual_tests/test_basic.py +1 -0
- parsl/tests/manual_tests/test_fan_in_out_htex_remote.py +4 -4
- parsl/tests/manual_tests/test_log_filter.py +3 -1
- parsl/tests/manual_tests/test_memory_limits.py +6 -6
- parsl/tests/manual_tests/test_regression_220.py +2 -1
- parsl/tests/manual_tests/test_udp_simple.py +4 -3
- parsl/tests/manual_tests/test_worker_count.py +3 -2
- parsl/tests/scaling_tests/htex_local.py +2 -2
- parsl/tests/scaling_tests/test_scale.py +0 -9
- parsl/tests/scaling_tests/vineex_condor.py +1 -2
- parsl/tests/scaling_tests/vineex_local.py +1 -2
- parsl/tests/site_tests/test_provider.py +3 -1
- parsl/tests/site_tests/test_site.py +2 -0
- parsl/tests/sites/test_affinity.py +7 -5
- parsl/tests/sites/test_dynamic_executor.py +3 -3
- parsl/tests/sites/test_ec2.py +3 -2
- parsl/tests/sites/test_local_adhoc.py +2 -1
- parsl/tests/sites/test_worker_info.py +4 -3
- parsl/tests/test_aalst_patterns.py +0 -1
- parsl/tests/test_bash_apps/test_apptimeout.py +2 -2
- parsl/tests/test_bash_apps/test_error_codes.py +1 -4
- parsl/tests/test_bash_apps/test_memoize_ignore_args.py +1 -0
- parsl/tests/test_bash_apps/test_memoize_ignore_args_regr.py +2 -2
- parsl/tests/test_bash_apps/test_pipeline.py +1 -1
- parsl/tests/test_bash_apps/test_std_uri.py +4 -3
- parsl/tests/test_callables.py +2 -2
- parsl/tests/test_checkpointing/test_python_checkpoint_1.py +1 -0
- parsl/tests/test_checkpointing/test_python_checkpoint_2.py +2 -1
- parsl/tests/test_checkpointing/test_regression_239.py +1 -1
- parsl/tests/test_checkpointing/test_task_exit.py +1 -1
- parsl/tests/test_docs/test_from_slides.py +2 -2
- parsl/tests/test_docs/test_kwargs.py +1 -1
- parsl/tests/test_docs/test_tutorial_1.py +1 -2
- parsl/tests/test_docs/test_workflow1.py +2 -2
- parsl/tests/test_docs/test_workflow2.py +0 -1
- parsl/tests/test_error_handling/test_rand_fail.py +2 -2
- parsl/tests/test_error_handling/test_resource_spec.py +4 -2
- parsl/tests/test_error_handling/test_retries.py +2 -1
- parsl/tests/test_error_handling/test_retry_handler.py +1 -0
- parsl/tests/test_error_handling/test_retry_handler_failure.py +2 -1
- parsl/tests/test_error_handling/test_serialization_fail.py +1 -1
- parsl/tests/test_error_handling/test_wrap_with_logs.py +1 -0
- parsl/tests/test_flux.py +1 -1
- parsl/tests/test_htex/test_command_client_timeout.py +9 -12
- parsl/tests/test_htex/test_connected_blocks.py +3 -2
- parsl/tests/test_htex/test_cpu_affinity_explicit.py +5 -2
- parsl/tests/test_htex/test_disconnected_blocks.py +6 -4
- parsl/tests/test_htex/test_drain.py +5 -5
- parsl/tests/test_htex/test_htex.py +1 -2
- parsl/tests/test_htex/test_managers_command.py +3 -2
- parsl/tests/test_htex/test_multiple_disconnected_blocks.py +6 -4
- parsl/tests/test_monitoring/test_app_names.py +3 -2
- parsl/tests/test_monitoring/test_basic.py +4 -4
- parsl/tests/test_monitoring/test_db_locks.py +6 -3
- parsl/tests/test_monitoring/test_fuzz_zmq.py +6 -3
- parsl/tests/test_monitoring/test_htex_init_blocks_vs_monitoring.py +5 -5
- parsl/tests/test_monitoring/test_incomplete_futures.py +5 -3
- parsl/tests/test_monitoring/test_memoization_representation.py +4 -1
- parsl/tests/test_monitoring/test_stdouterr.py +4 -4
- parsl/tests/test_monitoring/test_viz_colouring.py +1 -0
- parsl/tests/test_mpi_apps/test_bad_mpi_config.py +1 -1
- parsl/tests/test_mpi_apps/test_mpi_mode_disabled.py +2 -0
- parsl/tests/test_mpi_apps/test_mpi_mode_enabled.py +7 -5
- parsl/tests/test_mpi_apps/test_mpi_prefix.py +4 -4
- parsl/tests/test_mpi_apps/test_mpi_scheduler.py +7 -2
- parsl/tests/test_mpi_apps/test_mpiex.py +4 -3
- parsl/tests/test_mpi_apps/test_resource_spec.py +9 -10
- parsl/tests/test_providers/test_cobalt_deprecation_warning.py +2 -0
- parsl/tests/test_providers/test_local_provider.py +2 -1
- parsl/tests/test_providers/test_pbspro_template.py +1 -1
- parsl/tests/test_providers/test_slurm_template.py +1 -1
- parsl/tests/test_providers/test_submiterror_deprecation.py +2 -1
- parsl/tests/test_python_apps/test_context_manager.py +4 -3
- parsl/tests/test_python_apps/test_dep_standard_futures.py +2 -1
- parsl/tests/test_python_apps/test_futures.py +2 -1
- parsl/tests/test_python_apps/test_join.py +0 -1
- parsl/tests/test_python_apps/test_lifted.py +3 -3
- parsl/tests/test_python_apps/test_memoize_bad_id_for_memo.py +1 -0
- parsl/tests/test_python_apps/test_pluggable_future_resolution.py +1 -1
- parsl/tests/test_radical/test_mpi_funcs.py +1 -1
- parsl/tests/test_regression/test_1480.py +2 -1
- parsl/tests/test_regression/test_1653.py +2 -1
- parsl/tests/test_regression/test_2652.py +1 -0
- parsl/tests/test_regression/test_69a.py +0 -1
- parsl/tests/test_regression/test_854.py +4 -2
- parsl/tests/test_regression/test_97_parallelism_0.py +1 -2
- parsl/tests/test_regression/test_98.py +0 -1
- parsl/tests/test_scaling/test_block_error_handler.py +9 -4
- parsl/tests/test_scaling/test_scale_down.py +2 -3
- parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py +4 -5
- parsl/tests/test_scaling/test_scale_down_htex_unregistered.py +3 -4
- parsl/tests/test_scaling/test_shutdown_scalein.py +1 -2
- parsl/tests/test_serialization/test_2555_caching_deserializer.py +1 -1
- parsl/tests/test_serialization/test_basic.py +2 -1
- parsl/tests/test_serialization/test_htex_code_cache.py +3 -4
- parsl/tests/test_serialization/test_pack_resource_spec.py +2 -1
- parsl/tests/test_serialization/test_proxystore_configured.py +10 -5
- parsl/tests/test_serialization/test_proxystore_impl.py +5 -3
- parsl/tests/test_shutdown/test_kill_monitoring.py +3 -2
- parsl/tests/test_staging/staging_provider.py +2 -2
- parsl/tests/test_staging/test_1316.py +3 -2
- parsl/tests/test_staging/test_docs_1.py +1 -1
- parsl/tests/test_staging/test_docs_2.py +2 -1
- parsl/tests/test_staging/test_elaborate_noop_file.py +2 -2
- parsl/tests/test_staging/test_staging_https.py +2 -2
- parsl/tests/test_staging/test_staging_stdout.py +4 -3
- parsl/tests/test_staging/test_zip_in.py +6 -8
- parsl/tests/test_staging/test_zip_out.py +7 -9
- parsl/tests/test_staging/test_zip_to_zip.py +6 -8
- parsl/tests/test_summary.py +2 -1
- parsl/tests/test_thread_parallelism.py +0 -1
- parsl/tests/test_threads/test_configs.py +1 -1
- parsl/tests/test_threads/test_lazy_errors.py +2 -1
- parsl/usage_tracking/api.py +2 -3
- parsl/usage_tracking/usage.py +8 -18
- parsl/utils.py +13 -2
- parsl/version.py +1 -1
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py +5 -4
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py +31 -20
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/METADATA +2 -2
- parsl-2024.6.3.dist-info/RECORD +471 -0
- parsl-2024.5.27.dist-info/RECORD +0 -471
- {parsl-2024.5.27.data → parsl-2024.6.3.data}/scripts/parsl_coprocess.py +1 -1
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/LICENSE +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/WHEEL +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/entry_points.txt +0 -0
- {parsl-2024.5.27.dist-info → parsl-2024.6.3.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,14 @@
|
|
1
1
|
import logging
|
2
2
|
import os
|
3
|
+
import pickle
|
3
4
|
from unittest import mock
|
5
|
+
|
4
6
|
import pytest
|
5
|
-
|
6
|
-
from parsl.executors.high_throughput.mpi_resource_management import
|
7
|
+
|
8
|
+
from parsl.executors.high_throughput.mpi_resource_management import (
|
9
|
+
MPITaskScheduler,
|
10
|
+
TaskScheduler,
|
11
|
+
)
|
7
12
|
from parsl.multiprocessing import SpawnContext
|
8
13
|
from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
|
9
14
|
|
@@ -5,11 +5,12 @@ from pathlib import Path
|
|
5
5
|
import pytest
|
6
6
|
|
7
7
|
import parsl
|
8
|
-
from
|
9
|
-
from parsl import
|
8
|
+
from parsl import Config, HighThroughputExecutor
|
9
|
+
from parsl.executors.high_throughput.mpi_executor import MPIExecutor
|
10
10
|
from parsl.launchers import SimpleLauncher
|
11
11
|
from parsl.providers import LocalProvider
|
12
|
-
|
12
|
+
|
13
|
+
from .test_mpi_mode_enabled import get_env_vars
|
13
14
|
|
14
15
|
cwd = Path(__file__).parent.absolute()
|
15
16
|
pbs_nodefile = cwd.joinpath("mocks", "pbs_nodefile")
|
@@ -2,26 +2,25 @@ import contextlib
|
|
2
2
|
import logging
|
3
3
|
import os
|
4
4
|
import typing
|
5
|
-
|
5
|
+
import unittest
|
6
|
+
from typing import Dict
|
6
7
|
|
7
8
|
import pytest
|
8
|
-
import unittest
|
9
9
|
|
10
10
|
import parsl
|
11
11
|
from parsl.app.app import python_app
|
12
|
-
from parsl.
|
13
|
-
|
12
|
+
from parsl.executors.high_throughput.mpi_prefix_composer import (
|
13
|
+
InvalidResourceSpecification,
|
14
|
+
MissingResourceSpecification,
|
15
|
+
validate_resource_spec,
|
16
|
+
)
|
14
17
|
from parsl.executors.high_throughput.mpi_resource_management import (
|
18
|
+
get_nodes_in_batchjob,
|
15
19
|
get_pbs_hosts_list,
|
16
20
|
get_slurm_hosts_list,
|
17
|
-
get_nodes_in_batchjob,
|
18
21
|
identify_scheduler,
|
19
22
|
)
|
20
|
-
from parsl.
|
21
|
-
validate_resource_spec,
|
22
|
-
InvalidResourceSpecification,
|
23
|
-
MissingResourceSpecification
|
24
|
-
)
|
23
|
+
from parsl.tests.configs.htex_local import fresh_config
|
25
24
|
|
26
25
|
EXECUTOR_LABEL = "MPI_TEST"
|
27
26
|
|
@@ -1,7 +1,6 @@
|
|
1
1
|
import logging
|
2
2
|
import os
|
3
3
|
import pathlib
|
4
|
-
import pytest
|
5
4
|
import random
|
6
5
|
import shutil
|
7
6
|
import socket
|
@@ -10,6 +9,8 @@ import tempfile
|
|
10
9
|
import threading
|
11
10
|
import time
|
12
11
|
|
12
|
+
import pytest
|
13
|
+
|
13
14
|
from parsl.channels import LocalChannel, SSHChannel
|
14
15
|
from parsl.jobs.states import JobState
|
15
16
|
from parsl.launchers import SingleNodeLauncher
|
@@ -1,8 +1,9 @@
|
|
1
|
-
import parsl
|
2
|
-
from parsl.tests.configs.local_threads import fresh_config
|
3
1
|
import pytest
|
4
|
-
|
2
|
+
|
3
|
+
import parsl
|
5
4
|
from parsl.dataflow.dflow import DataFlowKernel
|
5
|
+
from parsl.errors import NoDataFlowKernelError
|
6
|
+
from parsl.tests.configs.local_threads import fresh_config
|
6
7
|
|
7
8
|
|
8
9
|
@parsl.python_app
|
@@ -11,9 +11,10 @@ Same applies to datafutures, and we need to know the behavior wrt.
|
|
11
11
|
2. done() called on 1, vs 2
|
12
12
|
|
13
13
|
"""
|
14
|
-
import pytest
|
15
14
|
from os.path import basename
|
16
15
|
|
16
|
+
import pytest
|
17
|
+
|
17
18
|
from parsl.app.app import python_app
|
18
19
|
from parsl.data_provider.files import File
|
19
20
|
|
@@ -7,8 +7,8 @@ import pytest
|
|
7
7
|
|
8
8
|
import parsl
|
9
9
|
from parsl.config import Config
|
10
|
-
from parsl.dataflow.errors import DependencyError
|
11
10
|
from parsl.dataflow.dependency_resolvers import DEEP_DEPENDENCY_RESOLVER
|
11
|
+
from parsl.dataflow.errors import DependencyError
|
12
12
|
|
13
13
|
|
14
14
|
def local_config():
|
@@ -1,11 +1,10 @@
|
|
1
1
|
import pytest
|
2
2
|
|
3
3
|
import parsl
|
4
|
-
|
5
4
|
from parsl.config import Config
|
6
5
|
from parsl.executors import HighThroughputExecutor
|
7
|
-
from parsl.providers import LocalProvider
|
8
6
|
from parsl.launchers import SimpleLauncher
|
7
|
+
from parsl.providers import LocalProvider
|
9
8
|
|
10
9
|
|
11
10
|
def local_config() -> Config:
|
@@ -1,11 +1,16 @@
|
|
1
|
+
from functools import partial
|
2
|
+
from unittest.mock import Mock
|
3
|
+
|
1
4
|
import pytest
|
2
5
|
|
3
6
|
from parsl.executors import HighThroughputExecutor
|
7
|
+
from parsl.jobs.error_handlers import (
|
8
|
+
noop_error_handler,
|
9
|
+
simple_error_handler,
|
10
|
+
windowed_error_handler,
|
11
|
+
)
|
12
|
+
from parsl.jobs.states import JobState, JobStatus
|
4
13
|
from parsl.providers import LocalProvider
|
5
|
-
from unittest.mock import Mock
|
6
|
-
from parsl.jobs.states import JobStatus, JobState
|
7
|
-
from parsl.jobs.error_handlers import simple_error_handler, windowed_error_handler, noop_error_handler
|
8
|
-
from functools import partial
|
9
14
|
|
10
15
|
|
11
16
|
@pytest.mark.local
|
@@ -4,13 +4,12 @@ import time
|
|
4
4
|
import pytest
|
5
5
|
|
6
6
|
import parsl
|
7
|
-
|
8
7
|
from parsl import File, python_app
|
9
|
-
from parsl.providers import LocalProvider
|
10
8
|
from parsl.channels import LocalChannel
|
11
|
-
from parsl.launchers import SingleNodeLauncher
|
12
9
|
from parsl.config import Config
|
13
10
|
from parsl.executors import HighThroughputExecutor
|
11
|
+
from parsl.launchers import SingleNodeLauncher
|
12
|
+
from parsl.providers import LocalProvider
|
14
13
|
|
15
14
|
logger = logging.getLogger(__name__)
|
16
15
|
|
@@ -1,15 +1,14 @@
|
|
1
|
+
from threading import Event
|
2
|
+
|
1
3
|
import pytest
|
2
4
|
|
3
5
|
import parsl
|
4
|
-
|
5
6
|
from parsl import File, python_app
|
6
|
-
from parsl.providers import LocalProvider
|
7
7
|
from parsl.channels import LocalChannel
|
8
|
-
from parsl.launchers import SingleNodeLauncher
|
9
8
|
from parsl.config import Config
|
10
9
|
from parsl.executors import HighThroughputExecutor
|
11
|
-
|
12
|
-
from
|
10
|
+
from parsl.launchers import SingleNodeLauncher
|
11
|
+
from parsl.providers import LocalProvider
|
13
12
|
|
14
13
|
_max_blocks = 5
|
15
14
|
_min_blocks = 0
|
@@ -4,14 +4,13 @@ import time
|
|
4
4
|
import pytest
|
5
5
|
|
6
6
|
import parsl
|
7
|
-
|
8
7
|
from parsl import File, python_app
|
9
|
-
from parsl.jobs.states import JobState, TERMINAL_STATES
|
10
|
-
from parsl.providers import LocalProvider
|
11
8
|
from parsl.channels import LocalChannel
|
12
|
-
from parsl.launchers import SingleNodeLauncher
|
13
9
|
from parsl.config import Config
|
14
10
|
from parsl.executors import HighThroughputExecutor
|
11
|
+
from parsl.jobs.states import TERMINAL_STATES, JobState
|
12
|
+
from parsl.launchers import SingleNodeLauncher
|
13
|
+
from parsl.providers import LocalProvider
|
15
14
|
|
16
15
|
logger = logging.getLogger(__name__)
|
17
16
|
|
@@ -1,3 +1,4 @@
|
|
1
|
+
import random
|
1
2
|
import threading
|
2
3
|
|
3
4
|
import pytest
|
@@ -9,8 +10,6 @@ from parsl.executors import HighThroughputExecutor
|
|
9
10
|
from parsl.launchers import SimpleLauncher
|
10
11
|
from parsl.providers import LocalProvider
|
11
12
|
|
12
|
-
import random
|
13
|
-
|
14
13
|
# we need some blocks, but it doesn't matter too much how many, as long
|
15
14
|
# as they can all start up and get registered within the try_assert
|
16
15
|
# timeout later on.
|
@@ -1,10 +1,9 @@
|
|
1
|
-
import parsl
|
2
|
-
import pytest
|
3
|
-
|
4
1
|
from typing import Any
|
5
2
|
|
6
|
-
|
3
|
+
import pytest
|
7
4
|
|
5
|
+
import parsl
|
6
|
+
from parsl.serialize.facade import methods_for_code
|
8
7
|
from parsl.tests.configs.htex_local import fresh_config as local_config
|
9
8
|
|
10
9
|
|
@@ -1,19 +1,24 @@
|
|
1
1
|
import logging
|
2
|
-
import pytest
|
3
2
|
import uuid
|
4
3
|
|
4
|
+
import pytest
|
5
|
+
|
5
6
|
import parsl
|
6
|
-
from parsl.serialize.facade import
|
7
|
+
from parsl.serialize.facade import (
|
8
|
+
additional_methods_for_deserialization,
|
9
|
+
methods_for_data,
|
10
|
+
register_method_for_data,
|
11
|
+
)
|
7
12
|
from parsl.tests.configs.htex_local import fresh_config
|
8
13
|
|
9
|
-
|
10
14
|
logger = logging.getLogger(__name__)
|
11
15
|
|
12
16
|
|
13
17
|
def local_setup():
|
14
|
-
from parsl.serialize.proxystore import ProxyStoreSerializer
|
15
|
-
from proxystore.store import Store, register_store
|
16
18
|
from proxystore.connectors.file import FileConnector
|
19
|
+
from proxystore.store import Store, register_store
|
20
|
+
|
21
|
+
from parsl.serialize.proxystore import ProxyStoreSerializer
|
17
22
|
|
18
23
|
parsl.load(fresh_config())
|
19
24
|
|
@@ -1,6 +1,7 @@
|
|
1
|
-
import pytest
|
2
1
|
import uuid
|
3
2
|
|
3
|
+
import pytest
|
4
|
+
|
4
5
|
|
5
6
|
def policy_example(o):
|
6
7
|
"""Example policy will proxy only lists."""
|
@@ -13,10 +14,11 @@ def test_proxystore_nonglobal():
|
|
13
14
|
"""
|
14
15
|
# import in function, because proxystore is not importable in base parsl
|
15
16
|
# installation.
|
16
|
-
from
|
17
|
+
from proxystore.connectors.file import FileConnector
|
17
18
|
from proxystore.proxy import Proxy
|
18
19
|
from proxystore.store import Store, register_store
|
19
|
-
|
20
|
+
|
21
|
+
from parsl.serialize.proxystore import ProxyStoreSerializer
|
20
22
|
|
21
23
|
store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp"))
|
22
24
|
register_store(store)
|
@@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk):
|
|
62
62
|
|
63
63
|
|
64
64
|
def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True):
|
65
|
-
import time
|
66
65
|
import logging
|
66
|
+
import time
|
67
67
|
logger = logging.getLogger(__name__)
|
68
68
|
logger.info("stage_out_noop")
|
69
69
|
time.sleep(1)
|
@@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk):
|
|
75
75
|
|
76
76
|
|
77
77
|
def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True):
|
78
|
-
import time
|
79
78
|
import logging
|
79
|
+
import time
|
80
80
|
logger = logging.getLogger(__name__)
|
81
81
|
logger.info("stage_in_noop")
|
82
82
|
time.sleep(1)
|
@@ -1,8 +1,9 @@
|
|
1
|
+
import time
|
2
|
+
|
1
3
|
import pytest
|
2
4
|
|
3
5
|
import parsl
|
4
|
-
import
|
5
|
-
from parsl import python_app, ThreadPoolExecutor
|
6
|
+
from parsl import ThreadPoolExecutor, python_app
|
6
7
|
from parsl.config import Config
|
7
8
|
from parsl.data_provider.files import File
|
8
9
|
from parsl.data_provider.staging import Staging
|
@@ -6,15 +6,15 @@
|
|
6
6
|
# of the globus staging provider
|
7
7
|
|
8
8
|
import logging
|
9
|
+
|
9
10
|
import pytest
|
10
11
|
|
11
12
|
import parsl
|
12
|
-
|
13
13
|
from parsl import bash_app, python_app
|
14
14
|
from parsl.config import Config
|
15
15
|
from parsl.data_provider.files import File
|
16
16
|
from parsl.executors.threads import ThreadPoolExecutor
|
17
|
-
from parsl.tests.test_staging.staging_provider import
|
17
|
+
from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging
|
18
18
|
|
19
19
|
logger = logging.getLogger(__name__)
|
20
20
|
|
@@ -1,9 +1,9 @@
|
|
1
|
+
import pytest
|
2
|
+
|
1
3
|
import parsl
|
2
4
|
from parsl.app.app import python_app
|
3
5
|
from parsl.data_provider.files import File
|
4
6
|
|
5
|
-
import pytest
|
6
|
-
|
7
7
|
# This config is for the local test which will adding an executor.
|
8
8
|
# Most tests in this file should be non-local and use the configuration
|
9
9
|
# specificed with --config, not this one.
|
@@ -1,12 +1,13 @@
|
|
1
1
|
import logging
|
2
2
|
import os
|
3
|
-
import parsl
|
4
|
-
import pytest
|
5
3
|
import zipfile
|
6
4
|
|
5
|
+
import pytest
|
6
|
+
|
7
|
+
import parsl
|
7
8
|
from parsl.app.futures import DataFuture
|
8
|
-
from parsl.tests.configs.htex_local import fresh_config as local_config
|
9
9
|
from parsl.data_provider.files import File
|
10
|
+
from parsl.tests.configs.htex_local import fresh_config as local_config
|
10
11
|
|
11
12
|
|
12
13
|
@parsl.bash_app
|
@@ -1,18 +1,16 @@
|
|
1
|
-
import parsl
|
2
|
-
import pytest
|
3
1
|
import random
|
4
2
|
import zipfile
|
5
3
|
|
6
|
-
|
7
|
-
from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
|
4
|
+
import pytest
|
8
5
|
|
9
|
-
|
6
|
+
import parsl
|
10
7
|
from parsl.channels import LocalChannel
|
11
|
-
from parsl.launchers import SimpleLauncher
|
12
|
-
|
13
8
|
from parsl.config import Config
|
9
|
+
from parsl.data_provider.files import File
|
10
|
+
from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
|
14
11
|
from parsl.executors import HighThroughputExecutor
|
15
|
-
|
12
|
+
from parsl.launchers import SimpleLauncher
|
13
|
+
from parsl.providers import LocalProvider
|
16
14
|
from parsl.tests.configs.htex_local import fresh_config as local_config
|
17
15
|
|
18
16
|
|
@@ -1,18 +1,16 @@
|
|
1
|
-
import parsl
|
2
|
-
import pytest
|
3
1
|
import zipfile
|
4
2
|
|
5
|
-
|
6
|
-
from parsl.data_provider.data_manager import default_staging
|
7
|
-
from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
|
3
|
+
import pytest
|
8
4
|
|
9
|
-
|
5
|
+
import parsl
|
10
6
|
from parsl.channels import LocalChannel
|
11
|
-
from parsl.launchers import SimpleLauncher
|
12
|
-
|
13
7
|
from parsl.config import Config
|
8
|
+
from parsl.data_provider.data_manager import default_staging
|
9
|
+
from parsl.data_provider.files import File
|
10
|
+
from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
|
14
11
|
from parsl.executors import HighThroughputExecutor
|
15
|
-
|
12
|
+
from parsl.launchers import SimpleLauncher
|
13
|
+
from parsl.providers import LocalProvider
|
16
14
|
from parsl.tests.configs.htex_local import fresh_config as local_config
|
17
15
|
|
18
16
|
|