parsl 2024.5.20__py3-none-any.whl → 2024.6.3__py3-none-any.whl

parsl/tests/test_scaling/test_scale_down_htex_auto_scale.py

@@ -1,15 +1,14 @@
+from threading import Event
+
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
-
-from threading import Event
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 _max_blocks = 5
 _min_blocks = 0

parsl/tests/test_scaling/test_scale_down_htex_unregistered.py

@@ -4,14 +4,13 @@ import time
 import pytest
 
 import parsl
-
 from parsl import File, python_app
-from parsl.jobs.states import JobState, TERMINAL_STATES
-from parsl.providers import LocalProvider
 from parsl.channels import LocalChannel
-from parsl.launchers import SingleNodeLauncher
 from parsl.config import Config
 from parsl.executors import HighThroughputExecutor
+from parsl.jobs.states import TERMINAL_STATES, JobState
+from parsl.launchers import SingleNodeLauncher
+from parsl.providers import LocalProvider
 
 logger = logging.getLogger(__name__)
 

parsl/tests/test_scaling/test_shutdown_scalein.py

@@ -1,3 +1,4 @@
+import random
 import threading
 
 import pytest
@@ -9,8 +10,6 @@ from parsl.executors import HighThroughputExecutor
 from parsl.launchers import SimpleLauncher
 from parsl.providers import LocalProvider
 
-import random
-
 # we need some blocks, but it doesn't matter too much how many, as long
 # as they can all start up and get registered within the try_assert
 # timeout later on.
@@ -72,7 +71,5 @@ def test_shutdown_scalein_blocks(tmpd_cwd, try_assert):
         # this will wait for everything to be scaled out fully
         try_assert(lambda: len(htex.connected_managers()) == BLOCK_COUNT)
 
-    parsl.clear()
-
     assert len(accumulating_provider.submit_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been launched"
     assert len(accumulating_provider.cancel_job_ids) == BLOCK_COUNT, f"Exactly {BLOCK_COUNT} blocks should have been scaled in"

parsl/tests/test_serialization/test_2555_caching_deserializer.py

@@ -1,6 +1,6 @@
-import parsl
 import pytest
 
+import parsl
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_basic.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import serialize, deserialize
+
+from parsl.serialize import deserialize, serialize
 from parsl.serialize.concretes import DillSerializer, PickleSerializer
 
 

parsl/tests/test_serialization/test_htex_code_cache.py

@@ -1,10 +1,9 @@
-import parsl
-import pytest
-
 from typing import Any
 
-from parsl.serialize.facade import methods_for_code
+import pytest
 
+import parsl
+from parsl.serialize.facade import methods_for_code
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_serialization/test_pack_resource_spec.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl.serialize import unpack_res_spec_apply_message, pack_res_spec_apply_message
+
+from parsl.serialize import pack_res_spec_apply_message, unpack_res_spec_apply_message
 
 
 def double(x: int, y: int = 2) -> int:

parsl/tests/test_serialization/test_proxystore_configured.py

@@ -1,19 +1,24 @@
 import logging
-import pytest
 import uuid
 
+import pytest
+
 import parsl
-from parsl.serialize.facade import additional_methods_for_deserialization, methods_for_data, register_method_for_data
+from parsl.serialize.facade import (
+    additional_methods_for_deserialization,
+    methods_for_data,
+    register_method_for_data,
+)
 from parsl.tests.configs.htex_local import fresh_config
 
-
 logger = logging.getLogger(__name__)
 
 
 def local_setup():
-    from parsl.serialize.proxystore import ProxyStoreSerializer
-    from proxystore.store import Store, register_store
     from proxystore.connectors.file import FileConnector
+    from proxystore.store import Store, register_store
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     parsl.load(fresh_config())
 
@@ -38,7 +43,6 @@ def local_setup():
 
 def local_teardown():
     parsl.dfk().cleanup()
-    parsl.clear()
 
     methods_for_data.clear()
     methods_for_data.update(previous_methods)

parsl/tests/test_serialization/test_proxystore_impl.py

@@ -1,6 +1,7 @@
-import pytest
 import uuid
 
+import pytest
+
 
 def policy_example(o):
     """Example policy will proxy only lists."""
@@ -13,10 +14,11 @@ def test_proxystore_nonglobal():
     """
     # import in function, because proxystore is not importable in base parsl
     # installation.
-    from parsl.serialize.proxystore import ProxyStoreSerializer
+    from proxystore.connectors.file import FileConnector
     from proxystore.proxy import Proxy
     from proxystore.store import Store, register_store
-    from proxystore.connectors.file import FileConnector
+
+    from parsl.serialize.proxystore import ProxyStoreSerializer
 
     store = Store(name='parsl_store_' + str(uuid.uuid4()), connector=FileConnector(store_dir="/tmp"))
     register_store(store)

parsl/tests/test_shutdown/test_kill_monitoring.py

@@ -1,9 +1,10 @@
 import os
-import parsl
-import pytest
 import signal
 import time
 
+import pytest
+
+import parsl
 from parsl.tests.configs.htex_local_alternate import fresh_config
 
 # This is a very generous upper bound on expected shutdown time of target
@@ -25,7 +26,6 @@ def test_no_kills():
     assert parsl.dfk().monitoring is not None, "This test requires monitoring"
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local
@@ -62,4 +62,3 @@ def test_kill_monitoring_helper_process(sig, process_attr, try_assert):
     simple_app().result()
 
     parsl.dfk().cleanup()
-    parsl.clear()

parsl/tests/test_staging/staging_provider.py

@@ -62,8 +62,8 @@ def make_stage_out_app(executor, dfk):
 
 
 def stage_out_noop(app_fu, inputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_out_noop")
     time.sleep(1)
@@ -75,8 +75,8 @@ def make_stage_in_app(executor, dfk):
 
 
 def stage_in_noop(parent_fut=None, outputs=[], _parsl_staging_inhibit=True):
-    import time
     import logging
+    import time
     logger = logging.getLogger(__name__)
     logger.info("stage_in_noop")
     time.sleep(1)

parsl/tests/test_staging/test_1316.py

@@ -1,8 +1,9 @@
+import time
+
 import pytest
 
 import parsl
-import time
-from parsl import python_app, ThreadPoolExecutor
+from parsl import ThreadPoolExecutor, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.data_provider.staging import Staging
@@ -60,7 +61,6 @@ def test_1316_local_path_on_execution_side_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local
@@ -83,4 +83,3 @@ def test_1316_local_path_setting_preserves_dependency_sp2():
     assert not file.local_path, "The local_path on the submit side should not be set"
 
     parsl.dfk().cleanup()
-    parsl.clear()

parsl/tests/test_staging/test_docs_1.py

@@ -1,6 +1,6 @@
 import pytest
 
-from parsl import python_app, File
+from parsl import File, python_app
 
 
 @python_app

parsl/tests/test_staging/test_docs_2.py

@@ -1,5 +1,6 @@
 import pytest
-from parsl import bash_app, File
+
+from parsl import File, bash_app
 from parsl.tests.configs.local_threads import fresh_config as local_config
 
 

parsl/tests/test_staging/test_elaborate_noop_file.py

@@ -6,15 +6,15 @@
 # of the globus staging provider
 
 import logging
+
 import pytest
 
 import parsl
-
 from parsl import bash_app, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.tests.test_staging.staging_provider import NoOpTestingFileStaging, NoOpError
+from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging
 
 logger = logging.getLogger(__name__)
 
@@ -44,7 +44,6 @@ def storage_access_parsl():
     yield _setup_config
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
 
 @pytest.mark.local

parsl/tests/test_staging/test_staging_https.py

@@ -1,9 +1,9 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 
-import pytest
-
 # This config is for the local test which will adding an executor.
 # Most tests in this file should be non-local and use the configuration
 # specificed with --config, not this one.

parsl/tests/test_staging/test_staging_stdout.py

@@ -1,12 +1,13 @@
 import logging
 import os
-import parsl
-import pytest
 import zipfile
 
+import pytest
+
+import parsl
 from parsl.app.futures import DataFuture
-from parsl.tests.configs.htex_local import fresh_config as local_config
 from parsl.data_provider.files import File
+from parsl.tests.configs.htex_local import fresh_config as local_config
 
 
 @parsl.bash_app

parsl/tests/test_staging/test_zip_in.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import random
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_staging/test_zip_out.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.data_manager import default_staging
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_staging/test_zip_to_zip.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import random
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_summary.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl.tests.configs.local_threads import fresh_config
 
 
@@ -22,7 +23,6 @@ def test_summary(caplog):
     fail().exception()
 
     parsl.dfk().cleanup()
-    parsl.clear()
 
     assert "Summary of tasks in DFK:" in caplog.text
     assert "Tasks in state States.exec_done: 1" in caplog.text

parsl/tests/test_thread_parallelism.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_threads/test_configs.py

@@ -7,9 +7,9 @@ from parsl.tests.configs.local_threads import fresh_config
 
 @python_app
 def worker_identify(x, sleep_dur=0.2):
-    import time
     import os
     import threading
+    import time
     time.sleep(sleep_dur)
     return {"pid": os.getpid(),
             "tid": threading.current_thread()}
@@ -30,5 +30,4 @@ def test_parallel_for():
     assert thread_count <= config.executors[0].max_threads, "More threads than allowed"
     assert process_count == 1, "More processes than allowed"
     dfk.cleanup()
-    parsl.clear()
     return d

parsl/tests/test_threads/test_lazy_errors.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl import python_app
 from parsl.tests.configs.local_threads import fresh_config
 
@@ -24,5 +25,4 @@ def test_lazy_behavior():
         assert f.done()
 
     parsl.dfk().cleanup()
-    parsl.clear()
     return

parsl/usage_tracking/api.py

@@ -1,11 +1,9 @@
 import inspect
-
-from parsl.utils import RepresentationMixin
-
 from abc import abstractmethod
 from functools import singledispatch
 from typing import Any, List, Sequence
 
+from parsl.utils import RepresentationMixin
 
 # Traverse the configuration hierarchy, returning a JSON component
 # for each one. Configuration components which implement
@@ -14,6 +12,7 @@ from typing import Any, List, Sequence
 # are traversed in sequence. Other types default to reporting no
 # usage information.
 
+
 @singledispatch
 def get_parsl_usage(obj) -> List[Any]:
     return []

parsl/usage_tracking/usage.py

@@ -1,21 +1,21 @@
-import uuid
-import time
-import os
 import json
 import logging
+import platform
 import socket
 import sys
-import platform
+import time
+import uuid
 
+from parsl.dataflow.states import States
+from parsl.multiprocessing import ForkProcess
 from parsl.usage_tracking.api import get_parsl_usage
 from parsl.utils import setproctitle
-from parsl.multiprocessing import ForkProcess
-from parsl.dataflow.states import States
 from parsl.version import VERSION as PARSL_VERSION
 
 logger = logging.getLogger(__name__)
 
 from typing import Callable
+
 from typing_extensions import ParamSpec
 
 # protocol version byte: when (for example) compression parameters are changed
@@ -116,22 +116,12 @@ class UsageTracker:
     def check_tracking_enabled(self):
         """Check if tracking is enabled.
 
-        Tracking will be enabled unless either of these is true:
+        Tracking will be enabled unless the following is true:
 
             1. dfk.config.usage_tracking is set to False
-            2. Environment variable PARSL_TRACKING is set to false (case insensitive)
 
         """
-        track = True
-
-        if not self.config.usage_tracking:
-            track = False
-
-        envvar = str(os.environ.get("PARSL_TRACKING", True)).lower()
-        if envvar == "false":
-            track = False
-
-        return track
+        return self.config.usage_tracking
 
     def construct_start_message(self) -> bytes:
         """Collect preliminary run info at the start of the DFK.

parsl/utils.py

@@ -7,7 +7,19 @@ import threading
 import time
 from contextlib import contextmanager
 from types import TracebackType
-from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
+from typing import (
+    IO,
+    Any,
+    AnyStr,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 import typeguard
 from typing_extensions import Type
@@ -16,7 +28,6 @@ import parsl
 from parsl.app.errors import BadStdStreamFile
 from parsl.version import VERSION
 
-
 try:
     import setproctitle as setproctitle_module
 except ImportError:

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.05.20'
+VERSION = '2024.06.03'

{parsl-2024.5.20.data → parsl-2024.6.3.data}/scripts/exec_parsl_function.py

@@ -1,10 +1,11 @@
+import pickle
+import sys
+import traceback
+
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
-import traceback
-import sys
-import pickle
 from parsl.serialize import serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in a file:
 #

{parsl-2024.5.20.data → parsl-2024.6.3.data}/scripts/process_worker_pool.py

@@ -1,39 +1,41 @@
 #!python
 
 import argparse
+import json
 import logging
+import math
+import multiprocessing
 import os
-import sys
+import pickle
 import platform
+import queue
+import sys
 import threading
-import pickle
 import time
-import queue
 import uuid
-from typing import Sequence, Optional, Dict, List
-
-import zmq
-import math
-import json
-import psutil
-import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
+from typing import Dict, List, Optional, Sequence
+
+import psutil
+import zmq
 
 from parsl import curvezmq
-from parsl.process_loggers import wrap_with_logs
-from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
-from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    VALID_LAUNCHERS,
+    compose_all,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
     TaskScheduler,
-    MPITaskScheduler
 )
-
-from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
+from parsl.executors.high_throughput.probe import probe_addresses
+from parsl.multiprocessing import SpawnContext
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 DRAINED_CODE = (2 ** 32) - 2
@@ -677,7 +679,8 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
+        # OSX does not implement os.sched_getaffinity
+        avail_cores = sorted(os.sched_getaffinity(0))  # type: ignore[attr-defined, unused-ignore]
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -717,7 +720,15 @@ def worker(
         os.environ["KMP_AFFINITY"] = f"explicit,proclist=[{proc_list}]"  # For Intel OpenMP
 
         # Set the affinity for this worker
-        os.sched_setaffinity(0, my_cores)
+        # OSX does not implement os.sched_setaffinity so type checking
+        # is ignored here in two ways:
+        # On a platform without sched_setaffinity, that attribute will not
+        # be defined, so ignore[attr-defined] will tell mypy to ignore this
+        # incorrect-for-OS X attribute access.
+        # On a platform with sched_setaffinity, that type: ignore message
+        # will be redundant, and ignore[unused-ignore] tells mypy to ignore
+        # that this ignore is unneeded.
+        os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
     # If desired, pin to accelerator