parsl 2024.5.27__py3-none-any.whl → 2024.6.10__py3-none-any.whl

parsl/tests/test_staging/test_elaborate_noop_file.py

@@ -6,15 +6,15 @@
 # of the globus staging provider
 
 import logging
+
 import pytest
 
 import parsl
-
 from parsl import bash_app, python_app
 from parsl.config import Config
 from parsl.data_provider.files import File
 from parsl.executors.threads import ThreadPoolExecutor
-from parsl.tests.test_staging.staging_provider import NoOpTestingFileStaging, NoOpError
+from parsl.tests.test_staging.staging_provider import NoOpError, NoOpTestingFileStaging
 
 logger = logging.getLogger(__name__)
 

parsl/tests/test_staging/test_staging_https.py

@@ -1,9 +1,9 @@
+import pytest
+
 import parsl
 from parsl.app.app import python_app
 from parsl.data_provider.files import File
 
-import pytest
-
 # This config is for the local test which will adding an executor.
 # Most tests in this file should be non-local and use the configuration
 # specificed with --config, not this one.

parsl/tests/test_staging/test_staging_stdout.py

@@ -1,12 +1,13 @@
 import logging
 import os
-import parsl
-import pytest
 import zipfile
 
+import pytest
+
+import parsl
 from parsl.app.futures import DataFuture
-from parsl.tests.configs.htex_local import fresh_config as local_config
 from parsl.data_provider.files import File
+from parsl.tests.configs.htex_local import fresh_config as local_config
 
 
 @parsl.bash_app

parsl/tests/test_staging/test_zip_in.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import random
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_staging/test_zip_out.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.data_manager import default_staging
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.data_manager import default_staging
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_staging/test_zip_to_zip.py

@@ -1,18 +1,16 @@
-import parsl
-import pytest
 import random
 import zipfile
 
-from parsl.data_provider.files import File
-from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
+import pytest
 
-from parsl.providers import LocalProvider
+import parsl
 from parsl.channels import LocalChannel
-from parsl.launchers import SimpleLauncher
-
 from parsl.config import Config
+from parsl.data_provider.files import File
+from parsl.data_provider.zip import ZipAuthorityError, ZipFileStaging
 from parsl.executors import HighThroughputExecutor
-
+from parsl.launchers import SimpleLauncher
+from parsl.providers import LocalProvider
 from parsl.tests.configs.htex_local import fresh_config as local_config
 
 

parsl/tests/test_summary.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl.tests.configs.local_threads import fresh_config
 
 

parsl/tests/test_thread_parallelism.py

@@ -5,7 +5,6 @@ import pytest
 from parsl.app.app import bash_app, python_app
 from parsl.tests.configs.local_threads import config
 
-
 local_config = config
 
 

parsl/tests/test_threads/test_configs.py

@@ -7,9 +7,9 @@ from parsl.tests.configs.local_threads import fresh_config
 
 @python_app
 def worker_identify(x, sleep_dur=0.2):
-    import time
     import os
     import threading
+    import time
     time.sleep(sleep_dur)
     return {"pid": os.getpid(),
             "tid": threading.current_thread()}

parsl/tests/test_threads/test_lazy_errors.py

@@ -1,5 +1,6 @@
-import parsl
 import pytest
+
+import parsl
 from parsl import python_app
 from parsl.tests.configs.local_threads import fresh_config
 

parsl/tests/unit/test_usage_tracking.py

@@ -0,0 +1,45 @@
+"""Test usage_tracking values."""
+
+import pytest
+
+import parsl
+from parsl.config import Config
+from parsl.errors import ConfigurationError
+
+
+@pytest.mark.local
+def test_config_load():
+    """Test loading a config with usage tracking."""
+    with parsl.load(Config(usage_tracking=3)):
+        pass
+    parsl.clear()
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", (0, 1, 2, 3, False, True))
+def test_valid(level):
+    """Test valid usage_tracking values."""
+    Config(usage_tracking=level)
+    assert Config(usage_tracking=level).usage_tracking == level
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", (12, 1000, -1))
+def test_invalid_values(level):
+    """Test invalid usage_tracking values."""
+    with pytest.raises(ConfigurationError):
+        Config(usage_tracking=level)
+
+
+@pytest.mark.local
+@pytest.mark.parametrize("level", ("abcd", None, bytes(1), 1.0, 1j, object()))
+def test_invalid_types(level):
+    """Test invalid usage_tracking types."""
+    with pytest.raises(Exception) as ex:
+        Config(usage_tracking=level)
+
+    # with typeguard 4.x this is TypeCheckError,
+    # with typeguard 2.x this is TypeError
+    # we can't instantiate TypeCheckError if we're in typeguard 2.x environment
+    # because it does not exist... so check name using strings.
+    assert ex.type.__name__ in ["TypeCheckError", "TypeError"]

parsl/usage_tracking/api.py

@@ -1,11 +1,9 @@
 import inspect
-
-from parsl.utils import RepresentationMixin
-
 from abc import abstractmethod
 from functools import singledispatch
 from typing import Any, List, Sequence
 
+from parsl.utils import RepresentationMixin
 
 # Traverse the configuration hierarchy, returning a JSON component
 # for each one. Configuration components which implement
@@ -14,6 +12,7 @@ from typing import Any, List, Sequence
 # are traversed in sequence. Other types default to reporting no
 # usage information.
 
+
 @singledispatch
 def get_parsl_usage(obj) -> List[Any]:
     return []

parsl/usage_tracking/levels.py

@@ -0,0 +1,6 @@
+"""Module for defining the usage tracking levels."""
+
+DISABLED = 0  # Tracking is disabled
+LEVEL_1 = 1  # Share info about Parsl version, Python version, platform
+LEVEL_2 = 2  # Share info about config + level 1
+LEVEL_3 = 3  # Share info about app count, app fails, execution time + level 2

parsl/usage_tracking/usage.py

@@ -1,21 +1,24 @@
-import uuid
-import time
-import os
 import json
 import logging
+import platform
 import socket
 import sys
-import platform
+import time
+import uuid
 
+from parsl.dataflow.states import States
+from parsl.errors import ConfigurationError
+from parsl.multiprocessing import ForkProcess
 from parsl.usage_tracking.api import get_parsl_usage
+from parsl.usage_tracking.levels import DISABLED as USAGE_TRACKING_DISABLED
+from parsl.usage_tracking.levels import LEVEL_3 as USAGE_TRACKING_LEVEL_3
 from parsl.utils import setproctitle
-from parsl.multiprocessing import ForkProcess
-from parsl.dataflow.states import States
 from parsl.version import VERSION as PARSL_VERSION
 
 logger = logging.getLogger(__name__)
 
 from typing import Callable
+
 from typing_extensions import ParamSpec
 
 # protocol version byte: when (for example) compression parameters are changed
@@ -110,28 +113,33 @@ class UsageTracker:
         self.python_version = "{}.{}.{}".format(sys.version_info.major,
                                                 sys.version_info.minor,
                                                 sys.version_info.micro)
-        self.tracking_enabled = self.check_tracking_enabled()
-        logger.debug("Tracking status: {}".format(self.tracking_enabled))
-
-    def check_tracking_enabled(self):
-        """Check if tracking is enabled.
-
-        Tracking will be enabled unless either of these is true:
-
-            1. dfk.config.usage_tracking is set to False
-            2. Environment variable PARSL_TRACKING is set to false (case insensitive)
-
+        self.tracking_level = self.check_tracking_level()
+        self.start_time = None
+        logger.debug("Tracking level: {}".format(self.tracking_level))
+
+    def check_tracking_level(self) -> int:
+        """Check if tracking is enabled and return level.
+
+        Checks usage_tracking in Config
+            - Possible values: [True, False, 0, 1, 2, 3]
+
+        True/False values are treated as Level 1/Level 0 respectively.
+
+        Returns: int
+            - 0 : Tracking is disabled
+            - 1 : Tracking is enabled with level 1
+                  Share info about Parsl version, Python version, platform
+            - 2 : Tracking is enabled with level 2
+                  Share info about config + level 1
+            - 3 : Tracking is enabled with level 3
+                  Share info about app count, app fails, execution time + level 2
         """
-        track = True
-
-        if not self.config.usage_tracking:
-            track = False
+        if not USAGE_TRACKING_DISABLED <= self.config.usage_tracking <= USAGE_TRACKING_LEVEL_3:
+            raise ConfigurationError(
+                f"Usage Tracking values must be 0, 1, 2, or 3 and not {self.config.usage_tracking}"
+            )
 
-        envvar = str(os.environ.get("PARSL_TRACKING", True)).lower()
-        if envvar == "false":
-            track = False
-
-        return track
+        return self.config.usage_tracking
 
     def construct_start_message(self) -> bytes:
         """Collect preliminary run info at the start of the DFK.
@@ -143,18 +151,28 @@ class UsageTracker:
                    'parsl_v': self.parsl_version,
                    'python_v': self.python_version,
                    'platform.system': platform.system(),
-                   'start': int(time.time()),
-                   'components': get_parsl_usage(self.dfk._config)}
+                   'tracking_level': int(self.tracking_level)}
+
+        if self.tracking_level >= 2:
+            message['components'] = get_parsl_usage(self.dfk._config)
+
+        if self.tracking_level == 3:
+            self.start_time = int(time.time())
+            message['start'] = self.start_time
+
         logger.debug(f"Usage tracking start message: {message}")
 
         return self.encode_message(message)
 
     def construct_end_message(self) -> bytes:
         """Collect the final run information at the time of DFK cleanup.
+        This is only called if tracking level is 3.
 
         Returns:
              - Message dict dumped as json string, ready for UDP
         """
+        end_time = int(time.time())
+
         app_count = self.dfk.task_count
 
         app_fails = self.dfk.task_state_counts[States.failed] + self.dfk.task_state_counts[States.dep_fail]
@@ -167,7 +185,8 @@ class UsageTracker:
                          'app_fails': app_fails}
 
         message = {'correlator': self.correlator_uuid,
-                   'end': int(time.time()),
+                   'end': end_time,
+                   'execution_time': end_time - self.start_time,
                    'components': [dfk_component] + get_parsl_usage(self.dfk._config)}
         logger.debug(f"Usage tracking end message (unencoded): {message}")
 
@@ -178,20 +197,22 @@ class UsageTracker:
 
     def send_UDP_message(self, message: bytes) -> None:
         """Send UDP message."""
-        if self.tracking_enabled:
-            try:
-                proc = udp_messenger(self.domain_name, self.UDP_PORT, self.sock_timeout, message)
-                self.procs.append(proc)
-            except Exception as e:
-                logger.debug("Usage tracking failed: {}".format(e))
+        try:
+            proc = udp_messenger(self.domain_name, self.UDP_PORT, self.sock_timeout, message)
+            self.procs.append(proc)
+        except Exception as e:
+            logger.debug("Usage tracking failed: {}".format(e))
 
     def send_start_message(self) -> None:
-        message = self.construct_start_message()
-        self.send_UDP_message(message)
+        if self.tracking_level:
+            self.start_time = time.time()
+            message = self.construct_start_message()
+            self.send_UDP_message(message)
 
     def send_end_message(self) -> None:
-        message = self.construct_end_message()
-        self.send_UDP_message(message)
+        if self.tracking_level == 3:
+            message = self.construct_end_message()
+            self.send_UDP_message(message)
 
     def close(self, timeout: float = 10.0) -> None:
         """First give each process one timeout period to finish what it is

parsl/utils.py

@@ -7,7 +7,19 @@ import threading
 import time
 from contextlib import contextmanager
 from types import TracebackType
-from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
+from typing import (
+    IO,
+    Any,
+    AnyStr,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 import typeguard
 from typing_extensions import Type
@@ -16,7 +28,6 @@ import parsl
 from parsl.app.errors import BadStdStreamFile
 from parsl.version import VERSION
 
-
 try:
     import setproctitle as setproctitle_module
 except ImportError:

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.05.27'
+VERSION = '2024.06.10'

{parsl-2024.5.27.data → parsl-2024.6.10.data}/scripts/exec_parsl_function.py

@@ -1,10 +1,11 @@
+import pickle
+import sys
+import traceback
+
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
-import traceback
-import sys
-import pickle
 from parsl.serialize import serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in a file:
 #

{parsl-2024.5.27.data → parsl-2024.6.10.data}/scripts/process_worker_pool.py

@@ -1,39 +1,41 @@
 #!python
 
 import argparse
+import json
 import logging
+import math
+import multiprocessing
 import os
-import sys
+import pickle
 import platform
+import queue
+import sys
 import threading
-import pickle
 import time
-import queue
 import uuid
-from typing import Sequence, Optional, Dict, List
-
-import zmq
-import math
-import json
-import psutil
-import multiprocessing
 from multiprocessing.managers import DictProxy
 from multiprocessing.sharedctypes import Synchronized
+from typing import Dict, List, Optional, Sequence
+
+import psutil
+import zmq
 
 from parsl import curvezmq
-from parsl.process_loggers import wrap_with_logs
-from parsl.version import VERSION as PARSL_VERSION
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.executors.high_throughput.errors import WorkerLost
-from parsl.executors.high_throughput.probe import probe_addresses
-from parsl.multiprocessing import SpawnContext
-from parsl.serialize import unpack_res_spec_apply_message, serialize
+from parsl.executors.high_throughput.mpi_prefix_composer import (
+    VALID_LAUNCHERS,
+    compose_all,
+)
 from parsl.executors.high_throughput.mpi_resource_management import (
+    MPITaskScheduler,
     TaskScheduler,
-    MPITaskScheduler
 )
-
-from parsl.executors.high_throughput.mpi_prefix_composer import compose_all, VALID_LAUNCHERS
+from parsl.executors.high_throughput.probe import probe_addresses
+from parsl.multiprocessing import SpawnContext
+from parsl.process_loggers import wrap_with_logs
+from parsl.serialize import serialize, unpack_res_spec_apply_message
+from parsl.version import VERSION as PARSL_VERSION
 
 HEARTBEAT_CODE = (2 ** 32) - 1
 DRAINED_CODE = (2 ** 32) - 2
@@ -677,7 +679,8 @@ def worker(
     # If desired, set process affinity
     if cpu_affinity != "none":
         # Count the number of cores per worker
-        avail_cores = sorted(os.sched_getaffinity(0))  # Get the available threads
+        # OSX does not implement os.sched_getaffinity
+        avail_cores = sorted(os.sched_getaffinity(0))  # type: ignore[attr-defined, unused-ignore]
         cores_per_worker = len(avail_cores) // pool_size
         assert cores_per_worker > 0, "Affinity does not work if there are more workers than cores"
 
@@ -717,7 +720,15 @@ def worker(
         os.environ["KMP_AFFINITY"] = f"explicit,proclist=[{proc_list}]"  # For Intel OpenMP
 
         # Set the affinity for this worker
-        os.sched_setaffinity(0, my_cores)
+        # OSX does not implement os.sched_setaffinity so type checking
+        # is ignored here in two ways:
+        # On a platform without sched_setaffinity, that attribute will not
+        # be defined, so ignore[attr-defined] will tell mypy to ignore this
+        # incorrect-for-OS X attribute access.
+        # On a platform with sched_setaffinity, that type: ignore message
+        # will be redundant, and ignore[unused-ignore] tells mypy to ignore
+        # that this ignore is unneeded.
+        os.sched_setaffinity(0, my_cores)  # type: ignore[attr-defined, unused-ignore]
         logger.info("Set worker CPU affinity to {}".format(my_cores))
 
     # If desired, pin to accelerator

{parsl-2024.5.27.dist-info → parsl-2024.6.10.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.5.27
+Version: 2024.6.10
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.05.27.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.06.10.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0