parsl 2024.3.18__py3-none-any.whl → 2025.1.13__py3-none-any.whl

parsl/usage_tracking/usage.py

@@ -1,22 +1,33 @@
-import uuid
-import time
-import os
 import json
 import logging
+import platform
 import socket
 import sys
-import platform
+import time
+import uuid
 
-from parsl.utils import setproctitle
-from parsl.multiprocessing import ForkProcess
 from parsl.dataflow.states import States
+from parsl.errors import ConfigurationError
+from parsl.multiprocessing import ForkProcess
+from parsl.usage_tracking.api import get_parsl_usage
+from parsl.usage_tracking.levels import DISABLED as USAGE_TRACKING_DISABLED
+from parsl.usage_tracking.levels import LEVEL_3 as USAGE_TRACKING_LEVEL_3
+from parsl.utils import setproctitle
 from parsl.version import VERSION as PARSL_VERSION
 
 logger = logging.getLogger(__name__)
 
 from typing import Callable
+
 from typing_extensions import ParamSpec
 
+# protocol version byte: when (for example) compression parameters are changed
+# that cannot be inferred from the compressed message itself, this version
+# ID needs to imply those parameters.
+
+# Earlier protocol versions: b'{' - the original pure-JSON protocol pre-March 2024
+PROTOCOL_VERSION = b'1'
+
 P = ParamSpec("P")
 
 
@@ -32,7 +43,7 @@ def async_process(fn: Callable[P, None]) -> Callable[P, None]:
 
 
 @async_process
-def udp_messenger(domain_name: str, UDP_PORT: int, sock_timeout: int, message: str) -> None:
+def udp_messenger(domain_name: str, UDP_PORT: int, sock_timeout: int, message: bytes) -> None:
     """Send UDP messages to usage tracker asynchronously
 
     This multiprocessing based messenger was written to overcome the limitations
@@ -46,16 +57,11 @@ def udp_messenger(domain_name: str, UDP_PORT: int, sock_timeout: int, message: s
     setproctitle("parsl: Usage tracking")
 
     try:
-        encoded_message = bytes(message, "utf-8")
-
         UDP_IP = socket.gethostbyname(domain_name)
 
-        if UDP_PORT is None:
-            raise Exception("UDP_PORT is None")
-
         sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)  # UDP
         sock.settimeout(sock_timeout)
-        sock.sendto(encoded_message, (UDP_IP, UDP_PORT))
+        sock.sendto(message, (UDP_IP, UDP_PORT))
         sock.close()
 
     except socket.timeout:
@@ -102,86 +108,119 @@ class UsageTracker:
         self.procs = []
         self.dfk = dfk
         self.config = self.dfk.config
-        self.uuid = str(uuid.uuid4())
+        self.correlator_uuid = str(uuid.uuid4())
         self.parsl_version = PARSL_VERSION
         self.python_version = "{}.{}.{}".format(sys.version_info.major,
                                                 sys.version_info.minor,
                                                 sys.version_info.micro)
-        self.tracking_enabled = self.check_tracking_enabled()
-        logger.debug("Tracking status: {}".format(self.tracking_enabled))
-
-    def check_tracking_enabled(self):
-        """Check if tracking is enabled.
-
-        Tracking will be enabled unless either of these is true:
-
-            1. dfk.config.usage_tracking is set to False
-            2. Environment variable PARSL_TRACKING is set to false (case insensitive)
-
+        self.tracking_level = self.check_tracking_level()
+        self.project_name = self.config.project_name
+        self.start_time = None
+        logger.debug("Tracking level: {}".format(self.tracking_level))
+
+    def check_tracking_level(self) -> int:
+        """Check if tracking is enabled and return level.
+
+        Checks usage_tracking in Config
+            - Possible values: [True, False, 0, 1, 2, 3]
+
+        True/False values are treated as Level 1/Level 0 respectively.
+
+        Returns: int
+            - 0 : Tracking is disabled
+            - 1 : Tracking is enabled with level 1
+                  Share info about Parsl version, Python version, platform
+            - 2 : Tracking is enabled with level 2
+                  Share info about config + level 1
+            - 3 : Tracking is enabled with level 3
+                  Share info about app count, app fails, execution time + level 2
         """
-        track = True
-
-        if not self.config.usage_tracking:
-            track = False
+        if not USAGE_TRACKING_DISABLED <= self.config.usage_tracking <= USAGE_TRACKING_LEVEL_3:
+            raise ConfigurationError(
+                f"Usage Tracking values must be 0, 1, 2, or 3 and not {self.config.usage_tracking}"
+            )
 
-        envvar = str(os.environ.get("PARSL_TRACKING", True)).lower()
-        if envvar == "false":
-            track = False
+        return self.config.usage_tracking
 
-        return track
-
-    def construct_start_message(self) -> str:
+    def construct_start_message(self) -> bytes:
         """Collect preliminary run info at the start of the DFK.
 
         Returns :
               - Message dict dumped as json string, ready for UDP
         """
-        message = {'uuid': self.uuid,
+        message = {'correlator': self.correlator_uuid,
                    'parsl_v': self.parsl_version,
                    'python_v': self.python_version,
-                   'os': platform.system(),
-                   'os_v': platform.release(),
-                   'start': time.time()}
+                   'platform.system': platform.system(),
+                   'tracking_level': int(self.tracking_level)}
+
+        if self.project_name:
+            message['project_name'] = self.project_name
+
+        if self.tracking_level >= 2:
+            message['components'] = get_parsl_usage(self.dfk._config)
+
+        if self.tracking_level == 3:
+            self.start_time = int(time.time())
+            message['start'] = self.start_time
 
-        return json.dumps(message)
+        logger.debug(f"Usage tracking start message: {message}")
 
-    def construct_end_message(self) -> str:
+        return self.encode_message(message)
+
+    def construct_end_message(self) -> bytes:
         """Collect the final run information at the time of DFK cleanup.
+        This is only called if tracking level is 3.
 
         Returns:
              - Message dict dumped as json string, ready for UDP
         """
-        app_count = self.dfk.task_count
+        end_time = int(time.time())
 
-        site_count = len(self.dfk.config.executors)
+        app_count = self.dfk.task_count
 
         app_fails = self.dfk.task_state_counts[States.failed] + self.dfk.task_state_counts[States.dep_fail]
 
-        message = {'uuid': self.uuid,
-                   'end': time.time(),
-                   't_apps': app_count,
-                   'sites': site_count,
-                   'failed': app_fails
-                   }
+        # the DFK is tangled into this code as a god-object, so it is
+        # handled separately from the usual traversal code, but presenting
+        # the same protocol-level report.
+        dfk_component = {'c': type(self.dfk).__module__ + "." + type(self.dfk).__name__,
+                         'app_count': app_count,
+                         'app_fails': app_fails}
+
+        message = {'correlator': self.correlator_uuid,
+                   'end': end_time,
+                   'execution_time': end_time - self.start_time,
+                   'components': [dfk_component] + get_parsl_usage(self.dfk._config)}
 
-        return json.dumps(message)
+        if self.project_name:
+            message['project_name'] = self.project_name
 
-    def send_UDP_message(self, message: str) -> None:
+        logger.debug(f"Usage tracking end message (unencoded): {message}")
+
+        return self.encode_message(message)
+
+    def encode_message(self, obj):
+        return PROTOCOL_VERSION + json.dumps(obj).encode()
+
+    def send_UDP_message(self, message: bytes) -> None:
         """Send UDP message."""
-        if self.tracking_enabled:
-            try:
-                proc = udp_messenger(self.domain_name, self.UDP_PORT, self.sock_timeout, message)
-                self.procs.append(proc)
-            except Exception as e:
-                logger.debug("Usage tracking failed: {}".format(e))
+        try:
+            proc = udp_messenger(self.domain_name, self.UDP_PORT, self.sock_timeout, message)
+            self.procs.append(proc)
+        except Exception as e:
+            logger.debug("Usage tracking failed: {}".format(e))
 
     def send_start_message(self) -> None:
-        message = self.construct_start_message()
-        self.send_UDP_message(message)
+        if self.tracking_level:
+            self.start_time = time.time()
+            message = self.construct_start_message()
+            self.send_UDP_message(message)
 
     def send_end_message(self) -> None:
-        message = self.construct_end_message()
-        self.send_UDP_message(message)
+        if self.tracking_level == 3:
+            message = self.construct_end_message()
+            self.send_UDP_message(message)
 
     def close(self, timeout: float = 10.0) -> None:
         """First give each process one timeout period to finish what it is
@@ -191,7 +230,10 @@ class UsageTracker:
         or won't respond to SIGTERM.
         """
         for proc in self.procs:
+            logger.debug("Joining usage tracking process %s", proc)
             proc.join(timeout=timeout)
             if proc.is_alive():
-                logger.info("Usage tracking process did not end itself; sending SIGKILL")
+                logger.warning("Usage tracking process did not end itself; sending SIGKILL")
                 proc.kill()
+
+            proc.close()

parsl/utils.py

@@ -1,21 +1,34 @@
 import inspect
 import logging
 import os
+import re
 import shlex
 import subprocess
 import threading
 import time
 from contextlib import contextmanager
 from types import TracebackType
-from typing import Any, Callable, List, Sequence, Tuple, Union, Generator, IO, AnyStr, Dict, Optional
+from typing import (
+    IO,
+    Any,
+    AnyStr,
+    Callable,
+    Dict,
+    Generator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 import typeguard
 from typing_extensions import Type
 
 import parsl
+from parsl.app.errors import BadStdStreamFile
 from parsl.version import VERSION
 
-
 try:
     import setproctitle as setproctitle_module
 except ImportError:
@@ -121,9 +134,17 @@ def get_std_fname_mode(
         if len(stdfspec) != 2:
             msg = (f"std descriptor {fdname} has incorrect tuple length "
                    f"{len(stdfspec)}")
-            raise pe.BadStdStreamFile(msg, TypeError('Bad Tuple Length'))
+            raise pe.BadStdStreamFile(msg)
         fname, mode = stdfspec
-    return str(fname), mode
+
+    path = os.fspath(fname)
+
+    if isinstance(path, str):
+        return path, mode
+    elif isinstance(path, bytes):
+        return path.decode(), mode
+    else:
+        raise BadStdStreamFile(f"fname has invalid type {type(path)}")
 
 
 @contextmanager
@@ -360,3 +381,115 @@ class AutoCancelTimer(threading.Timer):
         exc_tb: Optional[TracebackType]
     ) -> None:
         self.cancel()
+
+
+def sanitize_dns_label_rfc1123(raw_string: str) -> str:
+    """Convert input string to a valid RFC 1123 DNS label.
+
+    Parameters
+    ----------
+    raw_string : str
+        String to sanitize.
+
+    Returns
+    -------
+    str
+        Sanitized string.
+
+    Raises
+    ------
+    ValueError
+        If the string is empty after sanitization.
+    """
+    # Convert to lowercase and replace non-alphanumeric characters with hyphen
+    sanitized = re.sub(r'[^a-z0-9]', '-', raw_string.lower())
+
+    # Remove consecutive hyphens
+    sanitized = re.sub(r'-+', '-', sanitized)
+
+    # DNS label cannot exceed 63 characters
+    sanitized = sanitized[:63]
+
+    # Strip after trimming to avoid trailing hyphens
+    sanitized = sanitized.strip("-")
+
+    if not sanitized:
+        raise ValueError(f"Sanitized DNS label is empty for input '{raw_string}'")
+
+    return sanitized
+
+
+def sanitize_dns_subdomain_rfc1123(raw_string: str) -> str:
+    """Convert input string to a valid RFC 1123 DNS subdomain.
+
+    Parameters
+    ----------
+    raw_string : str
+        String to sanitize.
+
+    Returns
+    -------
+    str
+        Sanitized string.
+
+    Raises
+    ------
+    ValueError
+        If the string is empty after sanitization.
+    """
+    segments = raw_string.split('.')
+
+    sanitized_segments = []
+    for segment in segments:
+        if not segment:
+            continue
+        sanitized_segment = sanitize_dns_label_rfc1123(segment)
+        sanitized_segments.append(sanitized_segment)
+
+    sanitized = '.'.join(sanitized_segments)
+
+    # DNS subdomain cannot exceed 253 characters
+    sanitized = sanitized[:253]
+
+    # Strip after trimming to avoid trailing dots or hyphens
+    sanitized = sanitized.strip(".-")
+
+    if not sanitized:
+        raise ValueError(f"Sanitized DNS subdomain is empty for input '{raw_string}'")
+
+    return sanitized
+
+
+def execute_wait(cmd: str, walltime: Optional[int] = None) -> Tuple[int, str, str]:
+    ''' Synchronously execute a commandline string on the shell.
+
+    Args:
+        - cmd (string) : Commandline string to execute
+        - walltime (int) : walltime in seconds
+
+    Returns:
+        - retcode : Return code from the execution
+        - stdout  : stdout string
+        - stderr  : stderr string
+    '''
+    try:
+        logger.debug("Creating process with command '%s'", cmd)
+        proc = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=True,
+            preexec_fn=os.setpgrp
+        )
+        logger.debug("Created process with pid %s. Performing communicate", proc.pid)
+        (stdout, stderr) = proc.communicate(timeout=walltime)
+        retcode = proc.returncode
+        logger.debug("Process %s returned %s", proc.pid, proc.returncode)
+
+    except Exception:
+        logger.exception(f"Execution of command failed:\n{cmd}")
+        raise
+    else:
+        logger.debug("Execution of command in process %s completed normally", proc.pid)
+
+    return (retcode, stdout.decode("utf-8"), stderr.decode("utf-8"))

parsl/version.py

@@ -3,4 +3,4 @@
 Year.Month.Day[alpha/beta/..]
 Alphas will be numbered like this -> 2024.12.10a0
 """
-VERSION = '2024.03.18'
+VERSION = '2025.01.13'

{parsl-2024.3.18.data → parsl-2025.1.13.data}/scripts/exec_parsl_function.py

@@ -1,10 +1,11 @@
+import pickle
+import sys
+import traceback
+
 from parsl.app.errors import RemoteExceptionWrapper
 from parsl.data_provider.files import File
-from parsl.utils import get_std_fname_mode
-import traceback
-import sys
-import pickle
 from parsl.serialize import serialize
+from parsl.utils import get_std_fname_mode
 
 # This scripts executes a parsl function which is pickled in a file:
 #
@@ -93,7 +94,7 @@ def unpack_source_code_function(function_info, user_namespace):
 
 def unpack_byte_code_function(function_info, user_namespace):
     from parsl.serialize import unpack_apply_message
-    func, args, kwargs = unpack_apply_message(function_info["byte code"], user_namespace, copy=False)
+    func, args, kwargs = unpack_apply_message(function_info["byte code"])
     return (func, 'parsl_function_name', args, kwargs)