parsl 2024.2.26__py3-none-any.whl → 2024.3.11__py3-none-any.whl

parsl-2024.3.11.data/scripts/parsl_coprocess.py

@@ -0,0 +1,166 @@
+#!python
+
+import socket
+import json
+import os
+import sys
+
+# If enabled, coprocess will print to stdout
+debug_mode = False
+
+# Send a message on a binary I/O stream by sending the message length and then the (string) message.
+def send_message(stream, data):
+    size = len(data)
+    size_msg = "{}\n".format(size)
+    stream.write(size_msg)
+    stream.write(data)
+
+# Receive a standard message from a binary I/O stream by reading length and then returning the (string) message
+def recv_message(stream):
+    line = stream.readline()
+    length = int(line)
+    return stream.read(length)
+
+# Decorator for remotely execution functions to package things as json.
+def remote_execute(func):
+    def remote_wrapper(event):
+        kwargs = event["fn_kwargs"]
+        args = event["fn_args"]
+        try:
+            response = {
+                "Result": func(*args, **kwargs),
+                "StatusCode": 200
+            }
+        except Exception as e:
+            response = {
+                "Result": str(e),
+                "StatusCode": 500
+            }
+        return response
+    return remote_wrapper
+
+# Main loop of coprocess for executing network functions.
+def main():
+    # Listen on an arbitrary port to be reported to the worker.
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.bind(('localhost', 0))
+    except Exception as e:
+        s.close()
+        print(e, file=sys.stderr)
+        sys.exit(1)
+
+    # Inform the worker of name and port for later connection.
+    config = {
+        "name": name(),  # noqa: F821
+        "port": s.getsockname()[1],
+    }
+    send_message(sys.stdout, json.dumps(config))
+    sys.stdout.flush()
+
+    # Remember original working directory b/c we change for each invocation.
+    abs_working_dir = os.getcwd()
+
+    # Create pipe for communication with child process
+    rpipe, wpipe = os.pipe()
+    rpipestream = os.fdopen(rpipe, "r")
+
+    while True:
+        s.listen()
+        conn, addr = s.accept()
+        connstream = conn.makefile("rw", encoding="utf-8")
+
+        if debug_mode:
+            print('Network function: connection from {}'.format(addr), file=sys.stderr)
+
+        while True:
+            # Read the invocation header from the worker
+            line = connstream.readline()
+
+            # If end of file, then break out and accept again
+            if not line:
+                break
+
+            # Parse the invocation header.
+            input_spec = line.split()
+            function_name = input_spec[0]
+            task_id = int(input_spec[1])
+            event_size = int(input_spec[2])
+
+            # then read the contents of the event itself
+            event_str = connstream.read(event_size)
+            event = json.loads(event_str)
+            exec_method = event.get("remote_task_exec_method", None)
+
+            try:
+                # First move to target directory (is undone in finally block)
+                os.chdir(os.path.join(abs_working_dir, f't.{task_id}'))
+
+                # Then invoke function by desired method, resulting in
+                # response containing the text representation of the result.
+
+                if exec_method == "direct":
+                    response = json.dumps(globals()[function_name](event))
+                else:
+                    p = os.fork()
+                    if p == 0:
+                        response = globals()[function_name](event)
+                        wpipestream = os.fdopen(wpipe, "w")
+                        send_message(wpipestream, json.dumps(response))
+                        wpipestream.flush()
+                        os._exit(0)
+                    elif p < 0:
+                        if debug_mode:
+                            print(f'Network function: unable to fork to execute {function_name}', file=sys.stderr)
+                        response = {
+                            "Result": "unable to fork",
+                            "StatusCode": 500
+                        }
+                        response = json.dumps(response)
+                    else:
+                        # Get response string from child process.
+                        response = recv_message(rpipestream)
+                        # Wait for child process to complete
+                        os.waitpid(p, 0)
+
+                # At this point, response is set to a value one way or the other
+
+            except Exception as e:
+                if debug_mode:
+                    print("Network function encountered exception ", str(e), file=sys.stderr)
+                response = {
+                    'Result': f'network function encountered exception {e}',
+                    'Status Code': 500
+                }
+                response = json.dumps(response)
+            finally:
+                # Restore the working directory, no matter how the function ended.
+                os.chdir(abs_working_dir)
+
+            # Send response string back to parent worker process.
+            send_message(connstream, response)
+            connstream.flush()
+
+    return 0
+def name():
+    return 'parsl_coprocess'
+@remote_execute
+def run_parsl_task(a, b, c):
+    import parsl.executors.workqueue.exec_parsl_function as epf
+    try:
+        (map_file, function_file, result_file) = (a, b, c)
+        try:
+            (namespace, function_code, result_name) = epf.load_function(map_file, function_file)
+        except Exception:
+            raise
+        try:
+            result = epf.execute_function(namespace, function_code, result_name)
+        except Exception:
+            raise
+    except Exception:
+        result = RemoteExceptionWrapper(*sys.exc_info())
+    epf.dump_result_to_file(result_file, result)
+    return None
+if __name__ == "__main__":
+    main()
+

{parsl-2024.2.26.data → parsl-2024.3.11.data}/scripts/process_worker_pool.py

@@ -62,7 +62,7 @@ class Manager:
                  result_port,
                  cores_per_worker,
                  mem_per_worker,
-                 max_workers,
+                 max_workers_per_node,
                  prefetch_capacity,
                  uid,
                  block_id,
@@ -100,8 +100,8 @@ class Manager:
              the there's sufficient memory for each worker. If set to None, memory on node is not
              considered in the determination of workers to be launched on node by the manager.
 
-        max_workers : int
-             caps the maximum number of workers that can be launched.
+        max_workers_per_node : int | float
+             Caps the maximum number of workers that can be launched.
 
         prefetch_capacity : int
              Number of tasks that could be prefetched over available worker capacity.
@@ -140,7 +140,9 @@ class Manager:
             Path to the certificate directory.
         """
 
-        logger.info("Manager started")
+        logger.info("Manager initializing")
+
+        self._start_time = time.time()
 
         try:
             ix_address = probe_addresses(addresses.split(','), task_port, timeout=address_probe_timeout)
@@ -188,15 +190,15 @@ class Manager:
         else:
             available_mem_on_node = round(psutil.virtual_memory().available / (2**30), 1)
 
-        self.max_workers = max_workers
+        self.max_workers_per_node = max_workers_per_node
         self.prefetch_capacity = prefetch_capacity
 
-        mem_slots = max_workers
+        mem_slots = max_workers_per_node
         # Avoid a divide by 0 error.
         if mem_per_worker and mem_per_worker > 0:
             mem_slots = math.floor(available_mem_on_node / mem_per_worker)
 
-        self.worker_count: int = min(max_workers,
+        self.worker_count: int = min(max_workers_per_node,
                                      mem_slots,
                                      math.floor(cores_on_node / cores_per_worker))
 
@@ -237,7 +239,8 @@ class Manager:
     def create_reg_message(self):
         """ Creates a registration message to identify the worker to the interchange
         """
-        msg = {'parsl_v': PARSL_VERSION,
+        msg = {'type': 'registration',
+               'parsl_v': PARSL_VERSION,
                'python_v': "{}.{}.{}".format(sys.version_info.major,
                                              sys.version_info.minor,
                                              sys.version_info.micro),
@@ -258,8 +261,9 @@ class Manager:
     def heartbeat_to_incoming(self):
         """ Send heartbeat to the incoming task queue
         """
-        heartbeat = (HEARTBEAT_CODE).to_bytes(4, "little")
-        self.task_incoming.send(heartbeat)
+        msg = {'type': 'heartbeat'}
+        b_msg = json.dumps(msg).encode('utf-8')
+        self.task_incoming.send(b_msg)
         logger.debug("Sent heartbeat")
 
     @wrap_with_logs
@@ -284,9 +288,17 @@ class Manager:
         last_interchange_contact = time.time()
         task_recv_counter = 0
 
-        poll_timer = self.poll_period
-
         while not kill_event.is_set():
+
+            # This loop will sit inside poller.poll until either a message
+            # arrives or one of these event times is reached. This code
+            # assumes that the event times won't change except on iteration
+            # of this loop - so will break if a different thread does
+            # anything to bring one of the event times earlier - and that the
+            # time here are correctly copy-pasted from the relevant if
+            # statements.
+            next_interesting_event_time = min(last_beat + self.heartbeat_period,
+                                              last_interchange_contact + self.heartbeat_threshold)
             try:
                 pending_task_count = self.pending_task_queue.qsize()
             except NotImplementedError:
@@ -296,14 +308,14 @@ class Manager:
             logger.debug("ready workers: {}, pending tasks: {}".format(self.ready_worker_count.value,
                                                                        pending_task_count))
 
-            if time.time() > last_beat + self.heartbeat_period:
+            if time.time() >= last_beat + self.heartbeat_period:
                 self.heartbeat_to_incoming()
                 last_beat = time.time()
 
-            socks = dict(poller.poll(timeout=poll_timer))
+            poll_duration_s = max(0, next_interesting_event_time - time.time())
+            socks = dict(poller.poll(timeout=poll_duration_s * 1000))
 
             if self.task_incoming in socks and socks[self.task_incoming] == zmq.POLLIN:
-                poll_timer = 0
                 _, pkl_msg = self.task_incoming.recv_multipart()
                 tasks = pickle.loads(pkl_msg)
                 last_interchange_contact = time.time()
@@ -320,14 +332,9 @@ class Manager:
 
             else:
                 logger.debug("No incoming tasks")
-                # Limit poll duration to heartbeat_period
-                # heartbeat_period is in s vs poll_timer in ms
-                if not poll_timer:
-                    poll_timer = self.poll_period
-                poll_timer = min(self.heartbeat_period * 1000, poll_timer * 2)
 
                 # Only check if no messages were received.
-                if time.time() > last_interchange_contact + self.heartbeat_threshold:
+                if time.time() >= last_interchange_contact + self.heartbeat_threshold:
                     logger.critical("Missing contact with interchange beyond heartbeat_threshold")
                     kill_event.set()
                     logger.critical("Exiting")
@@ -364,7 +371,8 @@ class Manager:
                 logger.exception("Got an exception: {}".format(e))
 
             if time.time() > last_result_beat + self.heartbeat_period:
-                logger.info(f"Sending heartbeat via results connection: last_result_beat={last_result_beat} heartbeat_period={self.heartbeat_period} seconds")
+                heartbeat_message = f"last_result_beat={last_result_beat} heartbeat_period={self.heartbeat_period} seconds"
+                logger.info(f"Sending heartbeat via results connection: {heartbeat_message}")
                 last_result_beat = time.time()
                 items.append(pickle.dumps({'type': 'heartbeat'}))
 
@@ -405,7 +413,9 @@ class Manager:
                             raise WorkerLost(worker_id, platform.node())
                         except Exception:
                             logger.info("Putting exception for executor task {} in the pending result queue".format(task['task_id']))
-                            result_package = {'type': 'result', 'task_id': task['task_id'], 'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))}
+                            result_package = {'type': 'result',
+                                              'task_id': task['task_id'],
+                                              'exception': serialize(RemoteExceptionWrapper(*sys.exc_info()))}
                             pkl_package = pickle.dumps(result_package)
                             self.pending_result_queue.put(pkl_package)
                     except KeyError:
@@ -452,7 +462,6 @@ class Manager:
 
         TODO: Move task receiving to a thread
         """
-        start = time.time()
         self._kill_event = threading.Event()
         self._tasks_in_progress = self._mp_manager.dict()
 
@@ -502,7 +511,7 @@ class Manager:
         self.task_incoming.close()
         self.result_outgoing.close()
         self.zmq_context.term()
-        delta = time.time() - start
+        delta = time.time() - self._start_time
         logger.info("process_worker_pool ran for {} seconds".format(delta))
         return
 
@@ -787,7 +796,7 @@ if __name__ == "__main__":
                         help="GB of memory assigned to each worker process. Default=0, no assignment")
     parser.add_argument("-t", "--task_port", required=True,
                         help="REQUIRED: Task port for receiving tasks from the interchange")
-    parser.add_argument("--max_workers", default=float('inf'),
+    parser.add_argument("--max_workers_per_node", default=float('inf'),
                         help="Caps the maximum workers that can be launched, default:infinity")
     parser.add_argument("-p", "--prefetch_capacity", default=0,
                         help="Number of tasks that can be prefetched to the manager. Default is 0.")
@@ -841,7 +850,7 @@ if __name__ == "__main__":
         logger.info("task_port: {}".format(args.task_port))
         logger.info("result_port: {}".format(args.result_port))
         logger.info("addresses: {}".format(args.addresses))
-        logger.info("max_workers: {}".format(args.max_workers))
+        logger.info("max_workers_per_node: {}".format(args.max_workers_per_node))
         logger.info("poll_period: {}".format(args.poll))
         logger.info("address_probe_timeout: {}".format(args.address_probe_timeout))
         logger.info("Prefetch capacity: {}".format(args.prefetch_capacity))
@@ -860,7 +869,10 @@ if __name__ == "__main__":
                           block_id=args.block_id,
                           cores_per_worker=float(args.cores_per_worker),
                           mem_per_worker=None if args.mem_per_worker == 'None' else float(args.mem_per_worker),
-                          max_workers=args.max_workers if args.max_workers == float('inf') else int(args.max_workers),
+                          max_workers_per_node=(
+                              args.max_workers_per_node if args.max_workers_per_node == float('inf')
+                              else int(args.max_workers_per_node)
+                          ),
                           prefetch_capacity=int(args.prefetch_capacity),
                           heartbeat_threshold=int(args.hb_threshold),
                           heartbeat_period=int(args.hb_period),

{parsl-2024.2.26.dist-info → parsl-2024.3.11.dist-info}/METADATA

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: parsl
-Version: 2024.2.26
+Version: 2024.3.11
 Summary: Simple data dependent workflows in Python
 Home-page: https://github.com/Parsl/parsl
-Download-URL: https://github.com/Parsl/parsl/archive/2024.02.26.tar.gz
+Download-URL: https://github.com/Parsl/parsl/archive/2024.03.11.tar.gz
 Author: The Parsl Team
 Author-email: parsl@googlegroups.com
 License: Apache 2.0