robotframework-pabot 5.0.0__py3-none-any.whl → 5.2.0b1__py3-none-any.whl

pabot/ProcessManager.py

@@ -0,0 +1,376 @@
+import os
+import sys
+import time
+import signal
+import threading
+import subprocess
+import datetime
+import queue
+import locale
+
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+from .writer import get_writer, Color
+
+
+def split_on_first(lst, value):
+    for i, x in enumerate(lst):
+        if x == value:
+            return lst[:i], lst[i+1:]
+    return lst, []
+
+
+class ProcessManager:
+    def __init__(self):
+        self.processes = []
+        self.lock = threading.Lock()
+        self.writer = get_writer()
+
+        # Install SIGINT only in main thread
+        try:
+            if threading.current_thread() is threading.main_thread():
+                signal.signal(signal.SIGINT, self._handle_sigint)
+            else:
+                self.writer.write(
+                    "[ProcessManager] (test mode) signal handlers disabled (not in main thread)"
+                )
+        except Exception as e:
+            self.writer.write(f"[WARN] Could not register signal handler: {e}")
+
+    # -------------------------------
+    # SIGNAL HANDLING
+    # -------------------------------
+
+    def _handle_sigint(self, signum, frame):
+        self.writer.write("[ProcessManager] Ctrl+C detected — terminating all subprocesses", color=Color.RED)
+        self.terminate_all()
+        sys.exit(130)
+
+    # -------------------------------
+    # OUTPUT STREAM READERS
+    # -------------------------------
+
+    def _enqueue_output(self, pipe, q):
+        """
+        Reads lines from `pipe` and puts them into queue `q`.
+        When pipe is exhausted, pushes `None` sentinel.
+        """
+        try:
+            with pipe:
+                for line in iter(pipe.readline, b""):
+                    q.put(line)
+        finally:
+            q.put(None)  # sentinel → "this stream is finished"
+
+    def _safe_write_to_stream(self, stream, text):
+        """
+        Writes text safely to an output stream.
+        If encoding errors occur, fall back to bytes/replace.
+        """
+        try:
+            stream.write(text)
+            try:
+                stream.flush()
+            except Exception:
+                pass
+            return
+        except UnicodeEncodeError:
+            pass
+        except Exception:
+            pass
+
+        enc = getattr(stream, "encoding", None) or locale.getpreferredencoding(False) or "utf-8"
+
+        try:
+            b = text.encode(enc, errors="replace")
+            if hasattr(stream, "buffer"):
+                try:
+                    stream.buffer.write(b)
+                    stream.buffer.write(b"\n")
+                    stream.buffer.flush()
+                    return
+                except Exception:
+                    pass
+
+            safe = b.decode(enc, errors="replace")
+            stream.write(safe + "\n")
+            stream.flush()
+        except Exception:
+            try:
+                print(text)
+            except Exception:
+                pass
+
+    # -------------------------------
+    # STREAM OUTPUT MERGER
+    # -------------------------------
+
+    def _stream_output(self, process, stdout=None, stderr=None,
+                       item_name="process", log_file=None):
+
+        q_out = queue.Queue()
+        q_err = queue.Queue()
+
+        t_out = None
+        t_err = None
+
+        if process.stdout:
+            t_out = threading.Thread(target=self._enqueue_output, args=(process.stdout, q_out))
+            t_out.daemon = True
+            t_out.start()
+
+        if process.stderr:
+            t_err = threading.Thread(target=self._enqueue_output, args=(process.stderr, q_err))
+            t_err.daemon = True
+            t_err.start()
+
+        stdout_done = False
+        stderr_done = False
+
+        log_handle = None
+        if log_file:
+            os.makedirs(os.path.dirname(log_file), exist_ok=True)
+            log_handle = open(log_file, "a", encoding="utf-8")
+
+        try:
+            while True:
+                now = datetime.datetime.now()
+
+                # STDOUT
+                if not stdout_done:
+                    try:
+                        line = q_out.get(timeout=0.05)
+                        if line is None:
+                            stdout_done = True
+                        else:
+                            msg = line.decode(errors="replace").rstrip()
+                            self._safe_write_to_stream(stdout or sys.stdout, msg + "\n")
+                            if log_handle:
+                                log_handle.write(f"{now} {msg}\n")
+                    except queue.Empty:
+                        pass
+
+                # STDERR
+                if not stderr_done:
+                    try:
+                        line = q_err.get_nowait()
+                        if line is None:
+                            stderr_done = True
+                        else:
+                            msg = line.decode(errors="replace").rstrip()
+                            self._safe_write_to_stream(stderr or sys.stderr, msg + "\n")
+                            if log_handle:
+                                log_handle.write(f"{now} {msg}\n")
+                    except queue.Empty:
+                        pass
+
+                # Terminate when both streams finished
+                if stdout_done and stderr_done:
+                    break
+
+        finally:
+            if t_out:
+                t_out.join()
+            if t_err:
+                t_err.join()
+            if log_handle:
+                log_handle.close()
+
+    # -------------------------------
+    # PROCESS CREATION
+    # -------------------------------
+
+    def _start_process(self, cmd, env=None):
+        if sys.platform == "win32":
+            return subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=env,
+                shell=False,
+                creationflags=subprocess.CREATE_NEW_PROCESS_GROUP,
+            )
+        else:
+            return subprocess.Popen(
+                cmd,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env=env,
+                shell=False,
+                preexec_fn=os.setsid,
+            )
+
+    # -------------------------------
+    # PROCESS TREE TERMINATION
+    # -------------------------------
+
+    def _terminate_tree(self, process):
+        if process.poll() is not None:
+            return
+
+        self.writer.write(
+            f"[ProcessManager] Terminating process tree PID={process.pid}",
+            color=Color.YELLOW
+        )
+
+        # PRIMARY: psutil (best reliability)
+        if psutil:
+            try:
+                parent = psutil.Process(process.pid)
+                children = parent.children(recursive=True)
+                for c in children:
+                    try:
+                        c.terminate()
+                    except Exception:
+                        pass
+                psutil.wait_procs(children, timeout=5)
+
+                for c in children:
+                    if c.is_running():
+                        try:
+                            c.kill()
+                        except Exception:
+                            pass
+
+                try:
+                    parent.terminate()
+                except Exception:
+                    pass
+
+                try:
+                    parent.wait(timeout=5)
+                except psutil.TimeoutExpired:
+                    try:
+                        parent.kill()
+                    except Exception:
+                        pass
+
+                return
+            except Exception:
+                pass
+
+        # FALLBACK — Windows
+        if sys.platform == "win32":
+            subprocess.run(
+                ["taskkill", "/PID", str(process.pid), "/T", "/F"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+            return
+
+        # FALLBACK — Linux / macOS
+        try:
+            os.killpg(os.getpgid(process.pid), signal.SIGTERM)
+            time.sleep(2)
+            if process.poll() is None:
+                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+        except Exception:
+            if process.poll() is None:
+                try:
+                    process.kill()
+                except Exception:
+                    pass
+
+        try:
+            process.wait(timeout=5)
+        except Exception:
+            pass
+
+    # -------------------------------
+    # PUBLIC API
+    # -------------------------------
+
+    def terminate_all(self):
+        with self.lock:
+            for p in list(self.processes):
+                self._terminate_tree(p)
+            self.processes.clear()
+
+    def run(self, cmd, *, env=None, stdout=None, stderr=None,
+            timeout=None, verbose=False, item_name="process",
+            log_file=None, pool_id=0, item_index=0):
+
+        start = time.time()
+        process = self._start_process(cmd, env)
+
+        with self.lock:
+            self.processes.append(process)
+
+        ts = datetime.datetime.now()
+
+        if verbose:
+            self.writer.write(
+                f"{ts} [PID:{process.pid}] [{pool_id}] [ID:{item_index}] "
+                f"EXECUTING PARALLEL {item_name}:\n{' '.join(cmd)}"
+            )
+        else:
+            self.writer.write(
+                f"{ts} [PID:{process.pid}] [{pool_id}] [ID:{item_index}] EXECUTING {item_name}"
+            )
+
+        # Start logging thread
+        log_thread = threading.Thread(
+            target=self._stream_output,
+            args=(process, stdout, stderr, item_name, log_file),
+        )
+        log_thread.daemon = True
+        log_thread.start()
+
+        rc = None
+        ping_interval = 50   # 5s
+        next_ping = ping_interval
+        counter = 0
+
+        while rc is None:
+            rc = process.poll()
+
+            # TIMEOUT CHECK
+            if timeout and (time.time() - start > timeout):
+                ts = datetime.datetime.now()
+                self.writer.write(
+                    f"{ts} [PID:{process.pid}] [{pool_id}] [ID:{item_index}] "
+                    f"Process {item_name} killed due to exceeding the maximum timeout of {timeout} seconds"
+                )
+                self._terminate_tree(process)
+                rc = -1
+
+                # Dryrun process to mark all tests as failed due to timeout
+                this_dir = os.path.dirname(os.path.abspath(__file__))
+                listener_path = os.path.join(this_dir, "timeout_listener.py")
+                dry_run_env = env.copy() if env else os.environ.copy()
+                before, after = split_on_first(cmd, "-A")
+                dryrun_cmd = before + ["--dryrun", '--listener', listener_path, '-A'] + after
+
+                self.writer.write(
+                    f"{ts} [PID:{process.pid}] [{pool_id}] [ID:{item_index}] "
+                    f"Starting dry run to mark tests as failed due to timeout: {' '.join(dryrun_cmd)}"
+                )
+                subprocess.run(dryrun_cmd, env=dry_run_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE)                
+
+                break
+
+            # Progress ping
+            if counter == next_ping:
+                ts = datetime.datetime.now()
+                self.writer.write(
+                    f"{ts} [PID:{process.pid}] [{pool_id}] [ID:{item_index}] still running "
+                    f"{item_name} after {(counter * 0.1):.1f}s"
+                )
+                ping_interval += 50
+                next_ping += ping_interval
+
+            time.sleep(0.1)
+            counter += 1
+
+        log_thread.join()
+
+        elapsed = round(time.time() - start, 1)
+
+        with self.lock:
+            if process in self.processes:
+                self.processes.remove(process)
+
+        return process, (rc, elapsed)

pabot/__init__.py

@@ -7,4 +7,4 @@ try:
 except ImportError:
     pass
 
-__version__ = "5.0.0"
+__version__ = "5.2.0b1"

pabot/arguments.py

@@ -142,7 +142,20 @@ def _parse_shard(arg):
     return int(parts[0]), int(parts[1])
 
 
+def _parse_artifacts(arg):
+    # type: (str) -> Tuple[List[str], bool]
+    artifacts = arg.split(',')
+    if artifacts[-1] == 'notimestamps':
+        return (artifacts[:-1], False)
+    return (artifacts, True)
+
+
 def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str, object]]
+    """
+    Parse pabot-specific command line arguments.
+    Supports new --ordering syntax:
+        --ordering <file> [static|dynamic] [skip|run_all]
+    """
     pabot_args = {
         "command": ["pybot" if ROBOT_VERSION < "3.1" else "robot"],
         "verbose": False,
@@ -155,13 +168,15 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
         "processes": _processes_count(),
         "processtimeout": None,
         "artifacts": ["png"],
+        "artifactstimestamps": True,
         "artifactsinsubfolders": False,
         "shardindex": 0,
         "shardcount": 1,
         "chunk": False,
         "no-rebot": False,
     }
-    # Explicitly define argument types for validation
+
+    # Arguments that are flags (boolean)
     flag_args = {
         "verbose",
         "help",
@@ -169,8 +184,10 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
         "pabotlib",
         "artifactsinsubfolders",
         "chunk",
-        "no-rebot"
+        "no-rebot",
     }
+
+    # Arguments that expect values
     value_args = {
         "hive": str,
         "processes": lambda x: int(x) if x != "all" else None,
@@ -179,9 +196,9 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
         "pabotlibport": int,
         "pabotprerunmodifier": str,
         "processtimeout": int,
-        "ordering": str,
+        "ordering": str,  # special handling below
         "suitesfrom": str,
-        "artifacts": lambda x: x.split(","),
+        "artifacts": _parse_artifacts,
         "shard": _parse_shard,
     }
 
@@ -189,7 +206,7 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
     remaining_args = []
     i = 0
 
-    # Track conflicting options during parsing
+    # Track conflicting pabotlib options
     saw_pabotlib_flag = False
     saw_no_pabotlib = False
 
@@ -200,19 +217,20 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
             i += 1
             continue
 
-        arg_name = arg[2:]  # Strip '--'
+        arg_name = arg[2:]  # remove leading '--'
 
+        # Handle mutually exclusive pabotlib flags
         if arg_name == "no-pabotlib":
             saw_no_pabotlib = True
-            pabot_args["pabotlib"] = False  # Just set the main flag
-            args = args[1:]
+            pabot_args["pabotlib"] = False
+            i += 1
             continue
         if arg_name == "pabotlib":
             saw_pabotlib_flag = True
-            args = args[1:]
+            i += 1
             continue
 
-        # Special case for command
+        # Special handling for --command ... --end-command
         if arg_name == "command":
             try:
                 end_index = args.index("--end-command", i)
@@ -222,7 +240,7 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
             except ValueError:
                 raise DataError("--command requires matching --end-command")
 
-        # Handle flag arguments
+        # Handle boolean flags
         if arg_name in flag_args:
             pabot_args[arg_name] = True
             i += 1
@@ -233,20 +251,57 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
             if i + 1 >= len(args):
                 raise DataError(f"--{arg_name} requires a value")
             try:
-                value = value_args[arg_name](args[i + 1])
-                if arg_name == "shard":
-                    pabot_args["shardindex"], pabot_args["shardcount"] = value
-                elif arg_name == "pabotlibhost":
-                    pabot_args["pabotlib"] = False
-                    pabot_args[arg_name] = value
+                # Special parsing for --ordering <file> [mode] [failure_policy]
+                if arg_name == "ordering":
+                    if i + 1 >= len(args):
+                        raise DataError("--ordering requires at least a file path")
+
+                    ordering_file = args[i + 1]
+                    mode = "static"  # default
+                    failure_policy = "run_all"  # default
+
+                    # optional mode
+                    if i + 2 < len(args) and args[i + 2] in ("static", "dynamic"):
+                        mode = args[i + 2]
+                        i_mode_offset = 1
+                    else:
+                        i_mode_offset = 0
+
+                    # optional failure policy, only for dynamic mode
+                    if mode == "dynamic" and i + 2 + i_mode_offset < len(args) and args[i + 2 + i_mode_offset] in ("skip", "run_all"):
+                        failure_policy = args[i + 2 + i_mode_offset]
+                        i_failure_offset = 1
+                    else:
+                        i_failure_offset = 0
+
+                    # store
+                    pabot_args["ordering"] = {
+                        "file": ordering_file,
+                        "mode": mode,
+                        "failure_policy": failure_policy,
+                    }
+
+                    # move index past ordering args only
+                    i += 2 + i_mode_offset + i_failure_offset
+                    continue
                 else:
-                    pabot_args[arg_name] = value
-                i += 2
-                continue
-            except (ValueError, TypeError) as e:
+                    value = value_args[arg_name](args[i + 1])
+                    if arg_name == "shard":
+                        pabot_args["shardindex"], pabot_args["shardcount"] = value
+                    elif arg_name == "pabotlibhost":
+                        pabot_args["pabotlib"] = False
+                        pabot_args[arg_name] = value
+                    elif arg_name == "artifacts":
+                        pabot_args["artifacts"] = value[0]
+                        pabot_args["artifactstimestamps"] = value[1]
+                    else:
+                        pabot_args[arg_name] = value
+                    i += 2
+                    continue
+            except (ValueError, TypeError):
                 raise DataError(f"Invalid value for --{arg_name}: {args[i + 1]}")
-
-        # Handle argument files
+        
+        # Handle argumentfiles like --argumentfile1
         match = ARGSMATCHER.match(arg)
         if match:
             if i + 1 >= len(args):
@@ -255,10 +310,11 @@ def _parse_pabot_args(args):  # type: (List[str]) -> Tuple[List[str], Dict[str,
             i += 2
             continue
 
-        # If we get here, it's a non-pabot argument
+        # Any other non-pabot argument
         remaining_args.append(arg)
         i += 1
 
+    # Check for conflicting pabotlib flags
     if saw_pabotlib_flag and saw_no_pabotlib:
         raise DataError("Cannot use both --pabotlib and --no-pabotlib options together")