atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/basics/multiprocesses.py

@@ -1,13 +1,84 @@
 import multiprocessing
 import multiprocessing.managers
+import os
 import queue
+import threading
 import concurrent.futures
 from concurrent.futures import ProcessPoolExecutor, as_completed
+from collections import deque
+from typing import Callable
+import time
 
 from ..import system_resources
 
 
-def process_wrap_queue(function_reference, *args, **kwargs):
+def kill_processes(
+        processes: list
+):
+    """Terminate all children with SIGTERM (or SIGKILL if you like)."""
+    # Ask OS to terminate all processes in the list.
+    for p in processes:
+        if p.is_alive():
+            p.terminate()
+    time.sleep(1)  # give processes a chance to exit cleanly
+    # Force kill all processes in the list.
+    for p in processes:
+        if p.is_alive():
+            p.kill()
+    for p in processes:          # wait for everything to disappear
+        p.join()
+
+
+def is_process_crashed(
+        processes: list[multiprocessing.Process]
+) -> tuple[int, str] | tuple[None, None]:
+    """
+    Check if any of the processes in the list is not alive.
+    :param processes: list, list of multiprocessing.Process objects.
+    :return: tuple(int, string) or None.
+        tuple(0 if any finished cleanly, process name).
+        tuple(1 (or exit code integer) if any process crashed, process_name).
+        None if all processes are still alive.
+
+    ==============================================
+
+    Usage example:
+    processes = [multiprocessing.Process(target=some_function) for _ in range(5)]
+
+    for p in processes:
+        p.start()
+
+    # Check if any process has crashed
+    try:
+        while True:
+            # Poll every second; you can use a shorter sleep if you prefer.
+            result, process_name = is_process_crashed(processes)
+            # If result is None, all processes are still alive.
+            if result is not None:
+                # If result is 0 or 1, we can exit the loop.
+                print(f"Process [{process_name}] finished with exit code {result}.")
+                break
+            time.sleep(1)
+    except KeyboardInterrupt:
+        print("Ctrl-C caught – terminating children…")
+        kill_all(processes)
+        sys.exit(0)
+    """
+
+    for p in processes:
+        if p.exitcode is not None:  # the process is *dead*
+            kill_processes(processes)  # stop the rest
+            if p.exitcode == 0:
+                # print(f"{p.name} exited cleanly; shutting down.")
+                return 0, p.name
+            else:
+                # print(f"{p.name} crashed (exitcode {p.exitcode}). Shutting everything down.")
+                return p.exitcode, p.name
+
+    return None, None  # all processes are still alive
+
+
+def process_wrap_queue(function_reference: Callable, *args, **kwargs):
     """
     The function receives function reference and arguments, and executes the function in a thread.
     "_queue" means that a queue.put() is used to store the result of the function and queue.get() to output it.
@@ -38,7 +109,7 @@ def process_wrap_queue(function_reference, *args, **kwargs):
 class MultiProcessorRecursive:
     def __init__(
             self,
-            process_function,
+            process_function: Callable,
             input_list: list,
             max_workers: int = None,
             cpu_percent_max: int = 80,
@@ -52,7 +123,7 @@ class MultiProcessorRecursive:
         :param process_function: function, function to execute on the input list.
         :param input_list: list, list of inputs to process.
         :param max_workers: integer, number of workers to execute functions in parallel. Default is None, which
-            is the number of CPUs.
+            is the number of CPUs that will be counted automatically by the multiprocessing module.
         :param cpu_percent_max: integer, maximum CPU percentage. Above that usage, we will wait before starting new
             execution.
         :param memory_percent_max: integer, maximum memory percentage. Above that usage, we will wait, before starting
@@ -65,7 +136,7 @@ class MultiProcessorRecursive:
             If this is used, the system resources will be checked before starting each new execution from this
             shared dict instead of performing new checks.
 
-        Usage:
+        Usage Examples:
             def unpack_file(file_path):
                 # Process the file at file_path and unpack it.
                 # Return a list of new file paths that were extracted from the provided path.
@@ -74,64 +145,317 @@ class MultiProcessorRecursive:
             # List of file paths to process
             file_paths = ["path1", "path2", "path3"]
 
-            # Create an instance of MultiProcessor
-            # Note: unpacking.unpack_file is passed without parentheses
-            processor = MultiProcessor(
-                process_function=unpack_file,
-                input_list=file_paths,
-                max_workers=4,  # Number of parallel workers
-                cpu_percent_max=80,  # Max CPU usage percentage
-                memory_percent_max=80,  # Max memory usage percentage
-                wait_time=5  # Time to wait if resources are overused
+            # Note: unpack_file Callable is passed to init without parentheses.
+
+            1. Providing the list directly to process at once:
+                # Initialize the processor.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=file_paths,
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            2. Processing each file in the list differently then adding to the list of the multiprocessing instance then executing.
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            3. Processing each file in the list separately, since we're using an unpacking function that
+               will create more files, but the context for this operation is different for extraction
+               of each main file inside the list:
+
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+                    # Process the added file path separately.
+                    processor.run_process()
+
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+        """
+
+        self.process_function: Callable = process_function
+        self.input_list: list = input_list
+        self.max_workers: int = max_workers
+        self.cpu_percent_max: int = cpu_percent_max
+        self.memory_percent_max: int = memory_percent_max
+        self.wait_time: float = wait_time
+        self.system_monitor_manager_dict: multiprocessing.managers.DictProxy = system_monitor_manager_dict
+
+        # Create the pool once and reuse it
+        self.pool: multiprocessing.Pool = multiprocessing.Pool(processes=self.max_workers)
+
+        # Keep track of outstanding async results across calls
+        self.async_results: list = []
+
+    def run_process(self):
+        """
+        Start with the items currently in self.input_list, but whenever a task
+        finishes schedule the children it returns *right away*.
+        The loop ends when there are no more outstanding tasks.
+        """
+        # ----------  internal helpers  ----------
+        outstanding = 0  # tasks that have been submitted but not yet finished
+        done_event = threading.Event()  # let the main thread wait until work is over
+
+        def _submit(item):
+            nonlocal outstanding
+            # Wait for resources *before* submitting a new job
+            system_resources.wait_for_resource_availability(
+                cpu_percent_max=self.cpu_percent_max,
+                memory_percent_max=self.memory_percent_max,
+                wait_time=self.wait_time,
+                system_monitor_manager_dict=self.system_monitor_manager_dict
+            )
+            outstanding += 1
+            self.pool.apply_async(
+                self.process_function,
+                (item,),
+                callback=_on_finish,  # called in the main process when result is ready
+                error_callback=_on_error
             )
 
-            # Run the processing
-            processor.run_process()
+        def _on_finish(result):
+            """Pool calls this in the parent process thread when a job completes."""
+            nonlocal outstanding
+            outstanding -= 1
+
+            # The worker returned a list of new items – submit them immediately
+            if result:
+                for child in result:
+                    _submit(child)
+
+            # If no work left, release the waiter
+            if outstanding == 0:
+                done_event.set()
+
+        def _on_error(exc):
+            """Propagate the first exception and stop everything cleanly."""
+            done_event.set()
+            raise exc  # let your code deal with it – you can customise this
+
+        # ----------  kick‑off  ----------
+        # Schedule the items we already have
+        for item in self.input_list:
+            _submit(item)
+
+        # Clear the input list; after this point everything is driven by callbacks
+        self.input_list.clear()
+
+        # Wait until all recursively spawned work is finished
+        done_event.wait()
+
+    def shutdown(self):
+        """Shuts down the pool gracefully."""
+        if self.pool:
+            self.pool.close()  # Stop accepting new tasks
+            self.pool.join()  # Wait for all tasks to complete
+            self.pool = None
+
+
+class _MultiProcessorRecursiveWithProcessPoolExecutor:
+    def __init__(
+            self,
+            process_function: Callable,
+            input_list: list,
+            max_workers: int = None,
+            cpu_percent_max: int = 80,
+            memory_percent_max: int = 80,
+            wait_time: float = 5,
+            system_monitor_manager_dict: multiprocessing.managers.DictProxy = None
+    ):
         """
+        THIS CLASS USES THE concurrent.futures.ProcessPoolExecutor to achieve parallelism.
+        For some reason I got freezes on exceptions without the exception output after the run_process() method finished
+        and the pool remained open. So, using the MultiProcessorRecursive instead.
 
-        self.process_function = process_function
+        MultiProcessor class. Used to execute functions in parallel. The result of each execution is fed back
+            to the provided function. Making it sort of recursive execution.
+        :param process_function: function, function to execute on the input list.
+        :param input_list: list, list of inputs to process.
+        :param max_workers: integer, number of workers to execute functions in parallel. Default is None, which
+            is the number of CPUs that will be counted automatically by the multiprocessing module.
+        :param cpu_percent_max: integer, maximum CPU percentage. Above that usage, we will wait before starting new
+            execution.
+        :param memory_percent_max: integer, maximum memory percentage. Above that usage, we will wait, before starting
+            new execution.
+        :param wait_time: float, time to wait if the CPU or memory usage is above the maximum percentage.
+        :param system_monitor_manager_dict: multiprocessing.managers.DictProxy, shared manager dict for
+            system monitoring. The object is the output of atomicshop.system_resource_monitor.
+            If you are already running this monitor, you can pass the manager_dict to both the system monitor and this
+            class to share the system resources data.
+            If this is used, the system resources will be checked before starting each new execution from this
+            shared dict instead of performing new checks.
+
+        Usage Examples:
+            def unpack_file(file_path):
+                # Process the file at file_path and unpack it.
+                # Return a list of new file paths that were extracted from the provided path.
+                return [new_file_path1, new_file_path2]  # Example return value
+
+            # List of file paths to process
+            file_paths = ["path1", "path2", "path3"]
+
+            # Note: unpack_file Callable is passed to init without parentheses.
+
+            1. Providing the list directly to process at once:
+                # Initialize the processor.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=file_paths,
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            2. Processing each file in the list differently then adding to the list of the multiprocessing instance then executing.
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            3. Processing each file in the list separately, since we're using an unpacking function that
+               will create more files, but the context for this operation is different for extraction
+               of each main file inside the list:
+
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+                    # Process the added file path separately.
+                    processor.run_process()
+
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+        """
+
+        self.process_function: Callable = process_function
         self.input_list: list = input_list
-        self.max_workers: int = max_workers
         self.cpu_percent_max: int = cpu_percent_max
         self.memory_percent_max: int = memory_percent_max
         self.wait_time: float = wait_time
         self.system_monitor_manager_dict: multiprocessing.managers.DictProxy = system_monitor_manager_dict
 
+        if max_workers is None:
+            max_workers = os.cpu_count()
+        self.max_workers: int = max_workers
+
+        # Create the executor once and reuse it.
+        # noinspection PyTypeChecker
+        self.executor: ProcessPoolExecutor = None
+
+    def _ensure_executor(self):
+        """Create a new pool if we do not have one or if the old one was shut."""
+        if self.executor is None or getattr(self.executor, '_shutdown', False):
+            self.executor = ProcessPoolExecutor(max_workers=self.max_workers)
+
     def run_process(self):
-        with multiprocessing.Pool(processes=self.max_workers) as pool:
-            # Keep track of the async results
-            async_results = []
-
-            while self.input_list:
-                new_input_list = []
-                for item in self.input_list:
-                    # Check system resources before processing each item
-                    system_resources.wait_for_resource_availability(
-                        cpu_percent_max=self.cpu_percent_max,
-                        memory_percent_max=self.memory_percent_max,
-                        wait_time=self.wait_time,
-                        system_monitor_manager_dict=self.system_monitor_manager_dict)
-
-                    # Process the item
-                    async_result = pool.apply_async(self.process_function, (item,))
-                    async_results.append(async_result)
-
-                # Reset input_list for next round of processing
-                self.input_list = []
-
-                # Collect results as they complete
-                for async_result in async_results:
-                    try:
-                        result = async_result.get()
-                        # Assuming process_function returns a list, extend new_input_list
-                        new_input_list.extend(result)
-                    except Exception as e:
-                        print(f"An error occurred: {e}")
+        # Make sure we have a live executor
+        self._ensure_executor()
+
+        work_q = deque(self.input_list)  # breadth‑first queue
+        self.input_list.clear()
+        futures = set()
+
+        # helper to submit jobs up to the concurrency limit
+        def _fill():
+            while work_q and len(futures) < self.max_workers:
+                item = work_q.popleft()
+                system_resources.wait_for_resource_availability(
+                    cpu_percent_max=self.cpu_percent_max,
+                    memory_percent_max=self.memory_percent_max,
+                    wait_time=self.wait_time,
+                    system_monitor_manager_dict=self.system_monitor_manager_dict
+                )
+                futures.add(self.executor.submit(self.process_function, item))
+
+        _fill()  # start the first wave
 
-                # Update the input_list for the next iteration
-                self.input_list = new_input_list
-                # Clear the async_results for the next iteration
-                async_results.clear()
+        while futures:
+            for fut in as_completed(futures):
+                futures.remove(fut)  # a slot just freed up
+
+                # propagate worker exceptions immediately
+                children = fut.result()
+
+                # schedule the newly discovered items
+                if children:
+                    work_q.extend(children)
+
+                _fill()  # keep the pool saturated
+                break  # leave the for‑loop so as_completed resets
+
+    def shutdown(self):
+        """Shuts down the executor gracefully."""
+        if self.executor:
+            self.executor.shutdown(wait=True)  # blocks until all tasks complete
+            self.executor = None
 
 
 class ConcurrentProcessorRecursive:

atomicshop/basics/strings.py

@@ -4,7 +4,7 @@ from pathlib import Path
 import argparse
 
 from . import lists
-from ..print_api import print_api
+from .. import print_api
 
 
 def get_nth_character_from_start(input_string: str, nth: int):
@@ -423,7 +423,7 @@ def replace_words_with_values_from_dict(
 
 
 def replace_strings_with_values_from_dict(string_to_replace: str, dictionary: dict) -> str:
-    """
+    r"""
     Function replaces strings, which are keys with values from dictionary.
 
     :param string_to_replace: string, to replace words in.
@@ -537,7 +537,7 @@ def replace_string_in_file(
             file.writelines(lines)
 
     # Output the relevant line numbers
-    print_api(f"Target string found on the following lines: {changed_lines}", **(print_kwargs or {}))
+    print_api.print_api(f"Target string found on the following lines: {changed_lines}", **(print_kwargs or {}))
     return changed_lines
 
 
@@ -571,3 +571,72 @@ def replace_string_in_file_main_argparse():
         new_string=args.new_string,
         find_only=args.find_only
     )
+
+
+def _replace_string_in_variable():
+    """
+    Replace string in a string variable, but do it by a meta variable inside the string.
+    This is just an example, using the 'Template' class from the 'string' module is a better way to do it.
+    """
+
+    from string import Template
+    import os
+    import tempfile
+    import subprocess
+
+    get_docker_url: str = "https://get.docker.com"
+    docker_proxy_image_name: str = "rpardini/docker-registry-proxy:0.6.5"
+    preparation_output_dir: str = str(Path(__file__).parent / "offline-bundle")
+
+    class BashTemplate(Template):
+        # Anything that is not '$', which is a default delimiter in Template class, but also used in bash scripts.
+        # The below symbol can be printed from keyboard by holding 'Alt' and typing '0167' on the numeric keypad.
+        delimiter = '§'
+
+    bash_tmpl = BashTemplate(r"""#!/usr/bin/env bash
+#
+set -Eeuo pipefail
+
+die()       { echo "ERROR: $*" >&2; exit 1; }
+need_root() { [[ $EUID -eq 0 ]] || die "Run as root (use sudo)"; }
+need_cmd() {
+  local cmd=$1
+  local pkg=${2:-$1}               # default package == command
+  if ! command -v "$cmd" &>/dev/null; then
+    echo "[*] $cmd not found – installing $pkg ..."
+    apt-get update -qq
+    DEBIAN_FRONTEND=noninteractive \
+      apt-get install -y --no-install-recommends "$pkg" || \
+      die "Unable to install required package: $pkg"
+  fi
+}
+
+need_root
+need_cmd curl                 # binary and pkg are both “curl”
+need_cmd gpg                  # → apt-get install gpg
+
+DRY_LOG=$(curl -fsSL "§url" | bash -s -- --dry-run)
+IMAGE="§proxyimage"
+ARCHIVE="$OUTDIR/registry-proxy-image.tar.gz"
+zip -r "§output_zip" "$OUTDIR"
+        """)
+
+    # Substitute the variables in the bash script template.
+    bash_script = bash_tmpl.substitute(
+        url=get_docker_url, proxyimage=docker_proxy_image_name, output_zip=preparation_output_dir)
+
+    # Write it to a secure temporary file.
+    with tempfile.NamedTemporaryFile('w', delete=False, suffix='.sh') as f:
+        f.write(bash_script)
+        temp_path = f.name
+    os.chmod(temp_path, 0o755)  # make it executable
+
+    # Decide where the bundle should land (optional argument to the script).
+    cmd = ["sudo", temp_path, preparation_output_dir]  # use sudo because the script demands root
+
+    # Run it and stream output live.
+    try:
+        subprocess.run(cmd, check=True)
+    finally:
+        # 5. Clean up the temp file unless you want to inspect it.
+        os.remove(temp_path)

atomicshop/basics/threads.py

@@ -18,6 +18,20 @@ def current_thread_id():
     return thread_id
 
 
+def get_current_thread_name():
+    return threading.current_thread().name
+
+
+def set_current_thread_name(name: str):
+    threading.current_thread().name = name
+
+
+def set_current_thread_name_by_process_name():
+    import multiprocessing
+    current_process_name = multiprocessing.current_process().name
+    threading.current_thread().name = current_process_name
+
+
 def get_number_of_active_threads():
     return threading.active_count()
 

atomicshop/basics/tracebacks.py

@@ -1,16 +1,26 @@
 import traceback
 
 
-def get_as_string(one_line: bool = False, replace_end: str = str()) -> str:
+def get_as_string(
+        exc: BaseException = None,
+        one_line: bool = False,
+        replace_end: str = str()
+) -> str:
     """
     Returns traceback as string.
 
+    :param exc: Exception to get traceback from. If 'None', current exception will be used.
     :param one_line: If 'True', traceback will be returned as one line.
     :param replace_end: If 'one_line' is 'True', this string will be used to replace '\n' in traceback.
     :return: Traceback as string.
     """
 
+    if exc is None:
+        stringed_exception: str = traceback.format_exc()
+    else:
+        stringed_exception: str = ''.join(traceback.TracebackException.from_exception(exc).format())
+
     if not one_line:
-        return traceback.format_exc()
+        return stringed_exception
     else:
-        return traceback.format_exc().replace('\n', replace_end)
+        return stringed_exception.replace('\n', replace_end)