atomicshop 2.19.0__py3-none-any.whl → 2.19.2__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.19.0'
+__version__ = '2.19.2'

atomicshop/basics/multiprocesses.py

@@ -3,6 +3,7 @@ import multiprocessing.managers
 import queue
 import concurrent.futures
 from concurrent.futures import ProcessPoolExecutor, as_completed
+from typing import Callable
 
 from ..import system_resources
 
@@ -38,7 +39,7 @@ def process_wrap_queue(function_reference, *args, **kwargs):
 class MultiProcessorRecursive:
     def __init__(
             self,
-            process_function,
+            process_function: Callable,
             input_list: list,
             max_workers: int = None,
             cpu_percent_max: int = 80,
@@ -52,7 +53,7 @@ class MultiProcessorRecursive:
         :param process_function: function, function to execute on the input list.
         :param input_list: list, list of inputs to process.
         :param max_workers: integer, number of workers to execute functions in parallel. Default is None, which
-            is the number of CPUs.
+            is the number of CPUs that will be counted automatically by the multiprocessing module.
         :param cpu_percent_max: integer, maximum CPU percentage. Above that usage, we will wait before starting new
             execution.
         :param memory_percent_max: integer, maximum memory percentage. Above that usage, we will wait, before starting
@@ -65,7 +66,7 @@ class MultiProcessorRecursive:
             If this is used, the system resources will be checked before starting each new execution from this
             shared dict instead of performing new checks.
 
-        Usage:
+        Usage Examples:
             def unpack_file(file_path):
                 # Process the file at file_path and unpack it.
                 # Return a list of new file paths that were extracted from the provided path.
@@ -74,22 +75,71 @@ class MultiProcessorRecursive:
             # List of file paths to process
             file_paths = ["path1", "path2", "path3"]
 
-            # Create an instance of MultiProcessor
-            # Note: unpacking.unpack_file is passed without parentheses
-            processor = MultiProcessor(
-                process_function=unpack_file,
-                input_list=file_paths,
-                max_workers=4,  # Number of parallel workers
-                cpu_percent_max=80,  # Max CPU usage percentage
-                memory_percent_max=80,  # Max memory usage percentage
-                wait_time=5  # Time to wait if resources are overused
-            )
-
-            # Run the processing
-            processor.run_process()
+            # Note: unpack_file Callable is passed to init without parentheses.
+
+            1. Providing the list directly to process at once:
+                # Initialize the processor.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=file_paths,
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            2. Processing each file in the list differently then adding to the list of the multiprocessing instance then executing.
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            3. Processing each file in the list separately, since we're using an unpacking function that
+               will create more files, but the context for this operation is different for extraction
+               of each main file inside the list:
+
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+                    # Process the added file path separately.
+                    processor.run_process()
+
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
         """
 
-        self.process_function = process_function
+        self.process_function: Callable = process_function
         self.input_list: list = input_list
         self.max_workers: int = max_workers
         self.cpu_percent_max: int = cpu_percent_max
@@ -97,41 +147,50 @@ class MultiProcessorRecursive:
         self.wait_time: float = wait_time
         self.system_monitor_manager_dict: multiprocessing.managers.DictProxy = system_monitor_manager_dict
 
+        # Create the pool once and reuse it
+        self.pool: multiprocessing.Pool = multiprocessing.Pool(processes=self.max_workers)
+
+        # Keep track of outstanding async results across calls
+        self.async_results: list = []
+
     def run_process(self):
-        with multiprocessing.Pool(processes=self.max_workers) as pool:
-            # Keep track of the async results
-            async_results = []
-
-            while self.input_list:
-                new_input_list = []
-                for item in self.input_list:
-                    # Check system resources before processing each item
-                    system_resources.wait_for_resource_availability(
-                        cpu_percent_max=self.cpu_percent_max,
-                        memory_percent_max=self.memory_percent_max,
-                        wait_time=self.wait_time,
-                        system_monitor_manager_dict=self.system_monitor_manager_dict)
-
-                    # Process the item
-                    async_result = pool.apply_async(self.process_function, (item,))
-                    async_results.append(async_result)
-
-                # Reset input_list for next round of processing
-                self.input_list = []
-
-                # Collect results as they complete
-                for async_result in async_results:
-                    try:
-                        result = async_result.get()
-                        # Assuming process_function returns a list, extend new_input_list
-                        new_input_list.extend(result)
-                    except Exception:
-                        raise
-
-                # Update the input_list for the next iteration
-                self.input_list = new_input_list
-                # Clear the async_results for the next iteration
-                async_results.clear()
+        while self.input_list:
+            new_input_list = []
+            for item in self.input_list:
+                # Check system resources before processing each item
+                system_resources.wait_for_resource_availability(
+                    cpu_percent_max=self.cpu_percent_max,
+                    memory_percent_max=self.memory_percent_max,
+                    wait_time=self.wait_time,
+                    system_monitor_manager_dict=self.system_monitor_manager_dict)
+
+                # Process the item
+                async_result = self.pool.apply_async(self.process_function, (item,))
+                self.async_results.append(async_result)
+
+            # Reset input_list for next round of processing
+            self.input_list = []
+
+            # Collect results as they complete
+            for async_result in self.async_results:
+                try:
+                    result = async_result.get()
+                    # Assuming process_function returns a list, extend new_input_list
+                    new_input_list.extend(result)
+                except Exception:
+                    raise
+
+            # Update the input_list for the next iteration
+            self.input_list = new_input_list
+            # Clear the async_results for the next iteration
+            self.async_results.clear()
+
+    def shutdown_pool(self):
+        """Shuts down the pool gracefully."""
+        if self.pool:
+            self.pool.close()  # Stop accepting new tasks
+            self.pool.join()  # Wait for all tasks to complete
+            self.pool = None
 
 
 class ConcurrentProcessorRecursive:

{atomicshop-2.19.0.dist-info → atomicshop-2.19.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 2.19.0
+Version: 2.19.2
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License

{atomicshop-2.19.0.dist-info → atomicshop-2.19.2.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=OIXYlXkKerHNZWXlxoV3_8FloSrzjYDaCOGsfFfHcjg,123
+atomicshop/__init__.py,sha256=SBk9Y1eYsyMmy9xWJ483qY8NFoOX5qMgy1HMpVCjXO0,123
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
 atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
@@ -101,7 +101,7 @@ atomicshop/basics/isinstancing.py,sha256=fQ35xfqbguQz2BUn-3a4KVGskhTcIn8JjRtxV2r
 atomicshop/basics/list_of_classes.py,sha256=PJoE1VJdhhQ4gSFr88zW7IApXd4Ez7xLz-7vAM-7gug,978
 atomicshop/basics/list_of_dicts.py,sha256=tj0LNPf1ljNI_qpoO-PiOT4Ulmk1M-UpTGyn9twVcw8,8039
 atomicshop/basics/lists.py,sha256=I0C62vrDrNwCTNl0EjUZNa1Jsd8l0rTkp28GEx9QoEI,4258
-atomicshop/basics/multiprocesses.py,sha256=oU6LjcLLGBtPIGJzZBpDWoLU3HRmMoanITEOE2luAYw,18799
+atomicshop/basics/multiprocesses.py,sha256=vzL1lGiXpfPWbuLsQF9e7c9vbo59rITL2dBLi0CCpZ0,21741
 atomicshop/basics/numbers.py,sha256=ESX0z_7o_ok3sOmCKAUBoZinATklgMy2v-4RndqXlVM,1837
 atomicshop/basics/package_module.py,sha256=fBd0uVgFce25ZCVtLq83iyowRlbwdWYFj_t4Ml7LU14,391
 atomicshop/basics/randoms.py,sha256=DmYLtnIhDK29tAQrGP1Nt-A-v8WC7WIEB8Edi-nk3N4,282
@@ -322,8 +322,8 @@ atomicshop/wrappers/socketw/statistics_csv.py,sha256=fgMzDXI0cybwUEqAxprRmY3lqbh
 atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/winregw/winreg_installed_software.py,sha256=Qzmyktvob1qp6Tjk2DjLfAqr_yXV0sgWzdMW_9kwNjY,2345
 atomicshop/wrappers/winregw/winreg_network.py,sha256=AENV88H1qDidrcpyM9OwEZxX5svfi-Jb4N6FkS1xtqA,8851
-atomicshop-2.19.0.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-2.19.0.dist-info/METADATA,sha256=BxY2IEfmMAFhO4teHrr8TYRCzgnf-SsrWaIh_l29YOo,10630
-atomicshop-2.19.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-atomicshop-2.19.0.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-2.19.0.dist-info/RECORD,,
+atomicshop-2.19.2.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-2.19.2.dist-info/METADATA,sha256=hyYsPo_Wfx0n6Q8cdQNEWsP4RqH5_oo3og7-2M5hFBs,10630
+atomicshop-2.19.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+atomicshop-2.19.2.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-2.19.2.dist-info/RECORD,,