atomicshop 2.19.0__py3-none-any.whl → 2.19.1__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.19.0'
+__version__ = '2.19.1'

atomicshop/basics/multiprocesses.py

@@ -52,7 +52,7 @@ class MultiProcessorRecursive:
         :param process_function: function, function to execute on the input list.
         :param input_list: list, list of inputs to process.
         :param max_workers: integer, number of workers to execute functions in parallel. Default is None, which
-            is the number of CPUs.
+            is the number of CPUs that will be counted automatically by the multiprocessing module.
         :param cpu_percent_max: integer, maximum CPU percentage. Above that usage, we will wait before starting new
             execution.
         :param memory_percent_max: integer, maximum memory percentage. Above that usage, we will wait, before starting
@@ -65,7 +65,7 @@ class MultiProcessorRecursive:
             If this is used, the system resources will be checked before starting each new execution from this
             shared dict instead of performing new checks.
 
-        Usage:
+        Usage Examples:
             def unpack_file(file_path):
                 # Process the file at file_path and unpack it.
                 # Return a list of new file paths that were extracted from the provided path.
@@ -74,19 +74,68 @@ class MultiProcessorRecursive:
             # List of file paths to process
             file_paths = ["path1", "path2", "path3"]
 
-            # Create an instance of MultiProcessor
-            # Note: unpacking.unpack_file is passed without parentheses
-            processor = MultiProcessor(
-                process_function=unpack_file,
-                input_list=file_paths,
-                max_workers=4,  # Number of parallel workers
-                cpu_percent_max=80,  # Max CPU usage percentage
-                memory_percent_max=80,  # Max memory usage percentage
-                wait_time=5  # Time to wait if resources are overused
-            )
-
-            # Run the processing
-            processor.run_process()
+            # Note: unpack_file Callable is passed to init without parentheses.
+
+            1. Providing the list directly to process at once:
+                # Initialize the processor.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=file_paths,
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            2. Processing each file in the list differently then adding to the list of the multiprocessing instance then executing.
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+
+                # Process the list of files at once.
+                processor.run_process()
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
+
+            3. Processing each file in the list separately, since we're using an unpacking function that
+               will create more files, but the context for this operation is different for extraction
+               of each main file inside the list:
+
+                # Initialize the processor once, before the loop, with empty input_list.
+                processor = MultiProcessor(
+                    process_function=unpack_file,
+                    input_list=[],
+                    max_workers=4,  # Number of parallel workers
+                    cpu_percent_max=80,  # Max CPU usage percentage
+                    memory_percent_max=80,  # Max memory usage percentage
+                    wait_time=5  # Time to wait if resources are overused
+                )
+
+                for file_path in file_paths:
+                    # <Process each file>.
+                    # Add the result to the input_list of the processor.
+                    processor.input_list.append(file_path)
+                    # Process the added file path separately.
+                    processor.run_process()
+
+                # Shutdown the pool processes after processing.
+                processor.shutdown_pool()
         """
 
         self.process_function = process_function
@@ -97,41 +146,57 @@ class MultiProcessorRecursive:
         self.wait_time: float = wait_time
         self.system_monitor_manager_dict: multiprocessing.managers.DictProxy = system_monitor_manager_dict
 
+        # Create the pool once and reuse it
+        self.pool: multiprocessing.Pool = multiprocessing.Pool(processes=self.max_workers)
+
+        # Keep track of outstanding async results across calls
+        self.async_results: list = []
+
     def run_process(self):
-        with multiprocessing.Pool(processes=self.max_workers) as pool:
-            # Keep track of the async results
-            async_results = []
-
-            while self.input_list:
-                new_input_list = []
-                for item in self.input_list:
-                    # Check system resources before processing each item
-                    system_resources.wait_for_resource_availability(
-                        cpu_percent_max=self.cpu_percent_max,
-                        memory_percent_max=self.memory_percent_max,
-                        wait_time=self.wait_time,
-                        system_monitor_manager_dict=self.system_monitor_manager_dict)
-
-                    # Process the item
-                    async_result = pool.apply_async(self.process_function, (item,))
-                    async_results.append(async_result)
-
-                # Reset input_list for next round of processing
-                self.input_list = []
-
-                # Collect results as they complete
-                for async_result in async_results:
-                    try:
-                        result = async_result.get()
-                        # Assuming process_function returns a list, extend new_input_list
-                        new_input_list.extend(result)
-                    except Exception:
-                        raise
-
-                # Update the input_list for the next iteration
-                self.input_list = new_input_list
-                # Clear the async_results for the next iteration
-                async_results.clear()
+        # This method can be called multiple times to add new tasks without creating a new pool
+        if not self.input_list:
+            return  # Nothing to process
+
+        new_input_list = []
+
+        for item in self.input_list:
+            # Check system resources before scheduling each item
+            system_resources.wait_for_resource_availability(
+                cpu_percent_max=self.cpu_percent_max,
+                memory_percent_max=self.memory_percent_max,
+                wait_time=self.wait_time,
+                system_monitor_manager_dict=self.system_monitor_manager_dict
+            )
+
+            # Schedule the task on the existing pool
+            async_result = self.pool.apply_async(self.process_function, (item,))
+            self.async_results.append(async_result)
+
+        # Clear the input_list now that tasks are scheduled
+        self.input_list = []
+
+        # Collect results for the tasks that were scheduled
+        for async_result in self.async_results:
+            try:
+                result = async_result.get()  # Blocking wait for result
+                # Assuming process_function returns a list, extend new_input_list
+                new_input_list.extend(result)
+            except Exception as e:
+                # Handle exceptions as needed
+                raise e
+
+        # Clear collected async results since they've been processed
+        self.async_results.clear()
+
+        # Update input_list with new files (if any) for further processing
+        self.input_list = new_input_list
+
+    def shutdown_pool(self):
+        """Shuts down the pool gracefully."""
+        if self.pool:
+            self.pool.close()  # Stop accepting new tasks
+            self.pool.join()  # Wait for all tasks to complete
+            self.pool = None
 
 
 class ConcurrentProcessorRecursive:

{atomicshop-2.19.0.dist-info → atomicshop-2.19.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 2.19.0
+Version: 2.19.1
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License

{atomicshop-2.19.0.dist-info → atomicshop-2.19.1.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=OIXYlXkKerHNZWXlxoV3_8FloSrzjYDaCOGsfFfHcjg,123
+atomicshop/__init__.py,sha256=YLnFX8FxOuGbWDCdCSt4RjxgiYy5Cc-U3A7YXjRxSd8,123
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
 atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
@@ -101,7 +101,7 @@ atomicshop/basics/isinstancing.py,sha256=fQ35xfqbguQz2BUn-3a4KVGskhTcIn8JjRtxV2r
 atomicshop/basics/list_of_classes.py,sha256=PJoE1VJdhhQ4gSFr88zW7IApXd4Ez7xLz-7vAM-7gug,978
 atomicshop/basics/list_of_dicts.py,sha256=tj0LNPf1ljNI_qpoO-PiOT4Ulmk1M-UpTGyn9twVcw8,8039
 atomicshop/basics/lists.py,sha256=I0C62vrDrNwCTNl0EjUZNa1Jsd8l0rTkp28GEx9QoEI,4258
-atomicshop/basics/multiprocesses.py,sha256=oU6LjcLLGBtPIGJzZBpDWoLU3HRmMoanITEOE2luAYw,18799
+atomicshop/basics/multiprocesses.py,sha256=m8Vg6YDZX_Xz3r5uXvaTkFWQ-x6OvGOnTAVgWMFmvbU,21909
 atomicshop/basics/numbers.py,sha256=ESX0z_7o_ok3sOmCKAUBoZinATklgMy2v-4RndqXlVM,1837
 atomicshop/basics/package_module.py,sha256=fBd0uVgFce25ZCVtLq83iyowRlbwdWYFj_t4Ml7LU14,391
 atomicshop/basics/randoms.py,sha256=DmYLtnIhDK29tAQrGP1Nt-A-v8WC7WIEB8Edi-nk3N4,282
@@ -322,8 +322,8 @@ atomicshop/wrappers/socketw/statistics_csv.py,sha256=fgMzDXI0cybwUEqAxprRmY3lqbh
 atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/winregw/winreg_installed_software.py,sha256=Qzmyktvob1qp6Tjk2DjLfAqr_yXV0sgWzdMW_9kwNjY,2345
 atomicshop/wrappers/winregw/winreg_network.py,sha256=AENV88H1qDidrcpyM9OwEZxX5svfi-Jb4N6FkS1xtqA,8851
-atomicshop-2.19.0.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-2.19.0.dist-info/METADATA,sha256=BxY2IEfmMAFhO4teHrr8TYRCzgnf-SsrWaIh_l29YOo,10630
-atomicshop-2.19.0.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-atomicshop-2.19.0.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-2.19.0.dist-info/RECORD,,
+atomicshop-2.19.1.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-2.19.1.dist-info/METADATA,sha256=R3uYBPX5Re5tpvd0nokDfmh_F1z1rWPThXWTPLvoFE4,10630
+atomicshop-2.19.1.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+atomicshop-2.19.1.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-2.19.1.dist-info/RECORD,,