atomicshop 2.5.16__py3-none-any.whl → 2.6.0__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.5.16'
+__version__ = '2.6.0'

atomicshop/basics/multiprocesses.py

@@ -1,5 +1,8 @@
 import multiprocessing
 import queue
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+from ..import system_resources
 
 
 def process_wrap_queue(function_reference, *args, **kwargs):
@@ -28,3 +31,312 @@ def process_wrap_queue(function_reference, *args, **kwargs):
     process_object.join()
 
     return queue_object.get()
+
+
+class MultiProcessorRecursive:
+    def __init__(
+            self,
+            process_function,
+            input_list: list,
+            max_workers: int = None,
+            cpu_percent_max: int = 80,
+            memory_percent_max: int = 80,
+            wait_time: float = 5
+    ):
+        """
+        MultiProcessor class. Used to execute functions in parallel. The result of each execution is feeded back
+            to the provided function. Making it sort of recursive execution.
+        :param process_function: function, function to execute on the input list.
+        :param input_list: list, list of inputs to process.
+        :param max_workers: integer, number of workers to execute functions in parallel. Default is None, which
+            is the number of CPUs.
+        :param cpu_percent_max: integer, maximum CPU percentage. Above that usage, we will wait before starting new
+            execution.
+        :param memory_percent_max: integer, maximum memory percentage. Above that usage, we will wait, before starting
+            new execution.
+        :param wait_time: float, time to wait if the CPU or memory usage is above the maximum percentage.
+
+        Usage:
+            def unpack_file(file_path):
+                # Process the file at file_path and unpack it.
+                # Return a list of new file paths that were extracted from the provided path.
+                return [new_file_path1, new_file_path2]  # Example return value
+
+            # List of file paths to process
+            file_paths = ["path1", "path2", "path3"]
+
+            # Create an instance of MultiProcessor
+            # Note: unpacking.unpack_file is passed without parentheses
+            processor = MultiProcessor(
+                process_function=unpack_file,
+                input_list=file_paths,
+                max_workers=4,  # Number of parallel workers
+                cpu_percent_max=80,  # Max CPU usage percentage
+                memory_percent_max=80,  # Max memory usage percentage
+                wait_time=5  # Time to wait if resources are overused
+            )
+
+            # Run the processing
+            processor.run_process()
+        """
+
+        self.process_function = process_function
+        self.input_list: list = input_list
+        self.max_workers: int = max_workers
+        self.cpu_percent_max: int = cpu_percent_max
+        self.memory_percent_max: int = memory_percent_max
+        self.wait_time: float = wait_time
+
+    def run_process(self):
+        with multiprocessing.Pool(processes=self.max_workers) as pool:
+            while self.input_list:
+                system_resources.wait_for_resource_availability(
+                    cpu_percent_max=self.cpu_percent_max,
+                    memory_percent_max=self.memory_percent_max,
+                    wait_time=self.wait_time)
+
+                # Apply the provided function to the items in the input list
+                results = pool.map(self.process_function, self.input_list)
+
+                # Flatten list of lists and update the queue
+                self.input_list = [item for sublist in results for item in sublist]
+
+
+class _MultiProcessorTest:
+    def __init__(self, worker_count: int = 8, initialize_at_start: bool = True):
+        """
+        MultiProcessor class. Used to execute functions in parallel.
+        :param worker_count: integer, number of workers to execute functions in parallel.
+        :param initialize_at_start: boolean, if True, the workers will be initialized at the start of the class.
+
+        Usage:
+            # Example functions
+            def my_function1(x):
+                return x * x
+
+            def my_callback(result):
+                print(f"Result received: {result}")
+
+            # Usage example
+            if __name__ == "__main__":
+                processor = MultiProcessor()
+
+                # Adding tasks with callback (asynchronous)
+                for i in range(5):
+                    processor.add_task(my_function1, i, callback=my_callback)
+
+                # Adding a task and waiting for the result (synchronous)
+                result = processor.add_task(my_function1, 10, wait=True)
+                print(f"Synchronous result: {result}")
+
+                # Shutdown and wait for all tasks to complete
+                processor.shutdown()
+        """
+
+        self.worker_count: int = worker_count
+        self.task_queue = multiprocessing.Queue()
+        self.result_queue = multiprocessing.Queue()
+        self.workers: list = list()
+
+        if initialize_at_start:
+            self._init_workers()
+
+    def _worker(self):
+        while True:
+            task = self.task_queue.get()
+            if task is None:  # Termination signal
+                break
+            func, args, kwargs, callback, return_result = task
+            try:
+                result = func(*args, **kwargs)
+                if callback:
+                    callback(result)
+                if return_result:
+                    self.result_queue.put(result)
+            except Exception as e:
+                if callback:
+                    callback(e)
+                if return_result:
+                    self.result_queue.put(e)
+
+    def _init_workers(self):
+        for _ in range(self.worker_count):
+            p = multiprocessing.Process(target=self._worker)
+            p.start()
+            self.workers.append(p)
+
+    def add_task(self, func, *args, callback=None, wait=False, **kwargs):
+        """
+        Add a task to the queue.
+        :param func: reference, function to execute.
+        :param args: arguments for the function.
+        :param callback: reference, function to execute after the task is completed. The result of the task will be
+            passed to the callback function.
+        :param wait: boolean, if True, the function will wait for the task to complete and return the result.
+        :param kwargs: keyword arguments for the function.
+        :return:
+        """
+        if wait:
+            self.task_queue.put((func, args, kwargs, None, True))
+            return self.result_queue.get()
+        else:
+            self.task_queue.put((func, args, kwargs, callback, False))
+            return None
+
+    def shutdown(self):
+        # Signal workers to exit
+        for _ in self.workers:
+            self.task_queue.put(None)
+
+        # Wait for all workers to finish
+        for p in self.workers:
+            p.join()
+
+
+class _MultiConcuratorTest:
+    def __init__(self, max_workers: int = 2, initialize_at_start: bool = False):
+        """
+        MultiConcurator class. Used to execute functions in parallel with the ProcessPoolExecutor.
+
+        :param max_workers: integer, number of workers to execute functions in parallel.
+        :param initialize_at_start: boolean, if True, the workers will be initialized at the start of the class.
+
+        Usage 1:
+            if __name__ == "__main__":
+            concurator = MultiConcurator(max_workers=4, initialize_at_start=True)
+
+            # ... add tasks to concurator ...
+
+            concurator.wait_for_completion()
+
+            # Continuously get and process results
+            for result in concurator.get_results():
+                print(result)
+
+            concurator.shutdown()
+
+        Usage 2:
+            # Basic Usage - Executing Functions in Parallel
+            This example shows how to execute a simple function in parallel.
+
+            def square(number):
+                return number * number
+
+            if __name__ == "__main__":
+                concurator = MultiConcurator(max_workers=4, initialize_at_start=True)
+
+                # Submit tasks
+                for num in range(10):
+                    concurator.add_to_queue(square, num)
+
+                # Wait for completion and handle results
+                concurator.wait_for_completion()
+
+                # Retrieve results
+                while not concurator.queue.empty():
+                    result = concurator.queue.get()
+                    print(result)
+
+                concurator.shutdown()
+
+        Usage 3:
+            # Handling Results with a Callback Function
+            # This example demonstrates handling the results of each task using a callback function.
+
+            def multiply(x, y):
+                return x * y
+
+            def handle_result(result):
+                print(f"Result: {result}")
+
+            if __name__ == "__main__":
+                concurator = MultiConcurator(max_workers=4, initialize_at_start=True)
+
+                for i in range(5):
+                    concurator.add_to_queue(multiply, i, i+1)
+
+                concurator.wait_for_completion(handle_result)
+                concurator.shutdown()
+
+        Usage 4:
+            # Dynamically Initializing the Executor
+            # This example shows how to initialize the executor dynamically and process tasks.
+
+            def compute_power(base, exponent):
+                return base ** exponent
+
+            if __name__ == "__main__":
+                concurator = MultiConcurator(max_workers=3)
+
+                # Initialize executor when needed
+                concurator.init_executor()
+
+                for i in range(1, 5):
+                    concurator.add_to_queue(compute_power, i, 2)
+
+                concurator.wait_for_completion()
+                while not concurator.queue.empty():
+                    print(concurator.queue.get())
+
+                concurator.shutdown()
+
+        Usage 5:
+            # Handling Exceptions in Tasks
+            # This example demonstrates how you can handle exceptions that might occur in the tasks.
+
+            def risky_division(x, y):
+                if y == 0:
+                    raise ValueError("Cannot divide by zero")
+                return x / y
+
+            def handle_result(result):
+                if isinstance(result, Exception):
+                    print(f"Error occurred: {result}")
+                else:
+                    print(f"Result: {result}")
+
+            if __name__ == "__main__":
+                concurator = MultiConcurator(max_workers=4, initialize_at_start=True)
+
+                concurator.add_to_queue(risky_division, 10, 2)
+                concurator.add_to_queue(risky_division, 10, 0)  # This will raise an exception
+
+                concurator.wait_for_completion(handle_result)
+                concurator.shutdown()
+        """
+
+        self.executor = None
+        self.max_workers: int = max_workers
+        self.futures: list = list()
+        self.queue = multiprocessing.Queue()
+
+        if initialize_at_start:
+            self.init_executor()
+
+    def init_executor(self):
+        if self.executor is None:
+            self.executor = ProcessPoolExecutor(max_workers=self.max_workers)
+
+    def add_to_queue(self, func, *args, **kwargs):
+        if self.executor is None:
+            raise RuntimeError("Executor has not been initialized. Call init_executor() first.")
+        future = self.executor.submit(func, *args, **kwargs)
+        self.futures.append(future)
+        return future
+
+    def wait_for_completion(self, handle_result=None):
+        for future in as_completed(self.futures):
+            result = future.result()
+            if handle_result:
+                handle_result(result)
+            else:
+                self.queue.put(result)
+
+    def get_results(self):
+        while not self.queue.empty():
+            yield self.queue.get()
+
+    def shutdown(self):
+        if self.executor is not None:
+            self.executor.shutdown()
+            self.executor = None

atomicshop/basics/numbers.py

@@ -5,3 +5,16 @@ def is_divisible(n, k):
         return True
     else:
         return False
+
+
+def find_highest_number(numbers: list[float, int, str]):
+    """
+    Takes a list of numbers (float or integers) and returns the highest one.
+
+    :param numbers: List of floats or integers.
+    :return: The highest number in the list.
+    """
+    if not numbers:
+        raise ValueError('The list of numbers is empty.')
+
+    return max(numbers)

atomicshop/system_resources.py

@@ -0,0 +1,26 @@
+import psutil
+import time
+
+from .print_api import print_api
+
+
+def check_system_resources():
+    cpu_usage = psutil.cpu_percent(interval=0.1)
+    memory_usage = psutil.virtual_memory().percent
+    return cpu_usage, memory_usage
+
+
+def wait_for_resource_availability(cpu_percent_max: int = 80, memory_percent_max: int = 80, wait_time: float = 5):
+    """
+    Wait for system resources to be available.
+    :param cpu_percent_max: int, maximum CPU percentage. Above that usage, we will wait.
+    :param memory_percent_max: int, maximum memory percentage. Above that usage, we will wait.
+    :param wait_time: float, time to wait between checks.
+    :return: None
+    """
+    while True:
+        cpu, memory = check_system_resources()
+        if cpu < cpu_percent_max or memory < memory_percent_max:
+            break
+        print_api(f"Waiting for resources to be available... CPU: {cpu}%, Memory: {memory}%", color='yellow')
+        time.sleep(wait_time)  # Wait for 'wait_time' seconds before checking again

atomicshop/wrappers/loggingw/loggingw.py

@@ -231,3 +231,21 @@ def start_queue_listener_for_file_handler_and_get_queue_handler(file_handler):
     handlers.start_queue_listener_for_file_handler(file_handler, queue_object)
 
     return handlers.get_queue_handler(queue_object)
+
+
+def disable_default_logger():
+    """
+    Function to disable default logger.
+    """
+
+    # # Get the default logger.
+    # logger = logging.getLogger()
+    # # Remove all handlers from the logger.
+    # logger.handlers.clear()
+    # # Set the logger level to 'NOTSET'.
+    # logger.setLevel(logging.NOTSET)
+    # # Disable propagation from the 'root' logger, so we will not see the messages twice.
+    # loggers.set_propagation(logger)
+
+    # Disabling the default logger in Python
+    logging.disable(logging.CRITICAL)

{atomicshop-2.5.16.dist-info → atomicshop-2.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: atomicshop
-Version: 2.5.16
+Version: 2.6.0
 Summary: Atomic functions and classes to make developer life easier
 Author: Denis Kras
 License: MIT License

{atomicshop-2.5.16.dist-info → atomicshop-2.6.0.dist-info}/RECORD

@@ -1,4 +1,4 @@
-atomicshop/__init__.py,sha256=ayLlabN9C1kOn1T8udiljMqhA2XOEFO5ihTvhBJ00kA,123
+atomicshop/__init__.py,sha256=NXDiuw7KfoCVJPfpRXHeFAgfmcvLx-4zWrrJZldKjJ0,122
 atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
 atomicshop/appointment_management.py,sha256=N3wVGJgrqJfsj_lqiRfaL3FxMEe57by5Stzanh189mk,7263
 atomicshop/archiver.py,sha256=QuDKx6bJEM29u-7r2tsizXEEGJmPuv0Xm91JQOpg__8,7152
@@ -34,6 +34,7 @@ atomicshop/sound.py,sha256=KSzWRF8dkpEVXmFidIv-Eftc3kET-hQzQOxZRE7rMto,24297
 atomicshop/speech_recognize.py,sha256=55-dIjgkpF93mvJnJuxSFuft5H5eRvGNlUj9BeIOZxk,5903
 atomicshop/ssh_remote.py,sha256=Sas3nrQv8ardxR51t59xZZsYm8nvUcA7tMSqEDViRLk,17155
 atomicshop/sys_functions.py,sha256=MTBxRve5bh58SPvhX3gMiGqHlSBuI_rdNN1NnnBBWqI,906
+atomicshop/system_resources.py,sha256=rYRNsl9BaJgFSSgygXNtUsVFxUb9WGxZ1Ek7bKLomVo,1033
 atomicshop/tempfiles.py,sha256=uq1ve2WlWehZ3NOTXJnpBBMt6HyCdBufqedF0HyzA6k,2517
 atomicshop/timer.py,sha256=KxBBgVM8po6pUJDW8TgY1UXj0iiDmRmL5XDCq0VHAfU,1670
 atomicshop/urls.py,sha256=CQl1j1kjEVDlAuYJqYD9XxPF1SUSgrmG8PjlcXNEKsQ,597
@@ -72,8 +73,8 @@ atomicshop/basics/hexs.py,sha256=i8CTG-J0TGGa25yFSbWEvpVyHFnof_qSWUrmXY-ylKM,105
 atomicshop/basics/isinstancing.py,sha256=fQ35xfqbguQz2BUn-3a4KVGskhTcIn8JjRtxV2rFcRQ,876
 atomicshop/basics/list_of_dicts.py,sha256=fu0H6dUD9uUo2kl7FYIrWzOQMwCmg7qRxGnFM5S319E,5716
 atomicshop/basics/lists.py,sha256=ZyTjHyvkta-4_xCG1P-LFMENELmgAYlDdPq4hMRAOR8,2545
-atomicshop/basics/multiprocesses.py,sha256=GVeyF-r9Q6SAjpJ46eqkZQ4QUli554n-CAIE_W8pij8,958
-atomicshop/basics/numbers.py,sha256=FRjAH1Thk3z3ayxq0Ik6Wh93ELDRk1geyDYTv8amZBE,165
+atomicshop/basics/multiprocesses.py,sha256=On2PbXDpVBkhdocISNDTGFcElpSOONfYjkIPKIRCIHI,12887
+atomicshop/basics/numbers.py,sha256=II-jP8MG0IYvmjT9_BLEai65xzDJ0LTHK2eFVSrgVqo,527
 atomicshop/basics/randoms.py,sha256=DmYLtnIhDK29tAQrGP1Nt-A-v8WC7WIEB8Edi-nk3N4,282
 atomicshop/basics/strings.py,sha256=uhl-5cqZ3qLK7M38lOEDSGCAkG3xmeLoEU3V0oCgMxU,14358
 atomicshop/basics/threads.py,sha256=xvgdDJdmgN0wmmARoZ-H7Kvl1GOcEbvgaeGL4M3Hcx8,2819
@@ -162,7 +163,7 @@ atomicshop/wrappers/loggingw/checks.py,sha256=AGFsTsLxHQd1yAraa5popqLaGO9VM0KpcP
 atomicshop/wrappers/loggingw/formatters.py,sha256=mUtcJJfmhLNrwUVYShXTmdu40dBaJu4TS8FiuTXI7ys,7189
 atomicshop/wrappers/loggingw/handlers.py,sha256=qm5Fbu8eDmlstMduUe5nKUlJU5IazFkSnQizz8Qt2os,5479
 atomicshop/wrappers/loggingw/loggers.py,sha256=DHOOTAtqkwn1xgvLHSkOiBm6yFGNuQy1kvbhG-TDog8,2374
-atomicshop/wrappers/loggingw/loggingw.py,sha256=30uaJOJhPJExYptEmwc5HWScI2RbnDlNgybOsxW7klc,11609
+atomicshop/wrappers/loggingw/loggingw.py,sha256=v9WAseZXB50LluT9rIUcRvvevg2nLVKPgz3dbGejfV0,12151
 atomicshop/wrappers/loggingw/reading.py,sha256=cHco0EG8bEB6fJ7JxkFdV_d8J-voYRBCUzS-qVqv4aU,3697
 atomicshop/wrappers/playwrightw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atomicshop/wrappers/playwrightw/_tryouts.py,sha256=l1BLkFsiIMNlgv7nfZd1XGEvXQkIQkIcg48__9OaC00,4920
@@ -192,8 +193,8 @@ atomicshop/wrappers/socketw/socket_server_tester.py,sha256=VfNthyBvgI5tL9v3Qprh4
 atomicshop/wrappers/socketw/socket_wrapper.py,sha256=aXBwlEIJhFT0-c4i8iNlFx2It9VpCEpsv--5Oqcpxao,11624
 atomicshop/wrappers/socketw/ssl_base.py,sha256=k4V3gwkbq10MvOH4btU4onLX2GNOsSfUAdcHmL1rpVE,2274
 atomicshop/wrappers/socketw/statistics_csv.py,sha256=t3dtDEfN47CfYVi0CW6Kc2QHTEeZVyYhc57IYYh5nmA,826
-atomicshop-2.5.16.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
-atomicshop-2.5.16.dist-info/METADATA,sha256=02N26vDhqG2L4xhGjWCiug9P7OXAs6WlvUbzN9q6h7k,10268
-atomicshop-2.5.16.dist-info/WHEEL,sha256=Xo9-1PvkuimrydujYJAjF7pCkriuXBpUPEjma1nZyJ0,92
-atomicshop-2.5.16.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
-atomicshop-2.5.16.dist-info/RECORD,,
+atomicshop-2.6.0.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
+atomicshop-2.6.0.dist-info/METADATA,sha256=hIocyrjd0H35Wh9NFhQ_OldHC9DtqBZfV2LVpZ7IQdo,10267
+atomicshop-2.6.0.dist-info/WHEEL,sha256=oiQVh_5PnQM0E3gPdiz09WCNmwiHDMaGer_elqB3coM,92
+atomicshop-2.6.0.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
+atomicshop-2.6.0.dist-info/RECORD,,

{atomicshop-2.5.16.dist-info → atomicshop-2.6.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: bdist_wheel (0.41.3)
+Generator: bdist_wheel (0.42.0)
 Root-Is-Purelib: true
 Tag: py3-none-any