atomicshop 2.12.16__py3-none-any.whl → 2.12.17__py3-none-any.whl

atomicshop/__init__.py

@@ -1,4 +1,4 @@
 """Atomic Basic functions and classes to make developer life easier"""
 
 __author__ = "Den Kras"
-__version__ = '2.12.16'
+__version__ = '2.12.17'

atomicshop/diff_check.py

@@ -4,7 +4,7 @@ from typing import Union, Literal
 import json
 import queue
 
-from . import filesystem, datetimes
+from . import filesystem, timer
 from .file_io import file_io, jsons
 from .print_api import print_api
 from .basics import list_of_dicts, dicts
@@ -29,7 +29,6 @@ class DiffChecker:
             self,
             check_object: any = None,
             check_object_display_name: str = None,
-            aggregation: bool = False,
             input_file_path: str = None,
             input_file_write_only: bool = True,
             return_first_cycle: bool = True,
@@ -38,11 +37,13 @@ class DiffChecker:
                 'hit_statistics',
                 'all_objects',
                 'single_object'] = None,
-            input_file_rotation_cycle_hours: Union[
+            hit_statistics_input_file_rotation_cycle_hours: Union[
                 float,
                 Literal['midnight'],
                 None] = None,
-            enable_statistics_queue: bool = False
+            hit_statistics_enable_queue: bool = False,
+            new_objects_hours_then_difference: float = None
+
     ):
         """
         :param check_object: any, object to check if it changed.
@@ -85,11 +86,11 @@ class DiffChecker:
             'single_object': will store the object as is, without any comparison. Meaning, that the object will be
                  compared only to itself, and if it changes, it will be updated.
             None: Nothing will be done, you will get an exception.
-        :param input_file_rotation_cycle_hours:
+        :param hit_statistics_input_file_rotation_cycle_hours:
             float, the amount of hours the input file will be rotated in the 'hit_statistics' operation type.
             str, (only 'midnight' is valid), the input file will be rotated daily at midnight.
             This is valid only for the 'hit_statistics' operation type.
-        :param enable_statistics_queue: boolean, if True, the statistics queue will be enabled for the 'hit_statistics'
+        :param hit_statistics_enable_queue: boolean, if True, the statistics queue will be enabled for the 'hit_statistics'
             operation type. You can use this queue to process the statistics in another thread.
 
             Example:
@@ -99,8 +100,8 @@ class DiffChecker:
                 input_file_write_only=True,
                 return_first_cycle=True,
                 operation_type='hit_statistics',
-                input_file_rotation_cycle_hours='midnight',
-                enable_statistics_queue=True)
+                hit_statistics_input_file_rotation_cycle_hours='midnight',
+                hit_statistics_enable_queue=True)
 
             def process_statistics_queue():
                 while True:
@@ -110,6 +111,12 @@ class DiffChecker:
             threading.Thread(target=process_statistics_queue).start()
 
             <... Your checking operation for the object ...>
+        :param new_objects_hours_then_difference: float, This is only for the 'new_objects' operation type.
+            If the object is not in the list of objects, it will be added to the list.
+            If the object is in the list of objects, it will be ignored.
+            After the specified amount of hours, new objects will not be added to the input file list, so each new
+            object will be outputted from the function. This is useful for checking new objects that are not
+            supposed to be in the list of objects, but you want to know about them.
 
         --------------------------------------------------
 
@@ -180,61 +187,87 @@ class DiffChecker:
             print(message)
         """
 
+        self.check_object = check_object
+        self.check_object_display_name = check_object_display_name
+        self.input_file_path: str = input_file_path
+        self.input_file_write_only: bool = input_file_write_only
+        self.return_first_cycle: bool = return_first_cycle
+        self.operation_type = operation_type
+        self.hit_statistics_input_file_rotation_cycle_hours = hit_statistics_input_file_rotation_cycle_hours
+        self.hit_statistics_enable_queue = hit_statistics_enable_queue
+        self.new_objects_hours_then_difference: float = new_objects_hours_then_difference
+
+        # Previous content.
+        self.previous_content: Union['list', 'str', None] = None
+        # The format the file will be saved as (not used as extension): txt, json.
+        self.save_as: str = str()
+
+        self.statistics_queue = None
+        self.previous_day = None
+        self.new_objects_seconds_then_difference: Union[float, None] = None
+        self.timer = None
+
+    def _static_pre_process(self):
+        """
+        This function will be called before the actual checking of the object.
+        If you change any attribute of the class, you will need to call this function again.
+        """
         # 'check_object' can be none, so checking if it not equals empty string.
-        if check_object == "":
+        if self.check_object == "":
             raise ValueError("[check_object] option can't be empty string.")
 
-        if operation_type and operation_type not in ['new_objects', 'hit_statistics', 'all_objects', 'single_object']:
+        if (self.operation_type and self.operation_type not in
+                ['new_objects', 'hit_statistics', 'all_objects', 'single_object']):
             raise ValueError(f"[operation_type] must be one of the following: "
                              f"'new_objects', 'hit_statistics', 'all_objects', 'single_object'.")
 
-        if input_file_rotation_cycle_hours and operation_type != 'hit_statistics':
+        if self.hit_statistics_input_file_rotation_cycle_hours and self.operation_type != 'hit_statistics':
             raise ValueError("[input_file_rotation_cycle] can be specified only for 'hit_statistics' operation type.")
 
-        if enable_statistics_queue and operation_type != 'hit_statistics':
-            raise ValueError("[enable_statistics_queue] can be specified only for 'hit_statistics' operation type.")
+        if self.hit_statistics_enable_queue and self.operation_type != 'hit_statistics':
+            raise ValueError("[hit_statistics_enable_queue] can be specified only for 'hit_statistics' operation type.")
 
-        if input_file_rotation_cycle_hours and not isinstance(input_file_rotation_cycle_hours, float) and \
-                not isinstance(input_file_rotation_cycle_hours, int) and \
-                input_file_rotation_cycle_hours != 'midnight':
+        if (self.hit_statistics_input_file_rotation_cycle_hours and
+                not isinstance(self.hit_statistics_input_file_rotation_cycle_hours, float) and
+                not isinstance(self.hit_statistics_input_file_rotation_cycle_hours, int) and
+                self.hit_statistics_input_file_rotation_cycle_hours != 'midnight'):
             raise ValueError("[input_file_rotation_cycle] must be float, int or 'midnight' str.")
 
-        self.check_object = check_object
-        self.check_object_display_name = check_object_display_name
-        self.aggregation: bool = aggregation
-        self.input_file_path: str = input_file_path
-        self.input_file_write_only: bool = input_file_write_only
-        self.return_first_cycle: bool = return_first_cycle
-        self.operation_type = operation_type
-        self.input_file_rotation_cycle = input_file_rotation_cycle_hours
-        self.enable_statistics_queue = enable_statistics_queue
+        if self.new_objects_hours_then_difference and self.operation_type != 'new_objects':
+            raise ValueError(
+                "[new_objects_hours_then_difference] can be specified only for 'new_objects' operation type.")
 
         if not self.check_object_display_name:
             self.check_object_display_name = self.check_object
 
-        # Previous content.
-        self.previous_content: Union['list', 'str', None] = None
-        # The format the file will be saved as (not used as extension): txt, json.
-        self.save_as: str = str()
-
         # If the input file rotation cycle is set and the statistics queue is enabled, we will create the queue.
-        if self.input_file_rotation_cycle and self.enable_statistics_queue:
+        if self.hit_statistics_input_file_rotation_cycle_hours and self.hit_statistics_enable_queue:
             # You can use this queue to process the statistics in another thread.
             self.statistics_queue = queue.Queue()
         else:
             self.statistics_queue = None
 
         # If the input file rotation cycle is set to midnight, we will store the previous day as today.
-        if self.input_file_rotation_cycle == 'midnight':
+        if self.hit_statistics_input_file_rotation_cycle_hours == 'midnight':
             self.previous_day = datetime.datetime.now().strftime('%d')
         else:
             self.previous_day = None
 
+        if self.new_objects_hours_then_difference and self.operation_type != 'new_objects':
+            raise ValueError("The 'new_objects_hours_then_difference' variable must be set for 'new_objects' operation type.")
+
+        if self.new_objects_hours_then_difference:
+            self.new_objects_seconds_then_difference = self.new_objects_hours_then_difference * 60 * 60
+            self.timer = timer.Timer()
+            self.timer.start()
+
     def check_string(self, print_kwargs: dict = None):
         """
         The function will check file content for change by hashing it and comparing the hash.
         """
 
+        self._static_pre_process()
+
         if not isinstance(self.check_object, str):
             raise TypeError(f"[check_object] must be string, not {type(self.check_object)}.")
 
@@ -254,6 +287,8 @@ class DiffChecker:
         :param print_kwargs: dict, of kwargs to pass to 'print_api' function.
         """
 
+        self._static_pre_process()
+
         if not isinstance(self.check_object, list):
             raise TypeError(f'[check_object] must be list, not {type(self.check_object)}.')
 
@@ -343,7 +378,7 @@ class DiffChecker:
         return result, message
 
     def _hit_statistics_only_handling(self, current_content, result, message, sort_by_keys, print_kwargs: dict = None):
-        if self.input_file_rotation_cycle == 'midnight':
+        if self.hit_statistics_input_file_rotation_cycle_hours == 'midnight':
             # If the current time is midnight, we will rotate the file.
             # Get current date.
             current_date = datetime.datetime.now().strftime('%d')
@@ -417,29 +452,40 @@ class DiffChecker:
                     'object': self.check_object_display_name,
                     'old': list(self.previous_content),
                     'updated': current_content
-                    # 'type': self.object_type
                 }
 
                 # f"Type: {result['type']} | "
                 message = f"Object: {result['object']} | Old: {result['old']} | Updated: {result['updated']}"
 
-            # Make known content the current, since it is updated.
-            self.previous_content.extend(current_content)
+            # If the time has passed, we will stop updating the input file.
+            if ((self.new_objects_seconds_then_difference and
+                self.new_objects_seconds_then_difference > self.timer.measure())
+                    or not self.new_objects_hours_then_difference):
 
-            # Sort list of dicts by specified list of keys.
-            if sort_by_keys:
-                self.previous_content = list_of_dicts.sort_by_keys(
-                    self.previous_content, sort_by_keys, case_insensitive=True)
+                result['time_passed'] = False
 
-            # If 'input_file_path' was specified by the user, it means that we will use the input file to save
-            # our known content there for next iterations to compare.
-            if self.input_file_path:
-                if self.save_as == 'txt':
-                    # noinspection PyTypeChecker
-                    file_io.write_file(self.previous_content, self.input_file_path, **(print_kwargs or {}))
-                elif self.save_as == 'json':
-                    jsons.write_json_file(
-                        self.previous_content, self.input_file_path, use_default_indent=True, **(print_kwargs or {}))
+                # Make known content the current, since it is updated.
+                self.previous_content.extend(current_content)
+
+                # Sort list of dicts by specified list of keys.
+                if sort_by_keys:
+                    self.previous_content = list_of_dicts.sort_by_keys(
+                        self.previous_content, sort_by_keys, case_insensitive=True)
+
+                # If 'input_file_path' was specified by the user, it means that we will use the input file to save
+                # our known content there for next iterations to compare.
+                if self.input_file_path:
+                    if self.save_as == 'txt':
+                        # noinspection PyTypeChecker
+                        file_io.write_file(self.previous_content, self.input_file_path, **(print_kwargs or {}))
+                    elif self.save_as == 'json':
+                        jsons.write_json_file(
+                            self.previous_content, self.input_file_path, use_default_indent=True, **(print_kwargs or {}))
+            else:
+                result['time_passed'] = True
+                # We will stop the timer, since the time has passed, the 'measure' method will return the last measure
+                # when the timer was running.
+                self.timer.stop()
         else:
             message = f"Object didn't change: {self.check_object_display_name}"
 

atomicshop/monitor/change_monitor.py

@@ -1,9 +1,22 @@
 from typing import Literal, Union
-import queue
+from pathlib import Path
 
 from .checks import dns, network, hash, process_running
 from .. import filesystem, scheduling
-from ..diff_check import DiffChecker
+
+
+DNS__DEFAULT_SETTINGS = {
+    'learning_mode_create_unique_entries_list': True,
+    'learning_hours': 24,                                   # 0 - the learning will never stop.
+    'alert_about_missing_entries_after_learning': False,
+    'alert_about_missing_entries_always': True,
+    'create_alert_statistics': True,
+    'statistics_rotation_hours': 'midnight'                  # 'midnight' or float of hours
+}
+
+FILE__URL__DEFAULT_SETTINGS = {
+    'store_original_object': False
+}
 
 
 class ChangeMonitor:
@@ -12,6 +25,11 @@ class ChangeMonitor:
     """
     def __init__(
             self,
+            check_object: any = None,
+            input_file_name: str = None,
+            input_statistics_file_name: str = None,
+            input_directory: str = None,
+            input_file_write_only: bool = True,
             object_type: Union[
                 Literal[
                     'file',
@@ -24,20 +42,7 @@ class ChangeMonitor:
                     'url_playwright_png',
                     'url_playwright_jpeg'],
                 None] = None,
-            check_object_list: list = None,
-            input_file_directory: str = None,
-            input_file_name: str = None,
-            generate_input_file_name: bool = False,
-            input_file_write_only: bool = True,
-            store_original_object: bool = False,
-            operation_type: Literal[
-                'hit_statistics',
-                'all_objects'] = None,
-            input_file_rotation_cycle_hours: Union[
-                float,
-                Literal['midnight'],
-                None] = None,
-            enable_statistics_queue: bool = False
+            object_type_settings: dict = None
     ):
         """
         :param object_type: string, type of object to check. The type must be one of the following:
@@ -57,10 +62,15 @@ class ChangeMonitor:
                 be downloaded using 'playwright' library in PNG.
             'url_playwright_jpeg': 'check_object_list' must contain strings of full URL to a web page. The page will
                 be downloaded using 'playwright' library in JPEG.
-        :param check_object_list: list of objects to check if it changed. The list can contain as many objects as
-            needed and can contain only one object.
-            The list can be left empty if the object type is 'dns', 'network_sockets'.
-        :param input_file_directory: string, full directory path for storing input files for current state of objects,
+        :param object_type_settings: dict, specific settings for the object type.
+            'dns': Check the default settings example in 'DNS__DEFAULT_SETTINGS'.
+        :param check_object: The object to check if changed.
+            'dns': empty.
+            'network': empty.
+            'process_running': list of strings, process names to check if they are running.
+            'file': string, full path to the file.
+            'url_*': string, full URL to a web page.
+        :param input_directory: string, full directory path for storing input files for current state of objects,
             to check later if this state isn't updated. If this variable is left empty, all the content will be saved
             in memory and input file will not be used.
             If the file is not specified, the update of an object will be checked
@@ -77,116 +87,71 @@ class ChangeMonitor:
             False: write to input file each time there is an update, and read each check cycle from the file and not
                 from the memory.
         :param store_original_object: boolean, if True, the original object will be stored on the disk inside
-        'Original' folder, inside 'input_file_directory'.
+        'Original' folder, inside 'input_directory'.
         :param operation_type: string, type of operation to perform. The type must be one of the following:
             'hit_statistics': will only store the statistics of the entries in the input file.
             'all_objects': disable the DiffChecker features, meaning any new entries will be emitted as is.
             None: will use the default operation type, based on the object type.
-        :param input_file_rotation_cycle_hours:
+        :param hit_statistics_input_file_rotation_cycle_hours:
             float, the amount of hours the input file will be rotated in the 'hit_statistics' operation type.
             str, (only 'midnight' is valid), the input file will be rotated daily at midnight.
             This is valid only for the 'hit_statistics' operation type.
-        :param enable_statistics_queue: boolean, if True, the statistics queue will be enabled.
-
-        If 'input_file_directory' is not specified, the 'input_file_name' is not specified, and
+        :param hit_statistics_enable_queue: boolean, if True, the statistics queue will be enabled.
+        :param new_objects_hours_then_difference: float, currently works only for the 'dns' object_type.
+            This is only for the 'new_objects' operation type.
+            If the object is not in the list of objects, it will be added to the list.
+            If the object is in the list of objects, it will be ignored.
+            After the specified amount of hours, new objects will not be added to the input file list, so each new
+            object will be outputted from the function. This is useful for checking new objects that are not
+            supposed to be in the list of objects, but you want to know about them.
+
+        If 'input_directory' is not specified, the 'input_file_name' is not specified, and
         'generate_input_file_name' is False, then the input file will not be used and the object will be stored
         in memory. This means that the object will be checked only during the time that the script is running.
         """
 
-        # =================== Exception section ============================
-        if not input_file_directory and store_original_object:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [store_original_object] is True.')
-
-        if not input_file_directory and generate_input_file_name:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [generate_input_file_name] is True.')
-
-        if not input_file_directory and input_file_name:
-            raise ValueError('ERROR: [input_file_directory] must be specified if [input_file_name] is specified.')
-
-        if input_file_name and generate_input_file_name:
-            raise ValueError(
-                'ERROR: [input_file_name] and [generate_input_file_name] cannot be both specified and True.')
-
-        if operation_type:
-            if operation_type not in ['hit_statistics', 'all_objects']:
-                raise ValueError(
-                    'ERROR: [operation_type] must be one of the following: "hit_statistics", "all_objects".')
-
-        if input_file_rotation_cycle_hours and operation_type != 'hit_statistics':
-            raise ValueError("[input_file_rotation_cycle] can be specified only for 'hit_statistics' operation type.")
-
-        # === EOF Exception section ========================================
         # === Initialize Main variables ====================================
 
-        if not check_object_list:
-            check_object_list = list()
-
-        self.object_type = object_type
-        self.check_object_list: list = check_object_list
-        self.input_file_directory: str = input_file_directory
+        self.check_object: any = check_object
         self.input_file_name: str = input_file_name
-        self.generate_input_file_name: bool = generate_input_file_name
+        self.input_statistics_file_name: str = input_statistics_file_name
+        self.input_directory: str = input_directory
         self.input_file_write_only: bool = input_file_write_only
-        self.store_original_object: bool = store_original_object
-        self.operation_type = operation_type
-        self.input_file_rotation_cycle_hours = input_file_rotation_cycle_hours
-        self.enable_statistics_queue = enable_statistics_queue
-
-        # === EOF Initialize Main variables ================================
-        # === Initialize Secondary variables ===============================
-
-        # The 'diff_check' will store the list of DiffChecker classes.
-        self.diff_check_list: list = list()
-
-        # If 'check_object_list' is a list, loop through it and create a DiffChecker object for each object.
-        if self.check_object_list:
-            for index in range(len(self.check_object_list)):
-                self.diff_check_list.append(
-                    DiffChecker(
-                        input_file_write_only=self.input_file_write_only,
-                        operation_type=self.operation_type,
-                        input_file_rotation_cycle_hours=self.input_file_rotation_cycle_hours,
-                        enable_statistics_queue=self.enable_statistics_queue
-                    )
-                )
-        # Else, if 'check_object_list' is None, create a DiffChecker object only once.
-        else:
-            self.diff_check_list.append(
-                DiffChecker(
-                    input_file_write_only=self.input_file_write_only,
-                    operation_type=self.operation_type,
-                    input_file_rotation_cycle_hours=self.input_file_rotation_cycle_hours,
-                    enable_statistics_queue=self.enable_statistics_queue
-                )
-            )
-
-        self.input_file_path = None
-        self.original_object_directory = None
-        self.original_object_file_path = None
-
-        # If 'store_original_object' is True, create directory for original object.
-        if self.store_original_object:
-            # Make path for original object.
-            self.original_object_directory = filesystem.add_object_to_path(
-                self.input_file_directory, 'Original')
-            # Create directory if it doesn't exist.
-            filesystem.create_directory(self.original_object_directory)
-
-        self.first_cycle = True
+        self.object_type = object_type
+        self.object_type_settings: dict = object_type_settings
+
+        # === Additional variables ========================================
+
+        # self.original_object_directory = None
+        # self.original_object_file_path = None
 
+        # # If 'store_original_object' is True, create directory for original object.
+        # if self.store_original_object:
+        #     # Make path for original object.
+        #     self.original_object_directory = filesystem.add_object_to_path(
+        #         self.input_directory, 'Original')
+        #     # Create directory if it doesn't exist.
+        #     filesystem.create_directory(self.original_object_directory)
+        #
         # Initialize objects for DNS and Network monitoring.
-        self.fetch_engine = None
-        self.thread_looper = scheduling.ThreadLooper()
 
-    def _set_input_file_path(self, check_object_index: int = 0):
-        if self.first_cycle:
-            # If 'input_file_directory' and 'input_file_name' are specified, we'll use a filename to store.
-            if self.input_file_directory and self.input_file_name:
-                self.input_file_path = filesystem.add_object_to_path(
-                    self.input_file_directory, self.input_file_name)
+        self.checks_instance = None
+        self._setup_object()
+
+    def _setup_object(self):
+        if self.object_type == 'file' or 'url_' in self.object_type:
+            self.checks_instance = hash
+        if self.object_type == 'dns':
+            self.checks_instance = dns
+
+            if not self.object_type_settings:
+                self.object_type_settings = DNS__DEFAULT_SETTINGS
+        elif self.object_type == 'network':
+            self.checks_instance = network
+        elif self.object_type == 'process_running':
+            self.checks_instance = process_running
 
-            # Set the input file path.
-            self.diff_check_list[check_object_index].input_file_path = self.input_file_path
+        self.checks_instance.setup_check(self)
 
     def check_cycle(self, print_kwargs: dict = None):
         """
@@ -195,28 +160,6 @@ class ChangeMonitor:
         :return: None
         """
 
-        return_list = list()
-
-        if (
-                self.object_type == 'file' or
-                'url_' in self.object_type):
-            return_list = hash._execute_cycle(self, print_kwargs=print_kwargs)
-        if self.object_type == 'dns':
-            return_list = dns._execute_cycle(self, print_kwargs=print_kwargs)
-        elif self.object_type == 'network':
-            return_list = network._execute_cycle(self, print_kwargs=print_kwargs)
-        elif self.object_type == 'process_running':
-            return_list = process_running._execute_cycle(self, print_kwargs=print_kwargs)
-
-        # Set 'first_cycle' to False, since the first cycle is finished.
-        if self.first_cycle:
-            self.first_cycle = False
+        return_list = self.checks_instance.execute_cycle(print_kwargs=print_kwargs)
 
         return return_list
-
-    def run_loop(self, interval_seconds=0, print_kwargs: dict = None):
-        self.thread_looper.run_loop(
-            self.check_cycle, kwargs={'print_kwargs': print_kwargs}, interval_seconds=interval_seconds)
-
-    def emit_from_loop(self):
-        return self.thread_looper.emit_from_loop()