atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

atomicshop/datetimes.py

@@ -2,6 +2,8 @@ import datetime
 from datetime import timedelta
 import time
 import random
+import re
+from typing import Union
 
 
 class MonthToNumber:
@@ -46,6 +48,102 @@ class MonthToNumber:
         'דצמבר': '12'}
 
 
+# Mapping of datetime format specifiers to regex patterns
+DATE_TIME_STRING_FORMAT_SPECIFIERS_TO_REGEX: dict = {
+    '%Y': r'\d{4}',   # Year with century
+    '%m': r'\d{2}',   # Month as a zero-padded decimal number
+    '%d': r'\d{2}',   # Day of the month as a zero-padded decimal number
+    '%H': r'\d{2}',   # Hour (24-hour clock) as a zero-padded decimal number
+    '%I': r'\d{2}',   # Hour (12-hour clock) as a zero-padded decimal number
+    '%M': r'\d{2}',   # Minute as a zero-padded decimal number
+    '%S': r'\d{2}',   # Second as a zero-padded decimal number
+    '%f': r'\d{6}',   # Microsecond as a decimal number, zero-padded on the left
+    '%j': r'\d{3}',   # Day of the year as a zero-padded decimal number
+    '%U': r'\d{2}',   # Week number of the year (Sunday as the first day of the week)
+    '%W': r'\d{2}',   # Week number of the year (Monday as the first day of the week)
+    '%w': r'\d',      # Weekday as a decimal number (0 = Sunday, 6 = Saturday)
+    '%y': r'\d{2}',   # Year without century
+    '%p': r'(AM|PM)',   # AM or PM
+    '%z': r'[+-]\d{4}',  # UTC offset in the form ±HHMM
+    '%Z': r'[A-Z]+',     # Time zone name
+    '%%': r'%'           # Literal '%'
+}
+
+
+def get_datetime_from_complex_string_by_pattern(
+        complex_string: str,
+        date_pattern: str
+) -> tuple[
+    Union[datetime.datetime, None],
+    Union[str, None],
+    Union[float, None]
+]:
+    """
+    Function will get datetime object from a complex string by pattern.
+
+    :param complex_string: string that contains date and time.
+    :param date_pattern: pattern that will be used to extract date and time from the string.
+    :return: tuple(datetime object, date string, timestamp float)
+    """
+
+    # Convert the date pattern to regex pattern
+    regex_pattern = re.sub(r'%[a-zA-Z]', r'\\d+', date_pattern)
+
+    # Find the date part in the file name using the regex pattern
+    date_str = re.search(regex_pattern, complex_string)
+
+    if date_str:
+        # Convert the date string to a datetime object based on the given pattern
+        date_obj = datetime.datetime.strptime(date_str.group(), date_pattern)
+        date_timestamp = date_obj.timestamp()
+        return date_obj, date_str.group(), date_timestamp
+    else:
+        return None, None, None
+
+
+def datetime_format_to_regex(format_str: str) -> str:
+    """
+    Convert a datetime format string to a regex pattern.
+
+    :param format_str: The datetime format string to convert.
+    :return: The regex pattern that matches the format string.
+
+    Example:
+    datetime_format_to_regex("%Y-%m-%d")
+
+    Output:
+    '^\\d{4}-\\d{2}-\\d{2}$'
+    """
+
+    # Escape all non-format characters
+    escaped_format_str = re.escape(format_str)
+
+    # Replace escaped format specifiers with their regex equivalents
+    for specifier, regex in DATE_TIME_STRING_FORMAT_SPECIFIERS_TO_REGEX.items():
+        escaped_format_str = escaped_format_str.replace(re.escape(specifier), regex)
+
+    # Return the full regex pattern with start and end anchors
+    return f"^{escaped_format_str}$"
+
+
+def extract_datetime_format_from_string(complex_string: str) -> Union[str, None]:
+    """
+    Extract the datetime format from the suffix used in TimedRotatingFileHandler.
+
+    Args:
+    - suffix: The suffix string from the handler.
+
+    Returns:
+    - The datetime format string, or None if it cannot be determined.
+    """
+    # Regular expression to match datetime format components in the suffix
+    datetime_format_regex = r"%[a-zA-Z]"
+    matches = re.findall(datetime_format_regex, complex_string)
+    if matches:
+        return "".join(matches)
+    return None
+
+
 def convert_single_digit_to_zero_padded(string: str):
     """
     Function will check if string is a single character digit and will add zero in front of it.

atomicshop/diff_check.py

@@ -1,9 +1,13 @@
+import datetime
 from pathlib import Path
-from typing import Union
+from typing import Union, Literal
+import json
+import queue
 
+from . import filesystem, timer
 from .file_io import file_io, jsons
 from .print_api import print_api
-from .basics import list_of_dicts
+from .basics import list_of_dicts, dicts
 
 
 class DiffChecker:
@@ -25,10 +29,21 @@ class DiffChecker:
             self,
             check_object: any = None,
             check_object_display_name: str = None,
-            aggregation: bool = False,
             input_file_path: str = None,
             input_file_write_only: bool = True,
-            return_first_cycle: bool = True
+            return_first_cycle: bool = True,
+            operation_type: Literal[
+                'new_objects',
+                'hit_statistics',
+                'all_objects',
+                'single_object'] = None,
+            hit_statistics_input_file_rotation_cycle_hours: Union[
+                float,
+                Literal['midnight'],
+                None] = None,
+            hit_statistics_enable_queue: bool = False,
+            new_objects_hours_then_difference: float = None
+
     ):
         """
         :param check_object: any, object to check if it changed.
@@ -38,10 +53,10 @@ class DiffChecker:
             The object is 'None' by default, since there are objects that are needed to be provided in the
             function input for that object. So, not always you know what your object type during class initialization.
         :param check_object_display_name: string, name of the object to display in the message.
-            If not specified, the 'check_object' will be displayed.
-        :param aggregation: boolean, if True, the object will be aggregated with other objects in the list of objects.
-            Meaning, that the object will be checked against the existing objects in the list, and if it is not
-            in the list, it will be added to the list. If it is in the list, it will be ignored.
+            If not specified, the provided 'check_object' will be displayed.
+        #:param aggregation: boolean, if True, the object will be aggregated with other objects in the list of objects.
+        #    Meaning, that the object will be checked against the existing objects in the list, and if it is not
+        #    in the list, it will be added to the list. If it is in the list, it will be ignored.
         :param input_file_path: string, full file path for storing input file for current state of objects,
             to check later if this state isn't updated. If this variable is left empty, all the content will be saved
             in memory and input file will not be used.
@@ -62,27 +77,193 @@ class DiffChecker:
 
             True: return updated dictionary on first cycle. This is the default.
             False: don't return updated dictionary on first cycle.
+        :param operation_type: string, type of operation to perform. The type must be one of the following:
+            'new_objects': will only store the new objects in the input file.
+            'hit_statistics': will only store the statistics of the entries in the input file.
+                The file will be rotated after the specified time in the 'input_file_rotation_cycle' variable, if
+                it is specified.
+            'all_objects': disable the DiffChecker features, meaning any new entries will be emitted as is.
+            'single_object': will store the object as is, without any comparison. Meaning, that the object will be
+                 compared only to itself, and if it changes, it will be updated.
+            None: Nothing will be done, you will get an exception.
+        :param hit_statistics_input_file_rotation_cycle_hours:
+            float, the amount of hours the input file will be rotated in the 'hit_statistics' operation type.
+            str, (only 'midnight' is valid), the input file will be rotated daily at midnight.
+            This is valid only for the 'hit_statistics' operation type.
+        :param hit_statistics_enable_queue: boolean, if True, the statistics queue will be enabled for the 'hit_statistics'
+            operation type. You can use this queue to process the statistics in another thread.
+
+            Example:
+            diff_checker = DiffChecker(
+                check_object_display_name='List of Dicts',
+                input_file_path='D:\\input\\list_of_dicts.json',
+                input_file_write_only=True,
+                return_first_cycle=True,
+                operation_type='hit_statistics',
+                hit_statistics_input_file_rotation_cycle_hours='midnight',
+                hit_statistics_enable_queue=True)
+
+            def process_statistics_queue():
+                while True:
+                    statistics = diff_checker.statistics_queue.get()
+                    print(statistics)
+
+            thread = threading.Thread(target=process_statistics_queue)
+            thread.daemon = True
+            thread.start()
+
+            <... Your checking operation for the object ...>
+        :param new_objects_hours_then_difference: float, This is only for the 'new_objects' operation type.
+            If the object is not in the list of objects, it will be added to the list.
+            If the object is in the list of objects, it will be ignored.
+            After the specified amount of hours, new objects will not be added to the input file list, so each new
+            object will be outputted from the function. This is useful for checking new objects that are not
+            supposed to be in the list of objects, but you want to know about them.
+
+        --------------------------------------------------
+
+        Working example:
+        from atomicshop import diff_check
+
+
+        # Example of checking list of dicts.
+        check_list_of_dicts = [
+            {'name': 'John', 'age': 25},
+            {'name': 'Alice', 'age': 30}
+        ]
+
+        diff_checker = diff_check.DiffChecker(
+            check_object=check_list_of_dicts,
+            check_object_display_name='List of Dicts',
+            operation_type='new_objects'
+            input_file_path='D:\\input\\list_of_dicts.json',
+            input_file_write_only=True,
+            return_first_cycle=True
+        )
+
+        result, message = diff_checker.check_list_of_dicts(
+            sort_by_keys=['name']
+        )
+
+        # If result is not None, it means that the object was updated.
+        if result:
+            print(message)
+
+        --------------------------------------------------
+
+        Working example when you need to aggregate a list of dicts, meaning only new entries will be added to the list:
+        from atomicshop import diff_check
+
+
+        diff_checker = diff_check.DiffChecker(
+            check_object_display_name='List of Dicts',
+            input_file_path='D:\\input\\list_of_dicts.json',
+            input_file_write_only=True,
+            return_first_cycle=True,
+            operation_type='new_objects'
+        )
+
+        # Example of checking list of dicts.
+        check_list_of_dicts = [
+            {'name': 'John', 'age': 25},
+            {'name': 'Alice', 'age': 30}
+        ]
+
+        diff_checker.check_object = check_list_of_dicts
+        result, message = diff_checker.check_list_of_dicts()
+
+        # If result is not None, it means that the object was updated.
+        if result:
+            print(message)
+
+
+        check_list_of_dicts = [
+            {'name': 'John', 'age': 25},
+            {'name': 'Jessie', 'age': 50}
+        ]
+
+        diff_checker.check_object = check_list_of_dicts
+        result, message = diff_checker.check_list_of_dicts()
+
+        if result:
+            print(message)
         """
 
-        # 'check_object' can be none, so checking if it not equals empty string.
-        if check_object == "":
-            raise ValueError("[check_object] option can't be empty string.")
-
         self.check_object = check_object
         self.check_object_display_name = check_object_display_name
-        self.aggregation: bool = aggregation
         self.input_file_path: str = input_file_path
         self.input_file_write_only: bool = input_file_write_only
         self.return_first_cycle: bool = return_first_cycle
-
-        if not self.check_object_display_name:
-            self.check_object_display_name = self.check_object
+        self.operation_type = operation_type
+        self.hit_statistics_input_file_rotation_cycle_hours = hit_statistics_input_file_rotation_cycle_hours
+        self.hit_statistics_enable_queue = hit_statistics_enable_queue
+        self.new_objects_hours_then_difference: float = new_objects_hours_then_difference
 
         # Previous content.
         self.previous_content: Union['list', 'str', None] = None
         # The format the file will be saved as (not used as extension): txt, json.
         self.save_as: str = str()
 
+        self.statistics_queue = None
+        self.previous_day = None
+        self.new_objects_seconds_then_difference: Union[float, None] = None
+        self.timer = None
+
+    def initiate_before_action(self):
+        """
+        This function will be called before the actual checking of the object.
+        If you change any attribute of the class, you will need to call this function again.
+        """
+        # 'check_object' can be none, so checking if it not equals empty string.
+        if self.check_object == "":
+            raise ValueError("[check_object] option can't be empty string.")
+
+        if (self.operation_type and self.operation_type not in
+                ['new_objects', 'hit_statistics', 'all_objects', 'single_object']):
+            raise ValueError(f"[operation_type] must be one of the following: "
+                             f"'new_objects', 'hit_statistics', 'all_objects', 'single_object'.")
+
+        if self.hit_statistics_input_file_rotation_cycle_hours and self.operation_type != 'hit_statistics':
+            raise ValueError("[input_file_rotation_cycle] can be specified only for 'hit_statistics' operation type.")
+
+        if self.hit_statistics_enable_queue and self.operation_type != 'hit_statistics':
+            raise ValueError("[hit_statistics_enable_queue] can be specified only for 'hit_statistics' operation type.")
+
+        if (self.hit_statistics_input_file_rotation_cycle_hours and
+                not isinstance(self.hit_statistics_input_file_rotation_cycle_hours, float) and
+                not isinstance(self.hit_statistics_input_file_rotation_cycle_hours, int) and
+                self.hit_statistics_input_file_rotation_cycle_hours != 'midnight'):
+            raise ValueError("[input_file_rotation_cycle] must be float, int or 'midnight' str.")
+
+        if self.new_objects_hours_then_difference and self.operation_type != 'new_objects':
+            raise ValueError(
+                "[new_objects_hours_then_difference] can be specified only for 'new_objects' operation type.")
+
+        if not self.check_object_display_name:
+            self.check_object_display_name = self.check_object
+
+        # If the input file rotation cycle is set and the statistics queue is enabled, we will create the queue.
+        if self.hit_statistics_input_file_rotation_cycle_hours and self.hit_statistics_enable_queue:
+            # You can use this queue to process the statistics in another thread.
+            self.statistics_queue = queue.Queue()
+        else:
+            self.statistics_queue = None
+
+        # If the input file rotation cycle is set to midnight, we will store the previous day as today.
+        if self.hit_statistics_input_file_rotation_cycle_hours == 'midnight':
+            self.previous_day = datetime.datetime.now().strftime('%d')
+        else:
+            self.previous_day = None
+
+        if self.new_objects_hours_then_difference and self.operation_type != 'new_objects':
+            raise ValueError(
+                "The 'new_objects_hours_then_difference' variable must be set for 'new_objects' operation type.")
+
+        if self.new_objects_hours_then_difference:
+            self.new_objects_seconds_then_difference = self.new_objects_hours_then_difference * 60 * 60
+            self.timer = timer.Timer()
+            self.timer.start()
+
     def check_string(self, print_kwargs: dict = None):
         """
         The function will check file content for change by hashing it and comparing the hash.
@@ -119,6 +300,11 @@ class DiffChecker:
         return self._handle_input_file(sort_by_keys, print_kwargs=print_kwargs)
 
     def _handle_input_file(self, sort_by_keys=None, print_kwargs: dict = None):
+        # This point is the first one that is shared between the processing functions, so now we can check
+        # if the 'operation_type' is set.
+        if not self.operation_type:
+            raise ValueError("[operation_type] must be specified.")
+
         # If 'input_file_path' was specified, this means that the input file will be created for storing
         # content of the function to compare.
         if self.input_file_path:
@@ -130,14 +316,15 @@ class DiffChecker:
                 try:
                     if self.save_as == 'txt':
                         self.previous_content = file_io.read_file(
-                            self.input_file_path, stderr=False, **print_kwargs)
+                            self.input_file_path, stdout=False, stderr=False, **(print_kwargs or {}))
                     elif self.save_as == 'json':
                         self.previous_content = jsons.read_json_file(
-                            self.input_file_path, stderr=False, **print_kwargs)
+                            self.input_file_path, stdout=False, stderr=False, **(print_kwargs or {}))
                 except FileNotFoundError as except_object:
                     message = f"Input File [{Path(except_object.filename).name}] doesn't exist - Will create new one."
-                    print_api(message, color='yellow', **print_kwargs)
-                    pass
+                    print_api(message, color='yellow', **(print_kwargs or {}))
+                    if not self.input_file_write_only:
+                        self.previous_content = list()
 
             # get the content of current function.
             if isinstance(self.check_object, list):
@@ -149,10 +336,112 @@ class DiffChecker:
         result = None
         message = f'First Cycle on Object: {self.check_object_display_name}'
 
-        if self.aggregation:
-            return self._aggregation_handling(current_content, result, message, sort_by_keys=sort_by_keys, print_kwargs=print_kwargs)
+        if self.operation_type == 'all_objects':
+            return self._no_diffcheck_handling(
+                current_content, result, message, print_kwargs=print_kwargs)
+
+        if self.operation_type == 'hit_statistics':
+            return self._hit_statistics_only_handling(
+                current_content, result, message, sort_by_keys, print_kwargs=print_kwargs)
+
+        if self.operation_type == 'new_objects':
+            return self._aggregation_handling(
+                current_content, result, message, sort_by_keys=sort_by_keys, print_kwargs=print_kwargs)
+
+        if self.operation_type == 'single_object':
+            return self._singular_object_handling(current_content, result, message, print_kwargs=print_kwargs)
+
+    def _no_diffcheck_handling(self, current_content, result, message, print_kwargs: dict = None):
+        # if not self.previous_content:
+        #     self.previous_content = []
+
+        self.previous_content.append(f"{datetime.datetime.now()},{current_content}")
+
+        result = {
+            'object': self.check_object_display_name,
+            'entry': current_content
+        }
+
+        message = f"Object: {result['object']} | Entry: {result['entry']}"
+
+        # If 'input_file_path' was specified by the user, it means that we will use the input file to save
+        # our known content there for next iterations to compare.
+        if self.input_file_path:
+            if self.save_as == 'txt':
+                # noinspection PyTypeChecker
+                file_io.write_file(self.previous_content, self.input_file_path, **(print_kwargs or {}))
+            elif self.save_as == 'json':
+                jsons.write_json_file(
+                    self.previous_content, self.input_file_path, use_default_indent=True, **(print_kwargs or {}))
+
+        return result, message
+
+    def _hit_statistics_only_handling(self, current_content, result, message, sort_by_keys, print_kwargs: dict = None):
+        if self.hit_statistics_input_file_rotation_cycle_hours == 'midnight':
+            # If the current time is midnight, we will rotate the file.
+            # Get current date.
+            current_date = datetime.datetime.now().strftime('%d')
+            # If current date is different from previous date it means it is a new day, rotate the file.
+            if current_date != self.previous_day:
+                input_file_statistics = None
+                try:
+                    # Read the latest statistics from the input file.
+                    input_file_statistics = jsons.read_json_file(self.input_file_path)
+                # Basically, this means that the input statistics file doesn't exist yet, and no events hit
+                # yet and new day has come, so it doesn't matter, since there are no statistics to rotate the file.
+                except FileNotFoundError:
+                    pass
+
+                if input_file_statistics:
+                    # Rename the file.
+                    filesystem.backup_file(
+                        self.input_file_path, str(Path(self.input_file_path).parent), timestamp_as_prefix=False)
+                    # Update the previous date.
+                    self.previous_day = current_date
+
+                    previous_day_date_object = (datetime.datetime.now() - datetime.timedelta(days=1)).date()
+                    # Put the statistics in the queue to be processed.
+                    if self.statistics_queue:
+                        self.statistics_queue.put((input_file_statistics, previous_day_date_object))
+
+                    self.previous_content = {}
         else:
-            return self._non_aggregation_handling(current_content, result, message, print_kwargs=print_kwargs)
+            raise NotImplementedError("This feature is not implemented yet.")
+
+        # Convert the dictionary entry to string, since we will use it as a key in the dictionary.
+        current_entry = json.dumps(current_content[0])
+
+        if not self.previous_content:
+            self.previous_content = {}
+
+        if not self.previous_content.get(current_entry):
+            self.previous_content[current_entry] = 1
+        else:
+            self.previous_content[current_entry] += 1
+
+        result = {
+            'object': self.check_object_display_name,
+            'entry': current_entry,
+            'count': self.previous_content[current_entry]
+        }
+
+        message = f"Object: {result['object']} | Entry: {result['entry']} | Count: {result['count']}"
+
+        # Sort the dictionary by count of entries.
+        self.previous_content = dicts.sort_by_values(self.previous_content, reverse=True)
+
+        # If 'input_file_path' was specified by the user, it means that we will use the input file to save
+        # our known content there for next iterations to compare.
+        if self.input_file_path:
+            if self.save_as == 'txt':
+                # noinspection PyTypeChecker
+                file_io.write_file(self.previous_content, self.input_file_path, stdout=False, **(print_kwargs or {}))
+            elif self.save_as == 'json':
+                jsons.write_json_file(
+                    self.previous_content, self.input_file_path, use_default_indent=True, stdout=False,
+                    **(print_kwargs or {}))
+
+        return result, message
 
     def _aggregation_handling(self, current_content, result, message, sort_by_keys, print_kwargs: dict = None):
         if current_content[0] not in self.previous_content:
@@ -163,35 +452,46 @@ class DiffChecker:
                     'object': self.check_object_display_name,
                     'old': list(self.previous_content),
                     'updated': current_content
-                    # 'type': self.object_type
                 }
 
                 # f"Type: {result['type']} | "
                 message = f"Object: {result['object']} | Old: {result['old']} | Updated: {result['updated']}"
 
-            # Make known content the current, since it is updated.
-            self.previous_content.extend(current_content)
+            # If the time has passed, we will stop updating the input file.
+            if ((self.new_objects_seconds_then_difference and
+                self.new_objects_seconds_then_difference > self.timer.measure())
+                    or not self.new_objects_hours_then_difference):
 
-            # Sort list of dicts by specified list of keys.
-            if sort_by_keys:
-                self.previous_content = list_of_dicts.sort_by_keys(
-                    self.previous_content, sort_by_keys, case_insensitive=True)
+                result['time_passed'] = False
 
-            # If 'input_file_path' was specified by the user, it means that we will use the input file to save
-            # our known content there for next iterations to compare.
-            if self.input_file_path:
-                if self.save_as == 'txt':
-                    # noinspection PyTypeChecker
-                    file_io.write_file(self.previous_content, self.input_file_path, **print_kwargs)
-                elif self.save_as == 'json':
-                    jsons.write_json_file(
-                        self.previous_content, self.input_file_path, use_default_indent=True, **print_kwargs)
+                # Make known content the current, since it is updated.
+                self.previous_content.extend(current_content)
+
+                # Sort list of dicts by specified list of keys.
+                if sort_by_keys:
+                    self.previous_content = list_of_dicts.sort_by_keys(
+                        self.previous_content, sort_by_keys, case_insensitive=True)
+
+                # If 'input_file_path' was specified by the user, it means that we will use the input file to save
+                # our known content there for next iterations to compare.
+                if self.input_file_path:
+                    if self.save_as == 'txt':
+                        # noinspection PyTypeChecker
+                        file_io.write_file(self.previous_content, self.input_file_path, **(print_kwargs or {}))
+                    elif self.save_as == 'json':
+                        jsons.write_json_file(
+                            self.previous_content, self.input_file_path, use_default_indent=True, **(print_kwargs or {}))
+            else:
+                result['time_passed'] = True
+                # We will stop the timer, since the time has passed, the 'measure' method will return the last measure
+                # when the timer was running.
+                self.timer.stop()
         else:
             message = f"Object didn't change: {self.check_object_display_name}"
 
         return result, message
 
-    def _non_aggregation_handling(self, current_content, result, message, print_kwargs):
+    def _singular_object_handling(self, current_content, result, message, print_kwargs):
         if self.previous_content != current_content:
             # If known content is not empty (if it is, it means it is the first iteration, and we don't have the input
             # file, so we don't need to update the 'result', since there is nothing to compare yet).