sharedkernel 1.6.1__tar.gz → 1.6.3__tar.gz

{sharedkernel-1.6.1 → sharedkernel-1.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sharedkernel
-Version: 1.6.1
+Version: 1.6.3
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -20,6 +20,10 @@ Requires-Dist: persiantools
 this a shared kernel package
 
 # Change Log
+### Version 1.6.3
+- Fix minor bug in phone normalizer
+### Version 1.6.2
+- Minor update: normalize function name
 ### Version 1.6.1
 - Minor update normalize functions
 ### Version 1.6

{sharedkernel-1.6.1 → sharedkernel-1.6.3}/README.md

@@ -2,6 +2,10 @@
 this a shared kernel package
 
 # Change Log
+### Version 1.6.3
+- Fix minor bug in phone normalizer
+### Version 1.6.2
+- Minor update: normalize function name
 ### Version 1.6.1
 - Minor update normalize functions
 ### Version 1.6

{sharedkernel-1.6.1 → sharedkernel-1.6.3}/setup.py

@@ -33,7 +33,7 @@ setup(
         "persiantools"
     ],
     # *strongly* suggested for sharing
-    version="1.6.1",
+    version="1.6.3",
     description="sharekernel is a shared package between all python projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

{sharedkernel-1.6.1 → sharedkernel-1.6.3}/sharedkernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: sharedkernel
-Version: 1.6.1
+Version: 1.6.3
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -20,6 +20,10 @@ Requires-Dist: persiantools
 this a shared kernel package
 
 # Change Log
+### Version 1.6.3
+- Fix minor bug in phone normalizer
+### Version 1.6.2
+- Minor update: normalize function name
 ### Version 1.6.1
 - Minor update normalize functions
 ### Version 1.6

{sharedkernel-1.6.1 → sharedkernel-1.6.3}/sharedkernel.egg-info/SOURCES.txt

@@ -3,7 +3,6 @@ setup.py
 sharedkernel/common.py
 sharedkernel/date_converter.py
 sharedkernel/jwt_service.py
-sharedkernel/normalizer.py
 sharedkernel/regex_masking.py
 sharedkernel/string_extentions.py
 sharedkernel.egg-info/PKG-INFO

sharedkernel-1.6.1/sharedkernel/normalizer.py

@@ -1,498 +0,0 @@
-import re
-import itertools
-
-"""
-    Speech-to-Text Processing Systems for Iranian Dialects:
-
-    Speech-to-text processing systems often struggle to accurately transcribe phone numbers spoken by Iranians. 
-    As a result, phone numbers and sequential digits may not be written accurately. 
-    This is because individuals sometimes read phone numbers in a shortened or rounded manner. 
-    Therefore, it is necessary to design a module that considers all these variations, normalizes the text, 
-    and constructs a correct phone number.
-
-    Input and Output Specifications:
-
-    Input: The input will be in str format.
-    Output: The output will also be in str format.
-    If a phone number is identified, it will be processed and replaced with a corrected version. 
-    If no phone number is found, the text will be returned in its original format and structure.
-"""
-
-
-class DigitMapping:
-    PERSIAN_DIGITS = "۰۱۲۳۴۵۶۷۸۹"
-    WESTERN_DIGITS = "0123456789"
-    PHONE_NUMBER_LENGTH_FULL = 11
-    PHONE_NUMBER_LENGTH_PARTIAL = 10
-    SEVEN_DIGIT_PREFIX_LENGTH = 7
-    MIN_NUMBER_FOR_CHECK = 8
-    MIN_NUMBER_FOR_GENERATE = 5
-    MAX_ITERATIONS_CHECK = 4
-    PICK_FIRST_OR_LAST_LENGTH_NUMBER = 4
-    MAX_CHAR_CHECK = 30
-    MIN_CHAR_CHECK = 7
-    FIND_MIN_REPEAT_NUM = 3
-    GENERATIVE_VALID_NUMBER_COUNT = 1
-    number_words = {
-        # Persian
-        "صفر": "0",
-        "یک": "1",
-        "دو": "2",
-        "سه": "3",
-        "چهار": "4",
-        "پنج": "5",
-        "شش": "6",
-        "هفت": "7",
-        "هشت": "8",
-        "نه": "9",
-    }
-
-
-
-
-# Creating an instance of the DigitMapping class
-digit_mapping = DigitMapping()
-
-
-class PhoneNumberProcessor:
-    # Iranian mobile phone number area code
-
-    def __replace_number_words(self, text):
-        pattern = r"(\d+)([a-zA-Z]+)"
-        text = re.sub(pattern, r"\1 \2", text)
-        for word, digit in digit_mapping.number_words.items():
-            text = re.sub(r"\b" + word + r"\b", digit, text, flags=re.IGNORECASE)
-        return text
-
-    def __first_to_in_phone_number(self, number_str):
-        """
-        Identify patterns where a number is followed by "تا" and another number.
-        Generate all combinations for numbers preceding and following "تا"
-        and return them as a list.
-        """
-        generated_numbers = []
-        ta_pattern = re.compile(r"(\d)(?=\s*تا\s*(\d+))")
-        matches = ta_pattern.findall(number_str)
-
-        if matches:
-            num1, num2 = matches[0]  # Extract the first match
-            combinations = self.__generate_prefix_combinations(num1, num2)
-
-            for combo in combinations:
-                repeated_number = combo[1] * int(combo[0])
-                pattern = re.escape(f"{combo[0]} تا {combo[1]}")
-                generated_numbers.append(re.sub(pattern, repeated_number, number_str))
-
-            return generated_numbers
-        return number_str
-
-    def __process_to_constructions(self, number_str):
-        """
-        Process occurrences of 'تا' in the input string to generate possible number combinations.
-        This function calls `__first_to_in_phone_number` to handle the first 'تا' occurrence and
-        iteratively processes further occurrences if necessary.
-        """
-        # Start processing the first 'تا'
-        generated_numbers = self.__first_to_in_phone_number(number_str)
-
-        if isinstance(generated_numbers, str):
-            return generated_numbers
-
-        ta_pattern = re.compile(r"(\d+)(?=\s*تا\s*(\d+))")
-
-        i = 0
-        while i < len(generated_numbers):
-            current_number = generated_numbers[i]
-            if ta_pattern.search(current_number):
-                new_combinations = self.__first_to_in_phone_number(current_number)
-                generated_numbers.pop(i)
-                generated_numbers[i:i] = new_combinations
-            else:
-                i += 1
-
-        # Select the appropriate number from the generated list
-        selected_number = next(
-            (
-                num
-                for num in generated_numbers
-                if len("".join(num.split())) == digit_mapping.PHONE_NUMBER_LENGTH_FULL
-            ),
-            next(
-                (
-                    num
-                    for num in generated_numbers
-                    if len("".join(num.split()))
-                    == digit_mapping.PHONE_NUMBER_LENGTH_PARTIAL
-                ),
-                None,
-            ),
-        )
-
-        return "".join(selected_number.split()) if selected_number else number_str
-
-    def __generate_prefix_combinations(self, prefix1: str, prefix2: str):
-        """
-        Generate all possible combinations of the prefixes of `prefix1` and `prefix2`
-        using itertools to avoid explicit loops.
-        """
-        len1, len2 = len(prefix1), len(prefix2)
-        indices_product = itertools.product(range(1, len1 + 1), range(1, len2 + 1))
-
-        # Generate the combinations using the indices
-        return [(prefix1[:i], prefix2[:j]) for i, j in indices_product]
-
-    def __add_spaces_around_to(self, input_text):
-        """
-        Ensure "تا" has spaces around it and remove any redundant spaces.
-        """
-        # Use regular expressions to add spaces around "تا"
-        modified_text = re.sub(r"\s*تا\s*", " تا ", input_text)
-
-        return modified_text
-
-    def __generate_next_numbers(self, num):
-        str_num = str(num)
-        next_numbers = set()
-        for i in range(len(str_num) - 1):
-            current_digit = int(str_num[i])
-            next_digit = int(str_num[i + 1])
-            new_num = (
-                str_num[:i]
-                + str(int(current_digit) * str(next_digit))
-                + str_num[i + 2 :]
-            )
-            if len(new_num) <= 7:
-                next_numbers.add(str(new_num))
-        return next_numbers
-
-    def __find_seven_chain_numbers(self, start_num):
-        current_numbers = {start_num}
-        all_numbers = set(current_numbers)
-        while len(all_numbers) < digit_mapping.SEVEN_DIGIT_PREFIX_LENGTH:
-            next_numbers = set()
-
-            # Using a for loop to generate new numbers
-            for num in current_numbers:
-                generated_numbers = self.__generate_next_numbers(num)
-                new_numbers = generated_numbers - all_numbers
-                next_numbers.update(new_numbers)
-
-            if not next_numbers:  # Exit loop if no new numbers are generated
-                break
-
-            all_numbers.update(next_numbers)
-            current_numbers = next_numbers
-
-            # Early exit if we find any 7-digit number
-            if any(
-                len(str(num)) == digit_mapping.SEVEN_DIGIT_PREFIX_LENGTH
-                for num in next_numbers
-            ):
-                break
-        # Collect all 7-digit numbers and return them sorted
-        seven_digit_numbers = sorted(
-            num
-            for num in all_numbers
-            if len(str(num)) == digit_mapping.SEVEN_DIGIT_PREFIX_LENGTH
-        )
-        return seven_digit_numbers
-
-    def __apply_filter(self, numbers, index, condition):
-        return (
-            [num for num in numbers if condition(num)] if len(numbers) >= 2 else numbers
-        )
-
-    def __generate_valid_numbers(self, start_num):
-        numbers_list = self.__find_seven_chain_numbers(start_num)
-
-        # If the number list length matches the valid criteria, return the first number
-        if len(numbers_list) == digit_mapping.GENERATIVE_VALID_NUMBER_COUNT:
-            return numbers_list[0] if numbers_list else start_num
-
-        start_num_str = str(start_num)
-        unique_count = len(set(start_num_str))
-        # Step 1: Filter numbers with the same or one less unique digit count
-        filtered_numbers = [
-            num
-            for num in numbers_list
-            if len(set(str(num))) in {unique_count, unique_count - 1}
-        ]
-        # Apply additional filters only if there are at least 2 candidates remaining
-        # Step 2: Filter by last digit
-        filtered_numbers = self.__apply_filter(
-            filtered_numbers, -1, lambda num: str(num)[-1] == start_num_str[-1]
-        )
-        # Step 3: Filter by second digit
-        filtered_numbers = self.__apply_filter(
-            filtered_numbers, 1, lambda num: str(num)[1] == start_num_str[1]
-        )
-        # Step 4: Filter by last 4 digits
-        filtered_numbers = self.__apply_filter(
-            filtered_numbers, -4, lambda num: str(num)[-4:] == start_num_str[-4:]
-        )
-
-        # Return the first valid number or fallback
-        return (
-            filtered_numbers[0]
-            if filtered_numbers
-            else (numbers_list[0] if numbers_list else start_num)
-        )
-
-    def __check_area_code(self, input_text):
-        """
-        Processes the input Persian text to ensure it has a valid area
-            code and valid number format.
-        """
-        # Remove all non-digit characters from the input
-        digits_only = re.sub(r"\D", "", input_text)
-
-        # Ensure there are enough digits to process
-        if len(digits_only) <= digit_mapping.PICK_FIRST_OR_LAST_LENGTH_NUMBER:
-            return digits_only
-
-        # Extract the area code (first 4 digits) and the remaining number
-        area_code = digits_only[: digit_mapping.PICK_FIRST_OR_LAST_LENGTH_NUMBER]
-        number_part = digits_only[digit_mapping.PICK_FIRST_OR_LAST_LENGTH_NUMBER :]
-
-        # Generate valid number formats based on the remaining part
-        valid_number = self.__generate_valid_numbers(number_part)
-        # Return the formatted result
-        if not valid_number:
-            return digits_only
-        return f"{area_code}{valid_number}"
-
-    def __insert_repeated_number(self, number_str):
-        """
-        Finds the longest repeated sequence in the string and inserts one more instance of the repeated number
-        to extend the sequence, without using explicit `for` loops.
-        """
-        # Group consecutive identical characters and find the longest group
-        grouped = [
-            (char, len(list(group))) for char, group in itertools.groupby(number_str)
-        ]
-
-        # Find the max repeated group
-        max_repeated_char, max_repeated_len = max(grouped, key=lambda x: x[1])
-
-        # Only extend the sequence if the maximum repeated length is greater than 1
-        if max_repeated_len > 1:
-            # Find the start index of the sequence
-            start_index = number_str.find(max_repeated_char * max_repeated_len)
-
-            # Extend the sequence by one character and return the modified string
-            extended_sequence = max_repeated_char * (max_repeated_len + 1)
-            return (
-                number_str[:start_index]
-                + extended_sequence
-                + number_str[start_index + max_repeated_len :]
-            )
-
-        return number_str
-
-    def __add_single_repeating_digit_between_repeats(self, number_str):
-        # If the input string is already longer than the partial phone number length, return it as is
-        if len(number_str) > digit_mapping.PHONE_NUMBER_LENGTH_PARTIAL:
-            return number_str
-
-        # Split the string into two parts: the prefix (e.g., area code) and the remainder
-        prefix = number_str[: digit_mapping.PICK_FIRST_OR_LAST_LENGTH_NUMBER]
-        number_str = number_str[digit_mapping.PICK_FIRST_OR_LAST_LENGTH_NUMBER :]
-
-        result = []
-        i = 0
-        repeat_count = 1  # Track the count of consecutive repeating digits
-
-        while i < len(number_str):
-            # Check if the next digit is the same as the current one
-            if i > 0 and number_str[i] == number_str[i - 1]:
-                repeat_count += 1
-            else:
-                repeat_count = 1  # Reset the repeat count if the digit changes
-
-            # If we have a sequence of three or more repeating digits
-            if repeat_count == digit_mapping.FIND_MIN_REPEAT_NUM:
-                # Add a single extra repeating digit and then reset the counter
-                result.append(number_str[i])
-                repeat_count = 1
-
-            # Add the current digit to the result
-            result.append(number_str[i])
-            i += 1
-
-        # Concatenate the prefix with the processed result and return the final string
-        return prefix + "".join(result)
-
-    def __process_phone_number(self, phone_number):
-        """
-        Process the Persian phone number string by adding a leading zero if missing,
-        removing spaces, and checking the area code.
-        """
-        # Add leading zero to the area code if it's missing
-        if phone_number[0] != "0":
-            phone_number = "0" + phone_number
-
-        # Remove any spaces from the phone number
-        phone_number = "".join(phone_number.split())
-
-        # If the phone number is 10 digits or fewer, validate the area code
-        if len(phone_number) <= digit_mapping.PHONE_NUMBER_LENGTH_PARTIAL:
-            validated_number = self.__check_area_code(phone_number)
-            return validated_number, phone_number
-
-        # If the phone number is longer than expected, return it as-is
-        return phone_number, phone_number
-
-    def __clean_and_concatenate_numbers(self, text):
-        """
-        Process the input text by adding spaces around specific constructions,
-        removing unnecessary spaces, handling special constructions, and concatenating numbers.
-        """
-        # Add spaces around the Persian word "تا"
-        text = self.__add_spaces_around_to(text)
-        # Remove spaces between numbers
-        text = self.__remove_spaces_between_numbers(text)
-        """ Handle special constructions like replacing "تا" 
-            with corresponding numbers and removing it"""
-        text = self.__process_to_constructions(text)
-        # Concatenate digits without any spaces between them
-        text = re.sub(r"(\d)\s+(\d)", r"\1\2", text)
-
-        return text
-
-    def __english_to_persian(self, text):
-        farsi_to_latin = str.maketrans(
-            digit_mapping.WESTERN_DIGITS, digit_mapping.PERSIAN_DIGITS
-        )
-        return text.translate(farsi_to_latin)
-
-    def __persian_to_western(self, persian_number):
-        translation_table = str.maketrans(
-            digit_mapping.PERSIAN_DIGITS, digit_mapping.WESTERN_DIGITS
-        )
-        return persian_number.translate(translation_table)
-
-    def __remove_spaces_between_numbers(self, text):
-        # This regex will match spaces that are between two digits
-        return re.sub(r"(\d)\s+(\d)", r"\1\2", text)
-
-    def __process_patterned_numbers(self, number_sequence):
-        if number_sequence is None:
-            return number_sequence
-        # Convert Persian numbers to Western (English) numbers
-        western_number_sequence = self.__persian_to_western(number_sequence)
-
-        # Remove spaces between numbers to form a continuous sequence
-        contiguous_numbers = self.__remove_spaces_between_numbers(western_number_sequence)
-
-        # Clean up any remaining spaces
-        clean_number_sequence = "".join(contiguous_numbers.split())
-
-        # Clean and concatenate numbers to form the phone number
-        concatenated_numbers = self.__clean_and_concatenate_numbers(clean_number_sequence)
-
-        # Add single repeating digit between repeats in the phone number
-        processed_sequence = self.__add_single_repeating_digit_between_repeats(
-            concatenated_numbers
-        )
-
-        # Process and validate the phone number
-        final_number, original_sequence = self.__process_phone_number(processed_sequence)
-
-        # Limit the number of iterations to avoid infinite loops
-
-        iterations = 0
-
-        # Loop to handle sequences of length 8 to 10 digits
-        while (
-            digit_mapping.MIN_NUMBER_FOR_CHECK
-            <= len(str(final_number))
-            <= digit_mapping.PHONE_NUMBER_LENGTH_PARTIAL
-            and iterations < digit_mapping.MAX_ITERATIONS_CHECK
-        ):
-            if len(final_number) == 0:
-                modified_sequence = self.__insert_repeated_number(str(original_sequence))
-            else:
-                modified_sequence = self.__insert_repeated_number(str(final_number))
-
-            # Re-process the phone number
-            final_number, original_sequence = self.__process_phone_number(
-                modified_sequence
-            )
-
-            iterations += 1
-        # Handle case when final_number is a list
-        if isinstance(final_number, list):
-            final_number = final_number[0]
-
-        return final_number
-
-    def __update_text_with_number(self, text, new_number, old_prefix, new_prefix):
-        """
-        Update the text by replacing the old prefix with the new prefix and converting it to Persian.
-        """
-        processed_text = self.__process_patterned_numbers(new_number)
-        persian_text = self.__english_to_persian(processed_text)
-        text = re.sub(old_prefix, new_prefix, text, count=1)
-        return re.sub(re.escape(new_number), persian_text, text)
-
-    def __update_general_case(self, text, number):
-        """
-        Handle general number replacement cases and update the text.
-        """
-        processed_text = self.__process_patterned_numbers(number)
-        persian_text = self.__english_to_persian(processed_text)
-        return re.sub(re.escape(number), persian_text, text)
-
-    def __process_number_replacement(self, text, number):
-        """
-        Process specific number patterns like '۰۹۹' and '۹۹' and replace them with correct forms.
-        """
-        first_digits = number[:3]
-
-        if first_digits == "۰۹۹":
-            new_number = number.replace("۰۹۹", "۰۹۹۹", 1)
-            return self.__update_text_with_number(text, new_number, "۰۹۹", "۰۹۹۹")
-
-        return self.__update_general_case(text, number)
-
-    @staticmethod
-    def normalize(text):
-        """
-        Normalize phone numbers in the input Persian text by handling specific patterns
-        and replacing them with the correct numeric forms.
-        """
-
-        # Compile regex pattern to match number patterns
-        pattern_regex = re.compile(
-            r"((0?[1-9][0-9]{1,3})|(۰?[۱-۹][۰-۹]{1,3}))[\s۰-۹0-9تا]*"
-        )
-
-        # Clean up text by removing extra spaces
-        cleaned_text = re.sub(r"\s{2,}", " ", text)
-
-        processor = PhoneNumberProcessor()
-
-        # Replace Persian number words with numeric equivalents
-        cleaned_text = processor.__replace_number_words(cleaned_text)
-
-        # Find all matching number patterns
-        matches = pattern_regex.finditer(cleaned_text)
-        results = [
-            match.group()
-            for match in matches
-            if digit_mapping.MIN_CHAR_CHECK
-            <= len(match.group())
-            <= digit_mapping.MAX_CHAR_CHECK
-        ]
-
-        # Process each match to handle specific cases
-        for result in results:
-            digits = re.findall(r"\d", result)
-            if (
-                digit_mapping.MIN_NUMBER_FOR_GENERATE
-                <= len(digits)
-                <= digit_mapping.PHONE_NUMBER_LENGTH_PARTIAL
-            ):
-                cleaned_text = processor.__process_number_replacement(cleaned_text, result)
-
-        return cleaned_text if results else text