LanguageStatisticsLibPy 1.0.3__py3-none-any.whl → 1.0.4__py3-none-any.whl

languagestatisticslibpy/LanguageStatisticsFile.py

@@ -1,4 +1,4 @@
-'''
+"""
    Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
 
    Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,15 +12,12 @@
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
    limitations under the License.
-'''
-import struct
-import numpy as np
-import gzip
-
+"""
 import gzip
 import struct
 import numpy as np
 
+
 class LanguageStatisticsFile:
     """
     Class for handling the loading of language statistics from a compressed file.
@@ -28,29 +25,28 @@ class LanguageStatisticsFile:
     Attributes:
     - FILE_FORMAT_MAGIC_NUMBER (str): The expected magic number to identify valid language statistics files.
     - file_path (str): The path to the language statistics file.
-    - alphabet (str): The alphabet used in the language statistics.
+    - alphabet (str): The decoded alphabet used in the language statistics.
+    - alphabet_length (int): Number of symbols (characters) in `alphabet` (NOT bytes).
+    - alphabet_length_bytes (int): Length prefix from file (= number of bytes used to store the alphabet string).
     - language_code (str): The language code extracted from the statistics file.
     """
 
     FILE_FORMAT_MAGIC_NUMBER = "CTLS"
 
-    def __init__(self, file_path):
+    def __init__(self, file_path: str):
         """
         Initializes the LanguageStatisticsFile class.
 
         Parameters:
         - file_path (str): The path to the language statistics file.
-
-        Initializes:
-        - self.file_path (str): Stores the path to the file.
-        - self.alphabet (str): Initially empty, set after file loading.
-        - self.language_code (str): Initially empty, set after file loading.
         """
         self.file_path = file_path
-        self.alphabet = ''
-        self.language_code = ''
+        self.alphabet = ""
+        self.alphabet_length = 0
+        self.alphabet_length_bytes = 0
+        self.language_code = ""
 
-    def load_frequencies(self, array_dimensions):
+    def load_frequencies(self, array_dimensions: int) -> np.ndarray:
         """
         Loads the frequency data from the language statistics file.
 
@@ -61,48 +57,60 @@ class LanguageStatisticsFile:
         - np.ndarray: A numpy array containing the frequency data.
 
         Raises:
-        - Exception: If the file format is invalid or the dimensions of the frequency array do not match the expected value.
-
-        Process:
-        1. Validates the file by checking the magic number.
-        2. Reads the language code, gram length, and alphabet.
-        3. Verifies that the gram length matches the required dimensions.
-        4. Reads the frequency data and reshapes it into the appropriate numpy array format.
-        5. Copies the data into a new numpy array to allow modification.
+        - Exception: If the file format is invalid, gram length mismatches, or the file content size is inconsistent.
+
+        Notes:
+        - The alphabet length stored in the file is a *byte length* (UTF-8). For non-ASCII alphabets (e.g., German ÄÖÜß),
+          the number of bytes differs from the number of symbols. We therefore decode first and then compute the symbol count.
         """
-        with gzip.open(self.file_path, 'rb') as file:
+        if array_dimensions < 1:
+            raise Exception("array_dimensions must be >= 1")
+
+        with gzip.open(self.file_path, "rb") as file:
             # Validate the file format by checking the magic number.
-            magic_number = file.read(4).decode('utf-8')
+            magic_number = file.read(4).decode("utf-8")
             if magic_number != self.FILE_FORMAT_MAGIC_NUMBER:
                 raise Exception("File does not start with the expected magic number for language statistics.")
 
             # Read the language code (length-prefixed string).
             language_code_length_bytes = file.read(1)[0]
-            self.language_code = file.read(language_code_length_bytes).decode('utf-8')
+            self.language_code = file.read(language_code_length_bytes).decode("utf-8")
 
             # Read the gram length (32-bit signed integer).
-            gram_length = struct.unpack('<i', file.read(4))[0]
+            gram_length = struct.unpack("<i", file.read(4))[0]
 
             # Ensure the gram length matches the required dimensions.
             if gram_length != array_dimensions:
                 raise Exception("Gram length of statistics file differs from required dimensions of frequency array.")
 
-            # Read the alphabet (length-prefixed string).
-            self.alphabet_length = file.read(1)[0]
-            self.alphabet = file.read(self.alphabet_length).decode('utf-8')
+            # Read the alphabet (length-prefixed BYTES, UTF-8).
+            self.alphabet_length_bytes = file.read(1)[0]
+            alphabet_bytes = file.read(self.alphabet_length_bytes)
+            self.alphabet = alphabet_bytes.decode("utf-8")
+
+            # IMPORTANT: use number of symbols (characters), not bytes, for frequency tensor dimensions.
+            self.alphabet_length = len(self.alphabet)
 
             # Calculate the total number of frequency entries.
             frequency_entries = self.alphabet_length ** gram_length
+            expected_frequency_bytes = frequency_entries * 4  # float32
 
             # Read the frequency data (32-bit float array).
-            frequency_data = file.read(frequency_entries * 4)
-
-            # Reshape the data into the correct dimensionality and copy it for mutability.
+            frequency_data = file.read(expected_frequency_bytes)
+
+            # Hard validation: mismatched files should fail with a clear message.
+            if len(frequency_data) != expected_frequency_bytes:
+                raise Exception(
+                    f"Frequency data size mismatch. "
+                    f"Expected {expected_frequency_bytes} bytes ({frequency_entries} float32 values) "
+                    f"for alphabet_length={self.alphabet_length} and gram_length={gram_length}, "
+                    f"but got {len(frequency_data)} bytes. "
+                    f"(alphabet_length_bytes={self.alphabet_length_bytes}, alphabet='{self.alphabet}')"
+                )
+
+            # Convert and reshape.
+            data = np.frombuffer(frequency_data, dtype=np.float32)
             if array_dimensions == 1:
-                frequencies = np.frombuffer(frequency_data, dtype=np.float32).copy()
-            else:
-                frequencies = np.frombuffer(frequency_data, dtype=np.float32).reshape(
-                    tuple([self.alphabet_length] * array_dimensions)
-                ).copy()
+                return data.copy()
 
-            return frequencies
+            return data.reshape(tuple([self.alphabet_length] * array_dimensions)).copy()

languagestatisticslibpy/Tetragrams.py

@@ -1,125 +1,125 @@
-'''
-   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-'''
-import numpy as np
-import os
-from languagestatisticslibpy.Grams import Grams
-from languagestatisticslibpy.GramsType import GramsType
-from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
-
-class Tetragrams(Grams):
-    def __init__(self, language, language_statistics_directory, use_spaces=False):
-        """
-        Initializes the Tetragrams class by calling the parent class (Grams) initializer.
-
-        Parameters:
-        - language (str): The language of the tetragram statistics.
-        - language_statistics_directory (str): Path to the directory containing language statistics files.
-        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
-        """
-        super().__init__(language, language_statistics_directory, use_spaces)
-
-    def load_gz(self, filename, language_statistics_directory):
-        """
-        Loads a gzip-compressed file containing tetragram frequencies.
-
-        Parameters:
-        - filename (str): The name of the file to load.
-        - language_statistics_directory (str): The directory where the statistics file is located.
-
-        Sets:
-        - self.frequencies (np.ndarray): A 4D array of tetragram frequencies.
-        - self.alphabet (list): The alphabet used in the statistics file.
-        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
-        """
-        file_path = os.path.join(language_statistics_directory, filename)
-        language_statistics_file = LanguageStatisticsFile(file_path)
-        self.frequencies = language_statistics_file.load_frequencies(4)
-        self.alphabet = language_statistics_file.alphabet
-        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
-
-    def calculate_cost(self, text):
-        """
-        Calculates the cost of a given text based on tetragram frequencies.
-
-        Parameters:
-        - text (str): The text to analyze.
-
-        Returns:
-        - float: The average cost of tetragrams in the text. Returns 0.0 if the text length is less than 4.
-
-        Notes:
-        - Skips tetragrams containing characters outside the defined alphabet.
-        - If `add_letter_indices` is defined, modifies the index of the characters before computing the cost.
-        """
-        if len(text) < 4:
-            return 0.0
-
-        value = 0.0
-        alphabet_length = len(self.alphabet)
-        end = len(text) - 3
-
-        for i in range(end):
-            a, b, c, d = text[i:i+4]
-
-            if self.add_letter_indices:
-                a += self.add_letter_indices.get(a, 0)
-                b += self.add_letter_indices.get(b, 0)
-                c += self.add_letter_indices.get(c, 0)
-                d += self.add_letter_indices.get(d, 0)
-
-            if 0 <= a < alphabet_length and 0 <= b < alphabet_length and \
-               0 <= c < alphabet_length and 0 <= d < alphabet_length:
-                value += self.frequencies[a, b, c, d]
-
-        return value / end
-
-    def gram_size(self):
-        """
-        Returns the size of the grams being analyzed (tetragrams in this case).
-
-        Returns:
-        - int: The size of the grams (always 4 for tetragrams).
-        """
-        return 4
-
-    def grams_type(self):
-        """
-        Returns the type of grams being analyzed.
-
-        Returns:
-        - GramsType: An enum value representing the type of grams (GramsType.Tetragrams).
-        """
-        return GramsType.Tetragrams
-
-    def normalize(self, max_value):
-        """
-        Normalizes the tetragram frequencies based on the provided maximum value.
-
-        Parameters:
-        - max_value (float): The maximum value used for normalization.
-
-        Notes:
-        - Adjusts all frequencies proportionally to the new maximum value.
-        - Updates `self.max_value` to the new maximum after normalization.
-        """
-        super().normalize(max_value)
-        adjust_value = self.max_value * max_value
-        for a in range(len(self.alphabet)):
-            for b in range(len(self.alphabet)):
-                for c in range(len(self.alphabet)):
-                    for d in range(len(self.alphabet)):
-                        self.frequencies[a, b, c, d] = adjust_value / self.frequencies[a, b, c, d]
-        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
+'''
+   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+'''
+import numpy as np
+import os
+from languagestatisticslibpy.Grams import Grams
+from languagestatisticslibpy.GramsType import GramsType
+from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
+
+class Tetragrams(Grams):
+    def __init__(self, language, language_statistics_directory, use_spaces=False):
+        """
+        Initializes the Tetragrams class by calling the parent class (Grams) initializer.
+
+        Parameters:
+        - language (str): The language of the tetragram statistics.
+        - language_statistics_directory (str): Path to the directory containing language statistics files.
+        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
+        """
+        super().__init__(language, language_statistics_directory, use_spaces)
+
+    def load_gz(self, filename, language_statistics_directory):
+        """
+        Loads a gzip-compressed file containing tetragram frequencies.
+
+        Parameters:
+        - filename (str): The name of the file to load.
+        - language_statistics_directory (str): The directory where the statistics file is located.
+
+        Sets:
+        - self.frequencies (np.ndarray): A 4D array of tetragram frequencies.
+        - self.alphabet (list): The alphabet used in the statistics file.
+        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
+        """
+        file_path = os.path.join(language_statistics_directory, filename)
+        language_statistics_file = LanguageStatisticsFile(file_path)
+        self.frequencies = language_statistics_file.load_frequencies(4)
+        self.alphabet = language_statistics_file.alphabet
+        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
+
+    def calculate_cost(self, text):
+        """
+        Calculates the cost of a given text based on tetragram frequencies.
+
+        Parameters:
+        - text (str): The text to analyze.
+
+        Returns:
+        - float: The average cost of tetragrams in the text. Returns 0.0 if the text length is less than 4.
+
+        Notes:
+        - Skips tetragrams containing characters outside the defined alphabet.
+        - If `add_letter_indices` is defined, modifies the index of the characters before computing the cost.
+        """
+        if len(text) < 4:
+            return 0.0
+
+        value = 0.0
+        alphabet_length = len(self.alphabet)
+        end = len(text) - 3
+
+        for i in range(end):
+            a, b, c, d = text[i:i+4]
+
+            if self.add_letter_indices:
+                a += self.add_letter_indices.get(a, 0)
+                b += self.add_letter_indices.get(b, 0)
+                c += self.add_letter_indices.get(c, 0)
+                d += self.add_letter_indices.get(d, 0)
+
+            if 0 <= a < alphabet_length and 0 <= b < alphabet_length and \
+               0 <= c < alphabet_length and 0 <= d < alphabet_length:
+                value += self.frequencies[a, b, c, d]
+
+        return value / end
+
+    def gram_size(self):
+        """
+        Returns the size of the grams being analyzed (tetragrams in this case).
+
+        Returns:
+        - int: The size of the grams (always 4 for tetragrams).
+        """
+        return 4
+
+    def grams_type(self):
+        """
+        Returns the type of grams being analyzed.
+
+        Returns:
+        - GramsType: An enum value representing the type of grams (GramsType.Tetragrams).
+        """
+        return GramsType.Tetragrams
+
+    def normalize(self, max_value):
+        """
+        Normalizes the tetragram frequencies based on the provided maximum value.
+
+        Parameters:
+        - max_value (float): The maximum value used for normalization.
+
+        Notes:
+        - Adjusts all frequencies proportionally to the new maximum value.
+        - Updates `self.max_value` to the new maximum after normalization.
+        """
+        super().normalize(max_value)
+        adjust_value = self.max_value * max_value
+        for a in range(len(self.alphabet)):
+            for b in range(len(self.alphabet)):
+                for c in range(len(self.alphabet)):
+                    for d in range(len(self.alphabet)):
+                        self.frequencies[a, b, c, d] = adjust_value / self.frequencies[a, b, c, d]
+        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')

languagestatisticslibpy/Trigrams.py

@@ -1,125 +1,125 @@
-'''
-   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-'''
-import numpy as np
-import os
-from languagestatisticslibpy.Grams import Grams
-from languagestatisticslibpy.GramsType import GramsType
-from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
-
-class Trigrams(Grams):
-    def __init__(self, language, language_statistics_directory, use_spaces=False):
-        """
-        Initializes the Trigrams class by calling the parent class (Grams) initializer.
-
-        Parameters:
-        - language (str): The language of the trigram statistics.
-        - language_statistics_directory (str): Path to the directory containing language statistics files.
-        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
-        """
-        super().__init__(language, language_statistics_directory, use_spaces)
-
-    def load_gz(self, filename, language_statistics_directory):
-        """
-        Loads a gzip-compressed file containing trigram frequencies.
-
-        Parameters:
-        - filename (str): The name of the file to load.
-        - language_statistics_directory (str): The directory where the statistics file is located.
-
-        Sets:
-        - self.frequencies (np.ndarray): A 3D array of trigram frequencies.
-        - self.alphabet (list): The alphabet used in the statistics file.
-        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
-        """
-        file_path = os.path.join(language_statistics_directory, filename)
-        language_statistics_file = LanguageStatisticsFile(file_path)
-        self.frequencies = language_statistics_file.load_frequencies(3)
-        self.alphabet = language_statistics_file.alphabet
-        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
-
-    def calculate_cost(self, text):
-        """
-        Calculates the cost of a given text based on trigram frequencies.
-
-        Parameters:
-        - text (str): The text to analyze.
-
-        Returns:
-        - float: The average cost of trigrams in the text. Returns 0 if the text length is less than 3.
-
-        Notes:
-        - Skips trigrams containing characters outside the defined alphabet.
-        - If `add_letter_indices` is defined, modifies indices of the characters before computing the cost.
-        """
-        if len(text) < 3:
-            return 0
-
-        value = 0
-        alphabet_length = len(self.alphabet)
-        end = len(text) - 2
-
-        for i in range(end):
-            a = text[i]
-            b = text[i + 1]
-            c = text[i + 2]
-
-            if self.add_letter_indices:
-                a += self.add_letter_indices.get(a, 0)
-                b += self.add_letter_indices.get(b, 0)
-                c += self.add_letter_indices.get(c, 0)
-
-            if a >= alphabet_length or b >= alphabet_length or c >= alphabet_length or a < 0 or b < 0 or c < 0:
-                continue
-            value += self.frequencies[a, b, c]
-
-        return value / end
-
-    def gram_size(self):
-        """
-        Returns the size of the grams being analyzed (trigrams in this case).
-
-        Returns:
-        - int: The size of the grams (always 3 for trigrams).
-        """
-        return 3
-
-    def grams_type(self):
-        """
-        Returns the type of grams being analyzed.
-
-        Returns:
-        - GramsType: An enum value representing the type of grams (GramsType.Trigrams).
-        """
-        return GramsType.Trigrams
-
-    def normalize(self, max_value):
-        """
-        Normalizes the trigram frequencies based on the provided maximum value.
-
-        Parameters:
-        - max_value (float): The maximum value used for normalization.
-
-        Notes:
-        - Adjusts all frequencies proportionally to the new maximum value.
-        - Updates `self.max_value` to the new maximum after normalization.
-        """
-        super().normalize(max_value)
-        adjust_value = self.max_value * max_value
-        for a in range(len(self.alphabet)):
-            for b in range(len(self.alphabet)):
-                for c in range(len(self.alphabet)):
-                    self.frequencies[a, b, c] = adjust_value / self.frequencies[a, b, c]
-        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
+'''
+   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+'''
+import numpy as np
+import os
+from languagestatisticslibpy.Grams import Grams
+from languagestatisticslibpy.GramsType import GramsType
+from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
+
+class Trigrams(Grams):
+    def __init__(self, language, language_statistics_directory, use_spaces=False):
+        """
+        Initializes the Trigrams class by calling the parent class (Grams) initializer.
+
+        Parameters:
+        - language (str): The language of the trigram statistics.
+        - language_statistics_directory (str): Path to the directory containing language statistics files.
+        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
+        """
+        super().__init__(language, language_statistics_directory, use_spaces)
+
+    def load_gz(self, filename, language_statistics_directory):
+        """
+        Loads a gzip-compressed file containing trigram frequencies.
+
+        Parameters:
+        - filename (str): The name of the file to load.
+        - language_statistics_directory (str): The directory where the statistics file is located.
+
+        Sets:
+        - self.frequencies (np.ndarray): A 3D array of trigram frequencies.
+        - self.alphabet (list): The alphabet used in the statistics file.
+        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
+        """
+        file_path = os.path.join(language_statistics_directory, filename)
+        language_statistics_file = LanguageStatisticsFile(file_path)
+        self.frequencies = language_statistics_file.load_frequencies(3)
+        self.alphabet = language_statistics_file.alphabet
+        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
+
+    def calculate_cost(self, text):
+        """
+        Calculates the cost of a given text based on trigram frequencies.
+
+        Parameters:
+        - text (str): The text to analyze.
+
+        Returns:
+        - float: The average cost of trigrams in the text. Returns 0 if the text length is less than 3.
+
+        Notes:
+        - Skips trigrams containing characters outside the defined alphabet.
+        - If `add_letter_indices` is defined, modifies indices of the characters before computing the cost.
+        """
+        if len(text) < 3:
+            return 0
+
+        value = 0
+        alphabet_length = len(self.alphabet)
+        end = len(text) - 2
+
+        for i in range(end):
+            a = text[i]
+            b = text[i + 1]
+            c = text[i + 2]
+
+            if self.add_letter_indices:
+                a += self.add_letter_indices.get(a, 0)
+                b += self.add_letter_indices.get(b, 0)
+                c += self.add_letter_indices.get(c, 0)
+
+            if a >= alphabet_length or b >= alphabet_length or c >= alphabet_length or a < 0 or b < 0 or c < 0:
+                continue
+            value += self.frequencies[a, b, c]
+
+        return value / end
+
+    def gram_size(self):
+        """
+        Returns the size of the grams being analyzed (trigrams in this case).
+
+        Returns:
+        - int: The size of the grams (always 3 for trigrams).
+        """
+        return 3
+
+    def grams_type(self):
+        """
+        Returns the type of grams being analyzed.
+
+        Returns:
+        - GramsType: An enum value representing the type of grams (GramsType.Trigrams).
+        """
+        return GramsType.Trigrams
+
+    def normalize(self, max_value):
+        """
+        Normalizes the trigram frequencies based on the provided maximum value.
+
+        Parameters:
+        - max_value (float): The maximum value used for normalization.
+
+        Notes:
+        - Adjusts all frequencies proportionally to the new maximum value.
+        - Updates `self.max_value` to the new maximum after normalization.
+        """
+        super().normalize(max_value)
+        adjust_value = self.max_value * max_value
+        for a in range(len(self.alphabet)):
+            for b in range(len(self.alphabet)):
+                for c in range(len(self.alphabet)):
+                    self.frequencies[a, b, c] = adjust_value / self.frequencies[a, b, c]
+        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')