LanguageStatisticsLibPy 1.0.3__py3-none-any.whl → 1.0.4__py3-none-any.whl

languagestatisticslibpy/Unigrams.py

@@ -1,110 +1,110 @@
-'''
-   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-'''
-import numpy as np
-import os
-from languagestatisticslibpy.Grams import Grams
-from languagestatisticslibpy.GramsType import GramsType
-from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
-
-class Unigrams(Grams):
-    def __init__(self, language, language_statistics_directory, use_spaces=False):
-        """
-        Initializes the Unigrams class by calling the parent class (Grams) initializer.
-
-        Parameters:
-        - language (str): The language of the unigram statistics.
-        - language_statistics_directory (str): Path to the directory containing language statistics files.
-        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
-        """
-        super().__init__(language, language_statistics_directory, use_spaces)
-
-    def load_gz(self, filename, language_statistics_directory):
-        """
-        Loads a gzip-compressed file containing unigram frequencies.
-
-        Parameters:
-        - filename (str): The name of the file to load.
-        - language_statistics_directory (str): The directory where the statistics file is located.
-
-        Sets:
-        - self.frequencies (np.ndarray): A 1D array of unigram frequencies.
-        - self.alphabet (list): The alphabet used in the statistics file.
-        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
-        """
-        file_path = os.path.join(language_statistics_directory, filename)
-        language_statistics_file = LanguageStatisticsFile(file_path)
-        self.frequencies = language_statistics_file.load_frequencies(1)
-        self.alphabet = language_statistics_file.alphabet
-        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
-
-    def calculate_cost(self, text):
-        """
-        Calculates the cost of a given text based on unigram frequencies.
-
-        Parameters:
-        - text (str): The text to analyze.
-
-        Returns:
-        - float: The average cost of unigrams in the text. Returns 0.0 if the text is empty.
-
-        Notes:
-        - Skips characters that are outside the defined alphabet.
-        - If `add_letter_indices` is defined, modifies the index of the character before computing the cost.
-        """
-        if len(text) == 0:
-            return 0.0
-
-        value = 0.0
-        for i in text:
-            if self.add_letter_indices:
-                i += self.add_letter_indices.get(i, 0)
-            if 0 <= i < len(self.alphabet):
-                value += self.frequencies[i]
-        return value / len(text)
-
-    def gram_size(self):
-        """
-        Returns the size of the grams being analyzed (unigrams in this case).
-
-        Returns:
-        - int: The size of the grams (always 1 for unigrams).
-        """
-        return 1
-
-    def grams_type(self):
-        """
-        Returns the type of grams being analyzed.
-
-        Returns:
-        - GramsType: An enum value representing the type of grams (GramsType.Unigrams).
-        """
-        return GramsType.Unigrams
-
-    def normalize(self, max_value):
-        """
-        Normalizes the unigram frequencies based on the provided maximum value.
-
-        Parameters:
-        - max_value (float): The maximum value used for normalization.
-
-        Notes:
-        - Adjusts all frequencies proportionally to the new maximum value.
-        - Updates `self.max_value` to the new maximum after normalization.
-        """
-        super().normalize(max_value)
-        adjust_value = self.max_value * max_value
-        for a in range(len(self.alphabet)):
-            self.frequencies[a] = adjust_value / self.frequencies[a]
+'''
+   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+'''
+import numpy as np
+import os
+from languagestatisticslibpy.Grams import Grams
+from languagestatisticslibpy.GramsType import GramsType
+from languagestatisticslibpy.LanguageStatisticsFile import LanguageStatisticsFile
+
+class Unigrams(Grams):
+    def __init__(self, language, language_statistics_directory, use_spaces=False):
+        """
+        Initializes the Unigrams class by calling the parent class (Grams) initializer.
+
+        Parameters:
+        - language (str): The language of the unigram statistics.
+        - language_statistics_directory (str): Path to the directory containing language statistics files.
+        - use_spaces (bool): Whether to include spaces in the analysis (default: False).
+        """
+        super().__init__(language, language_statistics_directory, use_spaces)
+
+    def load_gz(self, filename, language_statistics_directory):
+        """
+        Loads a gzip-compressed file containing unigram frequencies.
+
+        Parameters:
+        - filename (str): The name of the file to load.
+        - language_statistics_directory (str): The directory where the statistics file is located.
+
+        Sets:
+        - self.frequencies (np.ndarray): A 1D array of unigram frequencies.
+        - self.alphabet (list): The alphabet used in the statistics file.
+        - self.max_value (float): The maximum value in the frequencies array, or -∞ if the array is empty.
+        """
+        file_path = os.path.join(language_statistics_directory, filename)
+        language_statistics_file = LanguageStatisticsFile(file_path)
+        self.frequencies = language_statistics_file.load_frequencies(1)
+        self.alphabet = language_statistics_file.alphabet
+        self.max_value = np.max(self.frequencies) if self.frequencies.size > 0 else float('-inf')
+
+    def calculate_cost(self, text):
+        """
+        Calculates the cost of a given text based on unigram frequencies.
+
+        Parameters:
+        - text (str): The text to analyze.
+
+        Returns:
+        - float: The average cost of unigrams in the text. Returns 0.0 if the text is empty.
+
+        Notes:
+        - Skips characters that are outside the defined alphabet.
+        - If `add_letter_indices` is defined, modifies the index of the character before computing the cost.
+        """
+        if len(text) == 0:
+            return 0.0
+
+        value = 0.0
+        for i in text:
+            if self.add_letter_indices:
+                i += self.add_letter_indices.get(i, 0)
+            if 0 <= i < len(self.alphabet):
+                value += self.frequencies[i]
+        return value / len(text)
+
+    def gram_size(self):
+        """
+        Returns the size of the grams being analyzed (unigrams in this case).
+
+        Returns:
+        - int: The size of the grams (always 1 for unigrams).
+        """
+        return 1
+
+    def grams_type(self):
+        """
+        Returns the type of grams being analyzed.
+
+        Returns:
+        - GramsType: An enum value representing the type of grams (GramsType.Unigrams).
+        """
+        return GramsType.Unigrams
+
+    def normalize(self, max_value):
+        """
+        Normalizes the unigram frequencies based on the provided maximum value.
+
+        Parameters:
+        - max_value (float): The maximum value used for normalization.
+
+        Notes:
+        - Adjusts all frequencies proportionally to the new maximum value.
+        - Updates `self.max_value` to the new maximum after normalization.
+        """
+        super().normalize(max_value)
+        adjust_value = self.max_value * max_value
+        for a in range(len(self.alphabet)):
+            self.frequencies[a] = adjust_value / self.frequencies[a]

languagestatisticslibpy/WordTree.py

@@ -1,162 +1,162 @@
-'''
-   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-'''
-from io import BufferedReader
-from collections import deque
-from languagestatisticslibpy.Node import Node
-
-class WordTree(Node):
-    """
-    Represents a tree data structure for storing words and efficiently querying them.
-
-    Inherits:
-    - Node: The base class for tree nodes, where each node represents a character.
-
-    Attributes:
-    - stored_words (int): The number of words stored in the tree.
-    - language_code (str): The language code for the words stored in the tree.
-    - alphabet (str): The alphabet used in the stored words.
-    """
-
-    def __init__(self):
-        """
-        Initializes an empty WordTree.
-
-        Initializes:
-        - stored_words (int): Set to 0, as no words are initially stored.
-        - language_code (str): Empty, to be set during deserialization.
-        - alphabet (str): Empty, to be set during deserialization.
-        """
-        super().__init__()
-        self.stored_words = 0
-        self.language_code = ''
-        self.alphabet = ''
-
-    @staticmethod
-    def deserialize(reader: BufferedReader):
-        """
-        Deserializes a WordTree from a binary file.
-
-        Parameters:
-        - reader (BufferedReader): A binary file reader containing the serialized WordTree.
-
-        Returns:
-        - WordTree: The deserialized WordTree object.
-
-        Raises:
-        - Exception: If the file format is invalid or the magic number does not match.
-
-        Process:
-        1. Reads the file header and validates the magic number.
-        2. Reads the language code and alphabet.
-        3. Reads the number of stored words.
-        4. Constructs the WordTree structure by iterating through the file's serialized data.
-        """
-        tree = WordTree()
-
-        # Load word tree header
-        magic_no = reader.read(6).decode('utf-8')
-        if magic_no != "CT2DIC":
-            raise Exception("File does not start with the expected magic number for word tree.")
-
-        # Read language code
-        tree.language_code = ''
-        char = reader.read(1).decode('utf-8')
-        while char != '\0':
-            tree.language_code += char
-            char = reader.read(1).decode('utf-8')
-
-        # Read alphabet
-        tree.alphabet = ''
-        char = reader.read(1).decode('utf-8')
-        while char != '\0':
-            tree.alphabet += char
-            char = reader.read(1).decode('utf-8')
-
-        # Read number of stored words
-        tree.stored_words = int.from_bytes(reader.read(4), 'little')
-
-        # Load word tree data structure
-        stack = deque([tree])
-        byte = reader.read(1)
-        while byte:
-            char = byte.decode('utf-8')
-            if char == Node.WordEndSymbol:
-                stack[-1].word_ends_here = True
-                tree.stored_words += 1
-            elif char == Node.TerminationSymbol:
-                stack.pop()
-            else:
-                new_node = Node(char)
-                stack[-1].child_nodes.append(new_node)
-                stack.append(new_node)
-            byte = reader.read(1)
-
-        return tree
-
-    def contains_word(self, word):
-        """
-        Checks whether a given word exists in the WordTree.
-
-        Parameters:
-        - word (str): The word to search for.
-
-        Returns:
-        - bool: True if the word exists in the tree, False otherwise.
-
-        Process:
-        1. Converts the word to uppercase for case-insensitive comparison.
-        2. Traverses the tree to find the sequence of characters in the word.
-        3. Returns False if any character is missing in the tree structure.
-        """
-        word = word.upper()
-        current_node = self
-        for char in word:
-            found_node = None
-            for child_node in current_node.child_nodes:
-                if child_node.value == char:
-                    current_node = child_node
-                    found_node = True
-                    break
-            if not found_node:
-                return False
-        return True
-
-    def to_list(self):
-        """
-        Converts all words stored in the WordTree into a list.
-
-        Returns:
-        - list: A list of all words stored in the tree.
-
-        Process:
-        1. Traverses the tree using a stack to collect characters.
-        2. Adds a word to the list whenever a node marks the end of a word.
-        """
-        list_of_words = []
-        stack = deque()
-
-        def add_node_to_list(node, stack):
-            stack.append(node.value)
-            if node.word_ends_here:
-                list_of_words.append(''.join(stack))
-            for child_node in node.child_nodes:
-                add_node_to_list(child_node, deque(stack))
-            stack.pop()
-
-        for node in self.child_nodes:
-            add_node_to_list(node, stack)
-
+'''
+   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+'''
+from io import BufferedReader
+from collections import deque
+from languagestatisticslibpy.Node import Node
+
+class WordTree(Node):
+    """
+    Represents a tree data structure for storing words and efficiently querying them.
+
+    Inherits:
+    - Node: The base class for tree nodes, where each node represents a character.
+
+    Attributes:
+    - stored_words (int): The number of words stored in the tree.
+    - language_code (str): The language code for the words stored in the tree.
+    - alphabet (str): The alphabet used in the stored words.
+    """
+
+    def __init__(self):
+        """
+        Initializes an empty WordTree.
+
+        Initializes:
+        - stored_words (int): Set to 0, as no words are initially stored.
+        - language_code (str): Empty, to be set during deserialization.
+        - alphabet (str): Empty, to be set during deserialization.
+        """
+        super().__init__()
+        self.stored_words = 0
+        self.language_code = ''
+        self.alphabet = ''
+
+    @staticmethod
+    def deserialize(reader: BufferedReader):
+        """
+        Deserializes a WordTree from a binary file.
+
+        Parameters:
+        - reader (BufferedReader): A binary file reader containing the serialized WordTree.
+
+        Returns:
+        - WordTree: The deserialized WordTree object.
+
+        Raises:
+        - Exception: If the file format is invalid or the magic number does not match.
+
+        Process:
+        1. Reads the file header and validates the magic number.
+        2. Reads the language code and alphabet.
+        3. Reads the number of stored words.
+        4. Constructs the WordTree structure by iterating through the file's serialized data.
+        """
+        tree = WordTree()
+
+        # Load word tree header
+        magic_no = reader.read(6).decode('utf-8')
+        if magic_no != "CT2DIC":
+            raise Exception("File does not start with the expected magic number for word tree.")
+
+        # Read language code
+        tree.language_code = ''
+        char = reader.read(1).decode('utf-8')
+        while char != '\0':
+            tree.language_code += char
+            char = reader.read(1).decode('utf-8')
+
+        # Read alphabet
+        tree.alphabet = ''
+        char = reader.read(1).decode('utf-8')
+        while char != '\0':
+            tree.alphabet += char
+            char = reader.read(1).decode('utf-8')
+
+        # Read number of stored words
+        tree.stored_words = int.from_bytes(reader.read(4), 'little')
+
+        # Load word tree data structure
+        stack = deque([tree])
+        byte = reader.read(1)
+        while byte:
+            char = byte.decode('utf-8')
+            if char == Node.WordEndSymbol:
+                stack[-1].word_ends_here = True
+                tree.stored_words += 1
+            elif char == Node.TerminationSymbol:
+                stack.pop()
+            else:
+                new_node = Node(char)
+                stack[-1].child_nodes.append(new_node)
+                stack.append(new_node)
+            byte = reader.read(1)
+
+        return tree
+
+    def contains_word(self, word):
+        """
+        Checks whether a given word exists in the WordTree.
+
+        Parameters:
+        - word (str): The word to search for.
+
+        Returns:
+        - bool: True if the word exists in the tree, False otherwise.
+
+        Process:
+        1. Converts the word to uppercase for case-insensitive comparison.
+        2. Traverses the tree to find the sequence of characters in the word.
+        3. Returns False if any character is missing in the tree structure.
+        """
+        word = word.upper()
+        current_node = self
+        for char in word:
+            found_node = None
+            for child_node in current_node.child_nodes:
+                if child_node.value == char:
+                    current_node = child_node
+                    found_node = True
+                    break
+            if not found_node:
+                return False
+        return True
+
+    def to_list(self):
+        """
+        Converts all words stored in the WordTree into a list.
+
+        Returns:
+        - list: A list of all words stored in the tree.
+
+        Process:
+        1. Traverses the tree using a stack to collect characters.
+        2. Adds a word to the list whenever a node marks the end of a word.
+        """
+        list_of_words = []
+        stack = deque()
+
+        def add_node_to_list(node, stack):
+            stack.append(node.value)
+            if node.word_ends_here:
+                list_of_words.append(''.join(stack))
+            for child_node in node.child_nodes:
+                add_node_to_list(child_node, deque(stack))
+            stack.pop()
+
+        for node in self.child_nodes:
+            add_node_to_list(node, stack)
+
         return list_of_words

languagestatisticslibpy/test1.py

@@ -1,26 +1,26 @@
-'''
-   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-   Usage: python3 test1.py
-   test1.py is a minimal working example (MWE) which just needs the
-   package LanguageStatisticsLibPy to be installed.
-'''
-
-from languagestatisticslibpy.LanguageStatistics import LanguageStatistics as LS
-
-plaintext = LS.map_text_into_number_space("HELLOWORD", LS.alphabets['en'])
-ioc = LS.calculate_ioc(plaintext)
-
-print(ioc)
+'''
+   Copyright 2024 Nils Kopal, Bernhard Esslinger, CrypTool Team
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+   Usage: python3 test1.py
+   test1.py is a minimal working example (MWE) which just needs the
+   package LanguageStatisticsLibPy to be installed.
+'''
+
+from languagestatisticslibpy.LanguageStatistics import LanguageStatistics as LS
+
+plaintext = LS.map_text_into_number_space("HELLOWORD", LS.alphabets['en'])
+ioc = LS.calculate_ioc(plaintext)
+
+print(ioc)