lionagi 0.0.115__py3-none-any.whl → 0.0.204__py3-none-any.whl

lionagi/utils/encrypt_util.py

@@ -1,16 +1,283 @@
-# this module has no internal dependency 
-from cryptography.fernet import Fernet
-
-def generate_encryption_key() -> str:
-    """Generates a key for encryption."""
-    return Fernet.generate_key().decode()
-
-def encrypt(data: str, key: str) -> str:
-    """Encrypts data using the provided key."""
-    fernet = Fernet(key.encode())
-    return fernet.encrypt(data.encode()).decode()
-
-def decrypt(data: str, key: str) -> str:
-    """Decrypts data using the provided key."""
-    fernet = Fernet(key.encode())
-    return fernet.decrypt(data.encode()).decode()
+import base64
+import binascii
+import hashlib
+import os
+import zipfile
+from base64 import urlsafe_b64encode
+from cryptography.fernet import Fernet, InvalidToken
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+from typing import Optional
+
+
+class EncrytionUtil:
+    """
+    A utility class for handling encryption, decryption, file operations, and password strength checking.
+    """
+
+    @staticmethod
+    def password_strength_checker(password: str) -> bool:
+        """
+        Check the strength of a password.
+
+        Args:
+            password (str): The password to check.
+
+        Returns:
+            bool: True if the password is strong, False otherwise.
+
+        Examples:
+            >>> password_strength_checker("Weakpass")
+            False
+            >>> password_strength_checker("Strongpass1")
+            True
+        """
+        if len(password) < 8 or not any(char.isdigit() for char in password) or not any(char.isupper() for char in password):
+            return False
+        return True
+
+    @staticmethod
+    def generate_encryption_key(password: Optional[str] = None, salt: Optional[bytes] = None) -> str:
+        """
+        Generate an encryption key from a password and salt.
+
+        Args:
+            password (Optional[str]): The password to derive the key from. If None, a random key is generated.
+            salt (Optional[bytes]): A salt for the key derivation. If None, a random salt is used.
+
+        Returns:
+            str: The generated encryption key as a URL-safe base64-encoded string.
+
+        Raises:
+            ValueError: If the password is too weak.
+
+        Examples:
+            >>> key = generate_encryption_key("Strongpass1")
+            >>> isinstance(key, str)
+            True
+        """
+        if password:
+            if not EncrytionUtil.password_strength_checker(password):
+                raise ValueError("Password is too weak.")
+            if not salt:
+                salt = os.urandom(16)
+            kdf = PBKDF2HMAC(
+                algorithm=hashes.SHA256(),
+                length=32,
+                salt=salt,
+                iterations=100000,
+                backend=default_backend()
+            )
+            key = kdf.derive(password.encode())
+            return urlsafe_b64encode(key).decode()
+        else:
+            return Fernet.generate_key().decode()
+
+    @staticmethod
+    def encrypt(data: str, key: str) -> str:
+        """
+        Encrypt the provided data using the provided key.
+
+        Args:
+            data (str): The data to encrypt.
+            key (str): The encryption key.
+
+        Returns:
+            str: The encrypted data.
+
+        Example:
+            >>> encrypt("This is some test data.", "esvCExTuhddRb8kSobU_gnNRYObyTRTI2LJF4nYai5I=")
+            'gAAAAABloX9m_S_G1VUtbfGUDB7ooHJNt8sPiSCwnp1ehe6dExvMEqtw_ua2ELk_uUbLoB6a1XkbKLOkM4UBvwmk6sMoY-yNvE-Lv-w-VNnfzf89zH82rgI='
+        """
+        fernet = Fernet(key.encode())
+        return fernet.encrypt(data.encode()).decode()
+
+    @staticmethod
+    def decrypt(data: str, key: str) -> str:
+        """
+        Decrypt the provided data using the provided key.
+
+        Args:
+            data (str): The data to decrypt.
+            key (str): The encryption key.
+
+        Returns:
+            str: The decrypted data.
+
+        Example:
+            >>> decrypt('gAAAAABloX9m_S_G1VUtbfGUDB7ooHJNt8sPiSCwnp1ehe6dExvMEqtw_ua2ELk_uUbLoB6a1XkbKLOkM4UBvwmk6sMoY-yNvE-Lv-w-VNnfzf89zH82rgI=', "esvCExTuhddRb8kSobU_gnNRYObyTRTI2LJF4nYai5I=")
+            'This is some test data.'
+        """
+        fernet = Fernet(key.encode())
+        return fernet.decrypt(data.encode()).decode()
+
+    @staticmethod
+    def encrypt_file(file_path: str, key: str, output_path: str = None):
+        """
+        Encrypt a file.
+
+        Args:
+            file_path (str): The path of the file to encrypt.
+            key (str): The encryption key.
+            output_path (str, optional): The path to save the encrypted file. If not provided, '.enc' is appended to the input file path.
+
+        Example:
+            >>> encrypt_file("test_file.txt", "esvCExTuhddRb8kSobU_gnNRYObyTRTI2LJF4nYai5I=")
+        """
+        if not output_path:
+            output_path = file_path + '.enc'
+        with open(file_path, 'rb') as file_to_encrypt:
+            encrypted_data = EncrytionUtil.encrypt(file_to_encrypt.read().decode(), key)
+        with open(output_path, 'wb') as encrypted_file:
+            encrypted_file.write(encrypted_data.encode())
+
+    @staticmethod
+    def decrypt_file(encrypted_file_path: str, key: str, output_path: str = None):
+        """
+        Decrypt a file.
+
+        Args:
+            encrypted_file_path (str): The path of the file to decrypt.
+            key (str): The encryption key.
+            output_path (str, optional): The path to save the decrypted file. If not provided, '.enc' is removed from the input file path.
+
+        Example:
+            >>> decrypt_file("test_file.txt.enc", "esvCExTuhddRb8kSobU_gnNRYObyTRTI2LJF4nYai5I=")
+        """
+        if not output_path:
+            output_path = encrypted_file_path.replace('.enc', '')
+        with open(encrypted_file_path, 'rb') as encrypted_file:
+            decrypted_data = EncrytionUtil.decrypt(encrypted_file.read().decode(), key)
+        with open(output_path, 'wb') as decrypted_file:
+            decrypted_file.write(decrypted_data.encode())
+
+    @staticmethod
+    def is_encrypted(file_path: str, key: str) -> bool:
+        """
+        Check if a file is encrypted.
+
+        Args:
+            file_path (str): The path of the file to check.
+            key (str): The encryption key.
+
+        Returns:
+            bool: True if the file is encrypted, False otherwise.
+
+        Example:
+            >>> is_encrypted("test_file.txt.enc", "esvCExTuhddRb8kSobU_gnNRYObyTRTI2LJF4nYai5I=")
+            True
+        """
+        try:
+            EncrytionUtil.decrypt_file(file_path, key, "temp_decrypted_file")
+            os.remove("temp_decrypted_file")
+            return True
+        except InvalidToken:
+            return False
+
+    @staticmethod
+    def decompress_file(file_path: str, output_path: str = None):
+        """
+        Decompress a file.
+
+        Args:
+            file_path (str): The path of the file to decompress.
+            output_path (str, optional): The path to save the decompressed file. If not provided, the file is decompressed in the same directory.
+
+        Example:
+            >>> decompress_file("test_file.txt.zip")
+        """
+        if not output_path:
+            output_path = os.path.dirname(file_path)
+        with zipfile.ZipFile(file_path, 'r') as zipf:
+            zipf.extractall(output_path)
+
+    @staticmethod
+    def compress_file(file_path: str, output_path: str = None):
+        """
+        Compress a file.
+
+        Args:
+            file_path (str): The path of the file to compress.
+            output_path (str, optional): The path to save the compressed file. If not provided, '.zip' is appended to the input file path.
+
+        Example:
+            >>> compress_file("test_file.txt")
+        """
+        if not output_path:
+            output_path = file_path + '.zip'
+        with zipfile.ZipFile(output_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+            zipf.write(file_path)
+
+    @staticmethod
+    def binary_to_hex(data: bytes) -> str:
+        """
+        Convert binary data to a hexadecimal string representation.
+        
+        Args:
+            data: A bytes object containing binary data.
+
+        Returns:
+            A string containing the hexadecimal representation of the binary data.
+
+        Examples:
+            >>> binary_to_hex(b'\x00\x0F')
+            '000f'
+            >>> binary_to_hex(b'hello')
+            '68656c6c6f'
+        """
+        return binascii.hexlify(data).decode()
+
+    @staticmethod
+    def create_hash(data: str, algorithm: str = 'sha256') -> str:
+        """
+        Create a hash of the given data using the specified algorithm.
+
+        Args:
+            data: The string to hash.
+            algorithm: The hashing algorithm to use (default is 'sha256').
+
+        Returns:
+            The hexadecimal digest of the hash.
+
+        Examples:
+            >>> create_hash('hello')
+            '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'
+        """
+        hasher = hashlib.new(algorithm)
+        hasher.update(data.encode())
+        return hasher.hexdigest()
+
+    @staticmethod
+    def decode_base64(data: str) -> str:
+        """
+        Decode a base64 encoded string.
+
+        Args:
+            data: A base64 encoded string.
+
+        Returns:
+            A decoded string.
+
+        Examples:
+            >>> decode_base64('SGVsbG8sIFdvcmxkIQ==')
+            'Hello, World!'
+        """
+        return base64.b64decode(data).decode()
+
+    @staticmethod
+    def encode_base64(data: str) -> str:
+        """
+        Encode a string using base64 encoding.
+
+        Args:
+            data: A string to be encoded.
+
+        Returns:
+            A base64 encoded string.
+
+        Examples:
+            >>> encode_base64("Hello, World!")
+            'SGVsbG8sIFdvcmxkIQ=='
+        """
+        return base64.b64encode(data.encode()).decode()

lionagi/utils/io_util.py

@@ -1,102 +1,187 @@
-# used flat_util
 import csv
 import json
 import os
 import tempfile
+from collections.abc import Iterable
 from typing import Any, Dict, List
-from .flat_util import to_list
 
-
-def to_temp(input: Any, 
-            flatten_dict: bool = False, 
-            flat: bool = False, 
-            dropna: bool = False):
+class IOUtil:
     """
-    Converts input to a list and writes it to a temporary file in JSON format, with flattening options.
-
-    This function serializes data to a temporary JSON file, useful for transient storage or testing. 
-    It includes options to flatten the input if it contains dictionaries or lists.
-
-    Parameters:
-        input (Any): The data to be converted and written to a file.
-
-        flatten_dict (bool, optional): Flatten dictionaries in the input. Defaults to False.
-
-        flat (bool, optional): Flatten lists in the input. Defaults to False.
-
-        dropna (bool, optional): Exclude 'None' values during flattening. Defaults to False.
-
-    Raises:
-        TypeError: If the input is not JSON serializable.
-
-    Example:
-        >>> temp_file = to_temp({'a': 1, 'b': [2, 3]}, flatten_dict=True)
-        >>> temp_file.name  # Doctest: +ELLIPSIS
-        '/var/folders/.../tmp...'
+    Utility class for reading and writing data from/to files in various formats.
     """
-    input = to_list(input, flatten_dict, flat, dropna)
     
-    temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
-    try:
-        json.dump(input, temp_file)
-    except TypeError as e:
-        temp_file.close()  # Ensuring file closure before raising error
-        raise TypeError(f"Data provided is not JSON serializable: {e}")
-    temp_file.close()
-    return temp_file
-
-def to_csv(input: List[Dict[str, Any]]=None,
-           filepath: str=None,
-           file_exist_ok: bool = False) -> None:
-    """
-    Writes a list of dictionaries to a CSV file, with dictionary keys as headers.
-
-    This function writes a list of dictionaries to a CSV file. It checks if the file exists 
-    and handles file creation based on the 'file_exist_ok' flag.
-
-    Parameters:
-        input (List[Dict[str, Any]]): Data to write to the CSV file.
-
-        filepath (str): Path of the output CSV file.
-
-        file_exist_ok (bool, optional): Create the file if it doesn't exist. Defaults to False.
-
-    Raises:
-        FileExistsError: If the file already exists and 'file_exist_ok' is False.
-
-    Example:
-        >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
-        >>> to_csv(data, 'people.csv')
-    """
-
-    if not os.path.exists(os.path.dirname(filepath)) and os.path.dirname(filepath) != '':
-        if file_exist_ok:
-            os.makedirs(os.path.dirname(filepath), exist_ok=True)
-        else:
-            raise FileNotFoundError(f"The directory {os.path.dirname(filepath)} does not exist.")
-
-    with open(filepath, 'w', newline='') as csv_file:
-        writer = csv.DictWriter(csv_file, fieldnames=input[0].keys())
-        writer.writeheader()
-        writer.writerows(input)
-
-def append_to_jsonl(data: Any, filepath: str) -> None:
-    """
-    Appends data to a JSON lines (jsonl) file.
-
-    Serializes given data to a JSON-formatted string and appends it to a jsonl file. 
-    Useful for logging or data collection where entries are added incrementally.
-
-    Parameters:
-        data (Any): Data to be serialized and appended.
-
-        filepath (str): Path to the jsonl file.
-
-    Example:
-        >>> append_to_jsonl({"key": "value"}, "data.jsonl")
-        # Appends {"key": "value"} to 'data.jsonl'
-    """
-    json_string = json.dumps(data)
-    with open(filepath, "a") as f:
-        f.write(json_string + "\n")
-        
+    @staticmethod
+    def read_csv(filepath: str) -> List[Dict[str, Any]]:
+        """
+        Reads a CSV file and returns its contents as a list of dictionaries.
+
+        Args:
+            filepath (str): The path to the CSV file to be read.
+
+        Returns:
+            List[Dict[str, Any]]: A list of dictionaries, each representing a row in the CSV file.
+        """
+        with open(filepath, 'r', newline='') as csv_file:
+            reader = csv.DictReader(csv_file)
+            return list(reader)
+
+    @staticmethod
+    def read_jsonl(filepath: str) -> List[Any]:
+        """
+        Reads a JSON Lines file and returns its contents as a list.
+
+        Args:
+            filepath (str): The path to the JSON Lines file to be read.
+
+        Returns:
+            List[Any]: A list where each element is a JSON object from a line in the file.
+        """
+        with open(filepath, 'r') as f:
+            return [json.loads(line) for line in f]
+
+    @staticmethod
+    def write_json(data: List[Dict[str, Any]], filepath: str) -> None:
+        """
+        Writes a list of dictionaries to a JSON file.
+
+        Args:
+            data (List[Dict[str, Any]]): The data to be written to the JSON file.
+            filepath (str): The path where the JSON file will be saved.
+
+        Returns:
+            None: This function does not return anything.
+        """
+        with open(filepath, 'w') as json_file:
+            json.dump(data, json_file, indent=4)
+
+    @staticmethod
+    def read_json(filepath: str) -> Any:
+        """
+        Reads a JSON file and returns its content.
+
+        Args:
+            filepath (str): The path to the JSON file to be read.
+
+        Returns:
+            Any: The content of the JSON file.
+        """
+        with open(filepath, 'r') as json_file:
+            return json.load(json_file)
+
+    @staticmethod
+    def merge_csv_files(filepaths: List[str], output_filepath: str) -> None:
+        """
+        Merges multiple CSV files into a single CSV file.
+
+        Args:
+            filepaths (List[str]): A list of file paths to the CSV files to be merged.
+            output_filepath (str): The path where the merged CSV file will be saved.
+
+        Returns:
+            None: This function does not return anything.
+        """
+        merged_data = []
+        fieldnames = set()
+        for filepath in filepaths:
+            with open(filepath, 'r', newline='') as csv_file:
+                reader = csv.DictReader(csv_file)
+                if reader.fieldnames is not None:
+                    fieldnames.update(reader.fieldnames)
+                    for row in reader:
+                        merged_data.append(row)
+        fieldnames = list(fieldnames)
+        with open(output_filepath, 'w', newline='') as csv_file:
+            writer = csv.DictWriter(csv_file, fieldnames=fieldnames)
+            writer.writeheader()
+            writer.writerows(merged_data)
+
+    @staticmethod
+    def to_csv(input: List[Dict[str, Any]],
+            filepath: str,
+            file_exist_ok: bool = False) -> None:
+        """
+        Writes a list of dictionaries to a CSV file.
+
+        Args:
+            input: A list of dictionaries where each dictionary represents a row in the CSV.
+            filepath: The path to the CSV file to write to.
+            file_exist_ok: If True, creates the directory for the file if it does not exist.
+
+        Raises:
+            FileNotFoundError: If the directory does not exist and file_exist_ok is False.
+
+        Examples:
+            >>> data = [{'name': 'Alice', 'age': 30}, {'name': 'Bob', 'age': 25}]
+            >>> to_csv(data, 'people.csv', file_exist_ok=True)
+        """
+        if not input:
+            return
+        if not os.path.exists(os.path.dirname(filepath)) and os.path.dirname(filepath) != '':
+            if file_exist_ok:
+                os.makedirs(os.path.dirname(filepath), exist_ok=True)
+            else:
+                raise FileNotFoundError(f"The directory {os.path.dirname(filepath)} does not exist.")
+
+        with open(filepath, 'w', newline='') as csv_file:
+            writer = csv.DictWriter(csv_file, fieldnames=input[0].keys())
+            writer.writeheader()
+            for row in input:
+                writer.writerow(row)
+
+    @staticmethod
+    def append_to_jsonl(data: Any, filepath: str) -> None:
+        """
+        Appends a data entry to a JSON Lines file.
+
+        Args:
+            data: The data to append, which can be any JSON serializable object.
+            filepath: The path to the JSON Lines file to append to.
+
+        Examples:
+            >>> append_to_jsonl({'name': 'Charlie', 'age': 35}, 'people.jsonl')
+        """
+        json_string = json.dumps(data)
+        with open(filepath, "a") as f:
+            f.write(json_string + "\n")
+            
+    @staticmethod
+    def to_temp(input: Any) -> tempfile.NamedTemporaryFile:
+        """
+        Writes the given input to a temporary file in JSON format.
+
+        Args:
+            input: The input data to write to the file. Can be a string or an iterable.
+
+        Returns:
+            A NamedTemporaryFile object representing the temporary file.
+
+        Raises:
+            TypeError: If the input data is not JSON serializable.
+
+        Examples:
+            >>> temp_file = to_temp("test string")
+            >>> with open(temp_file.name, 'r') as file:
+            ...     content = json.load(file)
+            >>> content
+            ["test string"]
+
+            >>> temp_file = to_temp(["test", "string"])
+            >>> with open(temp_file.name, 'r') as file:
+            ...     content = json.load(file)
+            >>> content
+            ["test", "string"]
+        """
+        if isinstance(input, str):
+            input = [input]
+        elif isinstance(input, Iterable):
+            input = [item for item in input if item is not None]
+
+        temp_file = tempfile.NamedTemporaryFile(mode='w', delete=False)
+        try:
+            json.dump(input, temp_file)
+        except TypeError as e:
+            temp_file.close()  # Ensuring file closure before raising error
+            raise TypeError(f"Data provided is not JSON serializable: {e}")
+        temp_file.close()
+        return temp_file
+