data-manipulation 0.46__tar.gz → 0.48__tar.gz

{data_manipulation-0.46/data_manipulation.egg-info → data_manipulation-0.48}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_manipulation
-Version: 0.46
+Version: 0.48
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng

{data_manipulation-0.46 → data_manipulation-0.48}/data_manipulation/__init__.py

@@ -10,9 +10,8 @@ from data_manipulation import (
     mysql_connector_python_,
     openldap_,
     pandas_,
+    postgres_,
     prometheus_,
-    psycopg2_,
-    psycopg_,
     pyspark_,
     smtplib_,
     sqlalchemy_,

{data_manipulation-0.46 → data_manipulation-0.48}/data_manipulation/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2024-11-24T20:58:50+0800",
+ "date": "2024-12-02T10:43:01+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "8d4f86d6f48df8cc57789006ba9a8b8ee47ab4c2",
- "version": "0.46"
+ "full-revisionid": "773fa65b8cf0c0897344e60183a793418eca4432",
+ "version": "0.48"
 }
 '''  # END VERSION_JSON
 

{data_manipulation-0.46 → data_manipulation-0.48}/data_manipulation/base.py

@@ -2,25 +2,46 @@ import itertools
 import os
 import re
 import subprocess
-from typing import List
+from functools import wraps
+from pathlib import Path
+from typing import Any, Dict, List, Optional, Tuple, Union
+from urllib.parse import urlencode, urlparse, urlunparse
 
 from loguru import logger
 
 
+def deprecated(func):
+    """Decorator to mark functions as deprecated."""
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        logger.warning(
+            f"Function {func.__name__} is deprecated and will be removed in future versions"
+        )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
 # DATA STRUCTURE
 def clean_string(
-    string: str, remove_parenthesis: bool = False, remove_brackets: bool = False
+    string: str,
+    remove_parenthesis: bool = False,
+    remove_brackets: bool = False,
 ) -> str:
     """Cleans and standardizes input string.
 
     Args:
         string (str): String to clean.
-        remove_parenthesis (bool, optional): Whether to remove content within parentheses. Defaults to False.
-        remove_brackets (bool, optional): Whether to remove content within square brackets. Defaults to False.
+        remove_parenthesis (bool): Whether to remove content within parentheses.
+        remove_brackets (bool): Whether to remove content within square brackets.
 
     Returns:
         str: Uppercase cleaned string with standardized spacing.
 
+    Raises:
+        ValueError: If input string is None or empty.
+
     Examples:
         >>> clean_string(" sHawn  tesT ")
         'SHAWN TEST'
@@ -31,35 +52,37 @@ def clean_string(
         >>> clean_string("shawn [ te  st ]", remove_brackets=True)
         'SHAWN'
     """
+    if not string or not isinstance(string, str):
+        raise ValueError("Input must be a non-empty string")
 
     if remove_parenthesis:
         string = re.sub(r"\(.*\)", "", string)
     if remove_brackets:
         string = re.sub(r"\[.*\]", "", string)
 
-    string = string.strip().upper()
-    string = " ".join(string.split())
-    return string
+    return " ".join(string.strip().upper().split())
 
 
-def get_string_case_combination(str_: str) -> list:
+def get_string_case_combination(str_: str) -> List[str]:
     """Generates all possible case combinations of a string.
 
     Args:
         str_ (str): Input string to generate combinations for.
 
     Returns:
-        list: List of all possible case combinations.
+        List[str]: List of all possible case combinations.
 
     Examples:
         >>> get_string_case_combination("abc")
         ['ABC', 'ABc', 'AbC', 'Abc', 'aBC', 'aBc', 'abC', 'abc']
     """
+    if not str_:
+        raise ValueError("Input string cannot be empty or None")
 
     return list(map("".join, itertools.product(*zip(str_.upper(), str_.lower()))))
 
 
-def get_none_variation() -> list:
+def get_none_variation() -> List[Union[None, str]]:
     """Returns a list of common variations of None/null values.
 
     Returns:
@@ -126,14 +149,14 @@ def get_country_code_variation() -> dict:
     return variations
 
 
-def list_tuple_without_none(list_tuple: list | tuple) -> list | tuple:
+def list_tuple_without_none(list_tuple: Union[List, Tuple]) -> Union[List, Tuple]:
     """Removes None variations from a list or tuple.
 
     Args:
-        list_tuple (list | tuple): Input list or tuple to clean.
+        list_tuple: Input list or tuple to clean.
 
     Returns:
-        list | tuple: List or tuple with None variations removed.
+        Union[List, Tuple]: Cleaned list or tuple.
 
     Raises:
         TypeError: If input is not a list or tuple.
@@ -144,21 +167,20 @@ def list_tuple_without_none(list_tuple: list | tuple) -> list | tuple:
         >>> list_tuple_without_none(("a", "none"))
         ('a',)
     """
+    if not isinstance(list_tuple, (list, tuple)):
+        raise TypeError("Input must be a list or tuple")
+
     none_variations = get_none_variation()
+
     if isinstance(list_tuple, list):
-        lt = [item for item in list_tuple if item and item not in none_variations]
-    elif isinstance(list_tuple, tuple):
-        lt = tuple(item for item in list_tuple if item and item not in none_variations)
-    else:
-        raise TypeError("Wrong datatype(s)")
-    return lt
+        return [item for item in list_tuple if item and item not in none_variations]
+    return tuple(item for item in list_tuple if item and item not in none_variations)
 
 
+@deprecated
 def string_boolean_to_int(boolean_str_rep: str) -> int:
     """Converts string boolean representations to integers.
 
-    Deprecated: Will be removed in Python 3.12+
-
     Args:
         boolean_str_rep (str): String representation of boolean value.
 
@@ -171,20 +193,24 @@ def string_boolean_to_int(boolean_str_rep: str) -> int:
         >>> string_boolean_to_int("True")
         1
     """
-    from distutils.util import strtobool
+    try:
+        from distutils.util import strtobool
 
-    int_ = strtobool(string_str_to_str(boolean_str_rep))
-    return int_
+        return strtobool(string_str_to_str(boolean_str_rep))
+    except ValueError as e:
+        raise ValueError(
+            f"Invalid boolean string representation: {boolean_str_rep}"
+        ) from e
 
 
-def string_dlt_to_dlt(dlt_str_rep: str) -> dict | list | tuple:
+def string_dlt_to_dlt(dlt_str_rep: str) -> Union[Dict, List, Tuple]:
     """Converts string representation of data structures to actual Python objects.
 
     Args:
         dlt_str_rep (str): String representation of dictionary/list/tuple.
 
     Returns:
-        dict | list | tuple: Converted Python data structure.
+        Union[Dict, List, Tuple]: Converted Python data structure.
 
     Examples:
         >>> string_dlt_to_dlt("[1, 2, 3]")
@@ -194,10 +220,12 @@ def string_dlt_to_dlt(dlt_str_rep: str) -> dict | list | tuple:
         >>> string_dlt_to_dlt("('1', '2', '3')")
         ('1', '2', '3')
     """
-    from ast import literal_eval
+    try:
+        from ast import literal_eval
 
-    dlt = literal_eval(dlt_str_rep)
-    return dlt
+        return literal_eval(dlt_str_rep)
+    except (ValueError, SyntaxError) as e:
+        raise ValueError(f"Invalid data structure string: {dlt_str_rep}") from e
 
 
 def string_str_to_str(string_str_rep: str) -> str:
@@ -215,11 +243,15 @@ def string_str_to_str(string_str_rep: str) -> str:
         >>> string_str_to_str('"test"')
         'test'
     """
-    str_ = string_str_rep.strip("'\"")
-    return str_
+    if not string_str_rep:
+        raise ValueError("Input string cannot be empty or None")
+    return string_str_rep.strip("'\"")
 
 
-def delete_list_indices(list_: list, indices: list) -> None:
+def delete_list_indices(
+    list_: list,
+    indices: List[int],
+) -> None:
     """Deletes multiple indices from a list in-place.
 
     Args:
@@ -237,7 +269,10 @@ def delete_list_indices(list_: list, indices: list) -> None:
 
 
 # FILESYSTEM
-def get_path_files(path: str, keywords: list) -> list:
+def get_path_files(
+    path: Union[str, Path],
+    keywords: List[str],
+) -> List[str]:
     """Returns sorted list of files from given path that contain specified keywords.
 
     Args:
@@ -245,22 +280,29 @@ def get_path_files(path: str, keywords: list) -> list:
         keywords (list): List of keywords to match in filenames.
 
     Returns:
-        list: Sorted list of matching filenames.
+        List[str]: Sorted list of matching filenames.
 
     Examples:
         >>> get_path_files("test_base_folder", ["py"])
         ['test1.py', 'test2.py', 'test3.py', 'test4.py', 'test5.py']
     """
-
-    list_ = []
-    for file in sorted(os.listdir(path)):
-        if any(word in file for word in keywords):
-            list_.append(file)
-    return list_
+    path = Path(path)
+    if not path.exists():
+        raise FileNotFoundError(f"Path does not exist: {path}")
+
+    return sorted(
+        f.name
+        for f in path.iterdir()
+        if f.is_file() and any(word in f.name for word in keywords)
+    )
 
 
-def remove_path_file(path: str, keyword: str, n: int = 2) -> None:
-    """Removes all but the n newest files matching the keyword from the specified path.
+def remove_path_file(
+    path: Union[str, Path],
+    keyword: str,
+    n: int = 2,
+) -> None:
+    """Removes all but the n newest files matching the keyword.
 
     Args:
         path (str): Directory path.
@@ -270,14 +312,26 @@ def remove_path_file(path: str, keyword: str, n: int = 2) -> None:
     Examples:
         >>> remove_path_file("test_base_folder", ".py")
     """
+    if n < 0:
+        raise ValueError("n must be non-negative")
 
+    path = Path(path)
     to_delete = get_path_files(path=path, keywords=[keyword])[:-n]
-    for file in to_delete:
-        os.remove(f"{path}/{file}")
-        logger.info(f"{path}/{file} deleted ...")
 
-
-def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:
+    for file in to_delete:
+        try:
+            file_path = path / file
+            file_path.unlink()
+            logger.info(f"Deleted file: {file_path}")
+        except OSError as e:
+            logger.error(f"Failed to delete {file_path}: {e}")
+
+
+def list_to_file(
+    filepath: Union[str, Path],
+    list_: List,
+    newline: bool = True,
+) -> None:
     """Writes list contents to a file.
 
     Args:
@@ -288,16 +342,24 @@ def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:
     Examples:
         >>> list_to_file("test.txt", [1, 2, 3])
     """
-    f = open(filepath, "w")
-    for line in list_:
-        f.write(str(line))
-        if newline:
-            f.write("\n")
-    f.close()
+    filepath = Path(filepath)
+    try:
+        with filepath.open("w") as f:
+            for line in list_:
+                f.write(str(line))
+                if newline:
+                    f.write("\n")
+        logger.info(f"Successfully wrote to file: {filepath}")
+    except IOError as e:
+        logger.error(f"Failed to write to file {filepath}: {e}")
+        raise
 
 
 # URLLIB
-def create_encode_url(url: str, query_params: dict = {}) -> str:
+def create_encode_url(
+    url: str,
+    query_params: Optional[Dict[str, Any]] = None,
+) -> str:
     """Creates an encoded URL with query parameters.
 
     Args:
@@ -306,67 +368,83 @@ def create_encode_url(url: str, query_params: dict = {}) -> str:
 
     Returns:
         str: Encoded URL with query parameters.
-    """
-    from urllib.parse import urlencode
 
-    return f"{url}{urlencode(query_params)}"
+    Raises:
+        ValueError: If URL is invalid.
+    """
+    query_params = query_params or {}
+
+    try:
+        parsed = urlparse(url)
+        if not parsed.scheme or not parsed.netloc:
+            raise ValueError("Invalid URL format")
+
+        return urlunparse(
+            (
+                parsed.scheme,
+                parsed.netloc,
+                parsed.path,
+                parsed.params,
+                urlencode(query_params),
+                parsed.fragment,
+            )
+        )
+    except Exception as e:
+        logger.error(f"Failed to encode URL {url}: {e}")
+        raise
 
 
 # SYSTEM
-def parse_ps_aux(ps_aux_commands: str) -> List[list]:
+def parse_ps_aux(ps_aux_commands: str) -> List[List[str]]:
     """Parses Linux ps aux command output into a list of records.
 
     Args:
         ps_aux_commands (str): Linux ps aux command string.
 
     Returns:
-        List[list]: List of lists, where each inner list represents a process record.
+        List[List[str]]: List of process records.
 
     Examples:
         >>> # parse_ps_aux("ps aux | egrep -i '%cpu|anaconda3' | head")
     """
-    output = subprocess.run(
-        ps_aux_commands,
-        capture_output=True,
-        text=True,
-        shell=True,
-        executable="/bin/bash",
-    )
-    lines = output.stdout.split("\n")
-    n_columns = len(lines[0].split()) - 1
-    rows = [line.split(None, n_columns) for line in lines if line]
-    return rows
+    try:
+        output = subprocess.run(
+            ps_aux_commands,
+            capture_output=True,
+            text=True,
+            shell=True,
+            executable="/bin/bash",
+            check=True,
+        )
+        lines = output.stdout.strip().split("\n")
+        if not lines:
+            return []
+
+        n_columns = len(lines[0].split()) - 1
+        return [line.split(None, n_columns) for line in lines if line]
+    except subprocess.SubprocessError as e:
+        logger.error(f"Failed to execute command: {e}")
+        raise
 
 
 if __name__ == "__main__":
     import doctest
 
-    subprocess.run(
-        "mkdir -p test_base_folder",
-        capture_output=True,
-        shell=True,
-        text=True,
-        executable="/bin/bash",
-    )
-    subprocess.run(
-        "touch test_base_folder/test{1..5}.py",
-        capture_output=True,
-        shell=True,
-        text=True,
-        executable="/bin/bash",
-    )
-    doctest.testmod()
-    subprocess.run(
-        "rm -rf test_base_folder",
-        capture_output=True,
-        shell=True,
-        text=True,
-        executable="/bin/bash",
-    )
-    subprocess.run(
-        "rm test.txt",
-        capture_output=True,
-        shell=True,
-        text=True,
-        executable="/bin/bash",
-    )
+    # Setup test environment
+    test_folder = Path("test_base_folder")
+    try:
+        test_folder.mkdir(exist_ok=True)
+        for i in range(1, 6):
+            (test_folder / f"test{i}.py").touch()
+
+        doctest.testmod()
+    finally:
+        # Cleanup
+        if test_folder.exists():
+            for file in test_folder.iterdir():
+                file.unlink()
+            test_folder.rmdir()
+
+        test_file = Path("test.txt")
+        if test_file.exists():
+            test_file.unlink()

{data_manipulation-0.46 → data_manipulation-0.48}/data_manipulation/beautifulsoup_.py

@@ -1,8 +1,17 @@
 import re
-from typing import Optional
+from typing import TYPE_CHECKING, Optional
 
 from loguru import logger
 
+if TYPE_CHECKING:
+    from bs4 import BeautifulSoup
+
+# Constants
+DEFAULT_TIMEOUT = 10
+DEFAULT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+}
+
 
 def preprocess(html: str) -> Optional[str]:
     """Removes whitespaces and newline characters from HTML string.
@@ -34,13 +43,21 @@ def preprocess(html: str) -> Optional[str]:
     return html
 
 
-def build_soup(url: str, features: str = "lxml", to_preprocess: bool = True):
+def build_soup(
+    url: str,
+    features: str = "lxml",
+    to_preprocess: bool = True,
+    timeout: int = DEFAULT_TIMEOUT,
+    headers: Optional[dict] = None,
+) -> "Optional[BeautifulSoup]":
     """Creates a BeautifulSoup object from a given URL.
 
     Args:
         url (str): URL to fetch and parse.
         features (str, optional): Parser to use. Defaults to "lxml".
         to_preprocess (bool, optional): Whether to preprocess the HTML. Defaults to True.
+        timeout (int, optional): Request timeout in seconds. Defaults to 10.
+        headers (Optional[dict], optional): Custom headers for the request. Defaults to None.
 
     Returns:
         Optional[BeautifulSoup]: Parsed BeautifulSoup object, or None if request fails.
@@ -56,15 +73,25 @@ def build_soup(url: str, features: str = "lxml", to_preprocess: bool = True):
     import requests
     from bs4 import BeautifulSoup
 
-    response = requests.get(url)
-    if response.status_code == 200:
-        if to_preprocess:
-            soup = BeautifulSoup(preprocess(response.text), features=features)
-        else:
-            soup = BeautifulSoup(response.text, features=features)
-        return soup
-    else:
-        logger.error(f"response status code: {response.status_code}")
+    if not url or not url.strip():
+        raise ValueError("URL cannot be empty")
+
+    if not url.startswith(("http://", "https://")):
+        raise ValueError("URL must start with http:// or https://")
+
+    request_headers = headers or DEFAULT_HEADERS
+
+    try:
+        with requests.Session() as session:
+            response = session.get(url, headers=request_headers, timeout=timeout)
+            response.raise_for_status()
+
+            html_content = preprocess(response.text) if to_preprocess else response.text
+            return BeautifulSoup(html_content, features=features)
+
+    except requests.RequestException as e:
+        logger.error(f"Failed to fetch URL {url}: {str(e)}")
+        return None
 
 
 if __name__ == "__main__":