data-manipulation 0.36__tar.gz → 0.37__tar.gz

{data_manipulation-0.36 → data_manipulation-0.37}/LICENSE

@@ -1,19 +1,19 @@
 BSD 3-Clause License
 
-Copyright (c) 2020, Ng Teng Qing (shawnngtq)
+Copyright (c) 2020, Shawn Ng (shawnngtq)
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-* Redistributions of source code must retain the above copyright notice, this
+- Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.
 
-* Redistributions in binary form must reproduce the above copyright notice, this
+- Redistributions in binary form must reproduce the above copyright notice, this
   list of conditions and the following disclaimer in the documentation and/or
   other materials provided with the distribution.
 
-* Neither the name of data_manipulation nor the names of its
+- Neither the name of data_manipulation nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.
 

{data_manipulation-0.36 → data_manipulation-0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_manipulation
-Version: 0.36
+Version: 0.37
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -19,6 +19,16 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: django
+Requires-Dist: lxml
+Requires-Dist: matplotlib
+Requires-Dist: mysql-connector-python
+Requires-Dist: pandas
+Requires-Dist: psycopg2
+Requires-Dist: pyspark
+Requires-Dist: tqdm
+Requires-Dist: xlsxwriter
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation/__init__.py

@@ -1,13 +1,16 @@
 from data_manipulation import (
     base,
     beautifulsoup_,
+    boto3_,
     cryptography_,
     django_,
+    flask_,
     geopandas_,
     kerberos_,
     mysql_connector_python_,
     openldap_,
     pandas_,
+    prometheus_,
     psycopg2_,
     pyspark_,
     smtplib_,

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-03T09:27:54+0800",
+ "date": "2023-12-30T10:16:23+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "263a1314653b0a2f32b9e1eea3efbf78ad990ca2",
- "version": "0.36"
+ "full-revisionid": "6f824ae7f46ddf5b2f60190bacfc2e4118d7f80f",
+ "version": "0.37"
 }
 '''  # END VERSION_JSON
 

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation/base.py

@@ -109,6 +109,7 @@ def get_country_name_variation() -> dict:
         "SLAVONIC": "SK",
         "SLOVAK REPUBLIC": "SK",
         "TURKEY": "TR",
+        "TURKIYE": "TR",
         "UNITED STATES": "US",
     }
     return variations
@@ -363,6 +364,28 @@ def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:
     f.close()
 
 
+# URLLIB
+def create_encode_url(url: str, query_params: dict = {}) -> str:
+    """
+    Create encoded url
+
+    Parameters
+    ----------
+    url : str
+        base url
+    query_params : dict, optional
+        dictionary of url query parameters, by default {}
+
+    Returns
+    -------
+    str
+        encoded url
+    """
+    from urllib.parse import urlencode
+
+    return f"{url}{urlencode(query_params)}"
+
+
 # SYSTEM
 def parse_ps_aux(ps_aux_commands: str) -> List[list]:
     """

data_manipulation-0.37/data_manipulation/boto3_.py

@@ -0,0 +1,82 @@
+def send_aws_ses_email(
+    sender: str,
+    recipient: list,
+    subject: str,
+    body_text: str,
+    body_type: str,
+    ses_client,
+    attachment: str = None,
+):
+    """
+    Send AWS SES email
+
+    Parameters
+    ----------
+    sender : str
+        sender email
+    recipient : list
+        list of recipient emails
+    subject : str
+        email subject
+    body_text : str
+        email body
+    body_type : str
+        email body type
+    ses_client : _type_
+        aws ses client
+    attachment : str, optional
+        attachment path, by default None
+
+    Returns
+    -------
+    str
+        aws ses client email response or none
+    """
+    import os
+    from email.mime.application import MIMEApplication
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+
+    from botocore.exceptions import BotoCoreError, ClientError
+
+    msg = MIMEMultipart()
+    msg["Subject"] = subject
+    msg["From"] = sender
+    msg["To"] = ", ".join(recipient)
+
+    part = MIMEText(body_text, body_type)
+    msg.attach(part)
+
+    if attachment:
+        try:
+            with open(attachment, "rb") as f:
+                part = MIMEApplication(f.read())
+                part.add_header(
+                    "Content-Disposition",
+                    "attachment",
+                    filename=os.path.basename(attachment),
+                )
+                msg.attach(part)
+        except FileNotFoundError:
+            print(f"{attachment} not found")
+            return
+
+    try:
+        response = ses_client.send_raw_email(
+            Source=sender,
+            Destinations=recipient,
+            RawMessage={
+                "Data": msg.as_string(),
+            },
+        )
+    except (BotoCoreError, ClientError) as error:
+        print(f"Error: {error}")
+        return
+    else:
+        return response
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

data_manipulation-0.37/data_manipulation/django_.py

@@ -0,0 +1,123 @@
+def init_django(django_dir: str, project_name: str) -> None:
+    """
+    Setup Django in Jupyter. Reference from https://gist.github.com/codingforentrepreneurs/76e570d759f83d690bf36a8a8fa4cfbe
+
+    Parameters
+    ----------
+    django_dir : str
+        Django project location
+    project_name : str
+        Django project name
+    """
+    import os
+    import sys
+
+    import django
+
+    # django_dir = os.getenv("PWD")
+    # os.chdir(django_dir)
+
+    project_name = project_name or os.environ.get("DJANGO_PROJECT") or None
+    if not project_name:
+        raise Exception(
+            """Set an environment variable: `DJANGO_PROJECT=your_project_name` or call `init_django(your_project_name)`"""
+        )
+
+    sys.path.insert(0, django_dir)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", f"{project_name}.settings")
+    os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+    django.setup()
+
+
+def get_django_countries_dict():
+    """
+    Get django_countries package's dictionaries
+        code_name - "SG": "SINGAPORE"
+        name_code - "SINGAPORE": "SG"
+    """
+    from django_countries import countries
+
+    code_name = {k: v.upper() for k, v in dict(countries).items()}
+    name_code = {v: k for k, v in code_name.items()}
+    return code_name, name_code
+
+
+def django_validate_email(email: str) -> str | None:
+    """
+    Django validate email
+
+    Parameters
+    ----------
+    email : str
+        email to check
+
+    Returns
+    -------
+    str | None
+        email or none
+    """
+    from django.core.validators import validate_email
+
+    try:
+        validate_email(email)
+        return email
+    except Exception as e:
+        return
+
+
+def django_validate_url(url: str) -> str | None:
+    """
+    Django validate url
+
+    Parameters
+    ----------
+    url : str
+        url to validate
+
+    Returns
+    -------
+    str | None
+        url or None
+    """
+    from django.core.validators import URLValidator
+
+    validator = URLValidator()
+    try:
+        validator(url)
+        return url
+    except Exception as e:
+        return
+
+
+def django_validate_phone(phone: str, region=None) -> str | None:
+    """
+    Django validate phone
+
+    Parameters
+    ----------
+    phone : str
+        phone to check
+    region : _type_, optional
+        phone region, by default None
+
+    Returns
+    -------
+    str | None
+        parsed phone or none
+    """
+    from phonenumber_field.phonenumber import PhoneNumber
+
+    try:
+        return PhoneNumber.from_string(phone).as_e164
+    except:
+        try:
+            return PhoneNumber.from_string(phone, region=region).as_e164
+        except Exception as e:
+            print(e)
+            return
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation/pandas_.py

@@ -364,15 +364,56 @@ def split_left_merged_dataframe(dataframe, dataframe2, columns):
     return df_both, df_left
 
 
+## AGGREGATE
+def aggregate_set_without_none(column: pd.Series, nested_set: bool = False) -> set:
+    """
+    Use in pandas groupby agg, create a set without none based on input column
+
+    Parameters
+    ----------
+    column : pd.Series
+        columns to set
+    nested_set : bool, optional
+        nested set within set, by default False
+
+    Returns
+    -------
+    set
+        set without None
+    """
+    if nested_set:
+        output = set()
+        for value in column:
+            if isinstance(value, set):
+                output.update(value)
+            # todo, what if there is valid list / dict / tuple / complex object?
+            elif value:
+                output.add(value)
+        return output
+    else:
+        return {value for value in column if value is not None}
+
+
 # DATA STRUCTURE
-def clean_none(dataframe, clean_variation=True, non_variations=[]):
+def clean_none(
+    dataframe: pd.DataFrame,
+    nan_to_none: bool = True,
+    clean_variation: bool = True,
+    none_variations: list = [],
+) -> pd.DataFrame:
     """
     Return a dataframe from given dataframe with standardized None. Deprecated as of pandas 1.3.0.
 
     Parameters
     ----------
-    dataframe : pandas.DataFrame
-        Dataframe
+    dataframe : pd.DataFrame
+        _description_
+    nan_to_none : bool, optional
+        replace np.NaN with None, by default True
+    clean_variation : bool, optional
+        clean custom variables, by default True
+    none_variations : list, optional
+        list of none variations, by default []
 
     Examples
     --------
@@ -410,12 +451,15 @@ def clean_none(dataframe, clean_variation=True, non_variations=[]):
 
     Returns
     -------
-    df : pandas.DataFrame
+    pd.DataFrame
+        _description_
     """
     df = dataframe.copy()
     df = df.replace(r"^\s*$", np.nan, regex=True)
+    if nan_to_none:
+        df = df.replace(np.NaN, None, inplace=True)
     if clean_variation:
-        df = df.replace(non_variations, np.nan)
+        df = df.replace(none_variations, np.nan)
     df = df.where(pd.notnull(df), None)
     return df
 

data_manipulation-0.37/data_manipulation/prometheus_.py

@@ -0,0 +1,26 @@
+# import logging
+# import subprocess
+
+# def ldapsearch(
+#     search_base: str,
+#     ldap_uri: str,
+#     bind_dn: str,
+#     password: str,
+#     filter: str,
+# ) -> subprocess.CompletedProcess:
+#     cmd = (
+#         "ldapsearch -b {search_base} -H {ldap_uri} -D {bind_dn} -w {password} {filter}"
+#     )
+#     output = subprocess.run(
+#         cmd,
+#         capture_output=True,
+#         shell=True,
+#         text=True,
+#     )
+#     return output
+
+
+# if __name__ == "__main__":
+#     import doctest
+
+#     doctest.testmod()

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation/sqlalchemy_.py

@@ -45,7 +45,7 @@ def create_sqlalchemy_url(
     return url
 
 
-def check_connection_valid(
+def create_sqlalchemy_engine(
     drivername: str,
     host: str,
     dbname: str,
@@ -54,7 +54,7 @@ def check_connection_valid(
     port=3306,
 ) -> sqlalchemy.engine.base.Engine:
     """
-    Check mysql connection validity
+    Create SQLalchemy engine
 
     Parameters
     ----------
@@ -87,9 +87,9 @@ def check_connection_valid(
     engine = sqlalchemy.create_engine(url)
     try:
         engine.connect()
-        logging.info("check_connection_valid: True")
+        logging.info("create_sqlalchemy_engine: True")
     except Exception as e:
-        logging.error(f"check_connection_valid: False ({e})")
+        logging.error(f"create_sqlalchemy_engine: False ({e})")
     return engine
 
 

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data-manipulation
-Version: 0.36
+Version: 0.37
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -19,6 +19,16 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: django
+Requires-Dist: lxml
+Requires-Dist: matplotlib
+Requires-Dist: mysql-connector-python
+Requires-Dist: pandas
+Requires-Dist: psycopg2
+Requires-Dist: pyspark
+Requires-Dist: tqdm
+Requires-Dist: xlsxwriter
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.36 → data_manipulation-0.37}/data_manipulation.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ data_manipulation/__init__.py
 data_manipulation/_version.py
 data_manipulation/base.py
 data_manipulation/beautifulsoup_.py
+data_manipulation/boto3_.py
 data_manipulation/cryptography_.py
 data_manipulation/django_.py
 data_manipulation/flask_.py

data_manipulation-0.36/data_manipulation/django_.py

@@ -1,48 +0,0 @@
-def init_django(django_dir: str, project_name: str) -> None:
-    """
-    Setup Django in Jupyter. Reference from https://gist.github.com/codingforentrepreneurs/76e570d759f83d690bf36a8a8fa4cfbe
-
-    Parameters
-    ----------
-    django_dir : str
-        Django project location
-    project_name : str
-        Django project name
-    """
-    import os
-    import sys
-
-    import django
-
-    # django_dir = os.getenv("PWD")
-    # os.chdir(django_dir)
-
-    project_name = project_name or os.environ.get("DJANGO_PROJECT") or None
-    if not project_name:
-        raise Exception(
-            """Set an environment variable: `DJANGO_PROJECT=your_project_name` or call `init_django(your_project_name)`"""
-        )
-
-    sys.path.insert(0, django_dir)
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", f"{project_name}.settings")
-    os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
-    django.setup()
-
-
-def get_django_countries_dict():
-    """
-    Get django_countries package's dictionaries
-        code_name - "SG": "SINGAPORE"
-        name_code - "SINGAPORE": "SG"
-    """
-    from django_countries import countries
-
-    code_name = {k: v.upper() for k, v in dict(countries).items()}
-    name_code = {v: k for k, v in code_name.items()}
-    return code_name, name_code
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()

data_manipulation-0.36/data_manipulation/prometheus_.py

@@ -1,27 +0,0 @@
-import logging
-import subprocess
-
-
-def ldapsearch(
-    search_base: str,
-    ldap_uri: str,
-    bind_dn: str,
-    password: str,
-    filter: str,
-) -> subprocess.CompletedProcess:
-    cmd = (
-        "ldapsearch -b {search_base} -H {ldap_uri} -D {bind_dn} -w {password} {filter}"
-    )
-    output = subprocess.run(
-        cmd,
-        capture_output=True,
-        shell=True,
-        text=True,
-    )
-    return output
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()