data-manipulation 0.36__tar.gz → 0.39__tar.gz

{data_manipulation-0.36 → data_manipulation-0.39}/LICENSE

@@ -1,19 +1,19 @@
 BSD 3-Clause License
 
-Copyright (c) 2020, Ng Teng Qing (shawnngtq)
+Copyright (c) 2020, Shawn Ng (shawnngtq)
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification,
 are permitted provided that the following conditions are met:
 
-* Redistributions of source code must retain the above copyright notice, this
+- Redistributions of source code must retain the above copyright notice, this
   list of conditions and the following disclaimer.
 
-* Redistributions in binary form must reproduce the above copyright notice, this
+- Redistributions in binary form must reproduce the above copyright notice, this
   list of conditions and the following disclaimer in the documentation and/or
   other materials provided with the distribution.
 
-* Neither the name of data_manipulation nor the names of its
+- Neither the name of data_manipulation nor the names of its
   contributors may be used to endorse or promote products derived from this
   software without specific prior written permission.
 

{data_manipulation-0.36 → data_manipulation-0.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_manipulation
-Version: 0.36
+Version: 0.39
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -19,6 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: boto3
+Requires-Dist: django
+Requires-Dist: flask
+Requires-Dist: geopandas
+Requires-Dist: mysql-connector-python
+Requires-Dist: pandas
+Requires-Dist: psycopy
+Requires-Dist: psycopg2
+Requires-Dist: pyspark
+Requires-Dist: sqlalchemy
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.36 → data_manipulation-0.39}/README.md

@@ -25,16 +25,7 @@ pip install data-manipulation
 
 ## Dependencies
 
-The dependencies will be installed automatically along with this package
-
-- beautifulsoup4
-- django
-- numpy
-- pandas
-- ipython
-- requests
-- psycopg2
-- pyspark
+The dependencies will be installed automatically along with this package, in `setup.py`
 
 ## License
 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/__init__.py

@@ -1,14 +1,18 @@
 from data_manipulation import (
     base,
     beautifulsoup_,
+    boto3_,
     cryptography_,
     django_,
+    flask_,
     geopandas_,
     kerberos_,
     mysql_connector_python_,
     openldap_,
     pandas_,
+    prometheus_,
     psycopg2_,
+    psycopg_,
     pyspark_,
     smtplib_,
     sqlalchemy_,

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-10-03T09:27:54+0800",
+ "date": "2024-05-26T15:36:56+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "263a1314653b0a2f32b9e1eea3efbf78ad990ca2",
- "version": "0.36"
+ "full-revisionid": "2c9d748a4051bd9c0d34c3dc58adb3d06d8b0a5b",
+ "version": "0.39"
 }
 '''  # END VERSION_JSON
 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/base.py

@@ -1,6 +1,10 @@
+import logging
 import subprocess
 from typing import List
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 # DATA STRUCTURE
 def clean_string(
@@ -109,6 +113,7 @@ def get_country_name_variation() -> dict:
         "SLAVONIC": "SK",
         "SLOVAK REPUBLIC": "SK",
         "TURKEY": "TR",
+        "TURKIYE": "TR",
         "UNITED STATES": "US",
     }
     return variations
@@ -335,7 +340,7 @@ def remove_path_file(path: str, keyword: str, n: int = 2) -> None:
     to_delete = get_path_files(path=path, keywords=[keyword])[:-n]
     for file in to_delete:
         os.remove(f"{path}/{file}")
-        print(f"{path}/{file} deleted ...")
+        logger.info(f"{path}/{file} deleted ...")
 
 
 def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:
@@ -363,6 +368,28 @@ def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:
     f.close()
 
 
+# URLLIB
+def create_encode_url(url: str, query_params: dict = {}) -> str:
+    """
+    Create encoded url
+
+    Parameters
+    ----------
+    url : str
+        base url
+    query_params : dict, optional
+        dictionary of url query parameters, by default {}
+
+    Returns
+    -------
+    str
+        encoded url
+    """
+    from urllib.parse import urlencode
+
+    return f"{url}{urlencode(query_params)}"
+
+
 # SYSTEM
 def parse_ps_aux(ps_aux_commands: str) -> List[list]:
     """

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/beautifulsoup_.py

@@ -1,6 +1,8 @@
+import logging
 from typing import Optional
 
-from bs4 import BeautifulSoup
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
 
 
 def preprocess(html: str) -> Optional[str]:
@@ -37,9 +39,7 @@ def preprocess(html: str) -> Optional[str]:
     return html
 
 
-def build_soup(
-    url: str, features: str = "lxml", to_preprocess: str = True
-) -> Optional[BeautifulSoup]:
+def build_soup(url: str, features: str = "lxml", to_preprocess: str = True):
     """
     Return Beautifulsoup object from given url
 
@@ -74,7 +74,7 @@ def build_soup(
             soup = BeautifulSoup(response.text, features=features)
         return soup
     else:
-        print(f"response status code: {response.status_code}")
+        logger.error(f"response status code: {response.status_code}")
 
 
 if __name__ == "__main__":

data_manipulation-0.39/data_manipulation/boto3_.py

@@ -0,0 +1,91 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
+def send_aws_ses_email(
+    sender: str,
+    recipient: list,
+    subject: str,
+    body_text: str,
+    body_type: str,
+    ses_client,
+    attachment: str = None,
+):
+    """
+    Send AWS SES email
+
+    Parameters
+    ----------
+    sender : str
+        sender email
+    recipient : list
+        list of recipient emails
+    subject : str
+        email subject
+    body_text : str
+        email body
+    body_type : str
+        email body type
+    ses_client : _type_
+        aws ses client
+    attachment : str, optional
+        attachment path, by default None
+
+    Returns
+    -------
+    dict
+        aws ses client email response or none
+
+    Reference
+    ---------
+    - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ses/client/send_raw_email.html
+    """
+    import os
+    from email.mime.application import MIMEApplication
+    from email.mime.multipart import MIMEMultipart
+    from email.mime.text import MIMEText
+
+    from botocore.exceptions import BotoCoreError, ClientError
+
+    msg = MIMEMultipart()
+    msg["Subject"] = subject
+    msg["From"] = sender
+    msg["To"] = ", ".join(recipient)
+
+    part = MIMEText(body_text, body_type)
+    msg.attach(part)
+
+    if attachment:
+        try:
+            with open(attachment, "rb") as f:
+                part = MIMEApplication(f.read())
+                part.add_header(
+                    "Content-Disposition",
+                    "attachment",
+                    filename=os.path.basename(attachment),
+                )
+                msg.attach(part)
+        except FileNotFoundError:
+            logger.error(f"{attachment} not found")
+            return
+
+    try:
+        response = ses_client.send_raw_email(
+            Source=sender,
+            Destinations=recipient,
+            RawMessage={
+                "Data": msg.as_string(),
+            },
+        )
+        return response
+    except (BotoCoreError, ClientError) as error:
+        logger.error(f"Error: {error}")
+        return
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/cryptography_.py

@@ -2,6 +2,9 @@ import logging
 
 from cryptography.fernet import Fernet
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def generate_fernet_key(output_directory: str, output_filename: str) -> bytes:
     """
@@ -29,9 +32,9 @@ def generate_fernet_key(output_directory: str, output_filename: str) -> bytes:
             f = open(filepath, "wb")
             f.write(key)
             f.close()
-            logging.info(f"generate_fernet_key: True")
+            logger.info(f"generate_fernet_key: True")
         except Exception as e:
-            logging.error(f"generate_fernet_key: False ({e})")
+            logger.error(f"generate_fernet_key: False ({e})")
     return key
 
 

data_manipulation-0.39/data_manipulation/django_.py

@@ -0,0 +1,129 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
+def init_django(django_dir: str, project_name: str) -> None:
+    """
+    Setup Django in Jupyter. Reference from https://gist.github.com/codingforentrepreneurs/76e570d759f83d690bf36a8a8fa4cfbe
+
+    Parameters
+    ----------
+    django_dir : str
+        Django project location
+    project_name : str
+        Django project name
+    """
+    import os
+    import sys
+
+    import django
+
+    # django_dir = os.getenv("PWD")
+    # os.chdir(django_dir)
+
+    project_name = project_name or os.environ.get("DJANGO_PROJECT") or None
+    if not project_name:
+        raise Exception(
+            """Set an environment variable: `DJANGO_PROJECT=your_project_name` or call `init_django(your_project_name)`"""
+        )
+
+    sys.path.insert(0, django_dir)
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", f"{project_name}.settings")
+    os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
+    django.setup()
+
+
+def get_django_countries_dict():
+    """
+    Get django_countries package's dictionaries
+        code_name - "SG": "SINGAPORE"
+        name_code - "SINGAPORE": "SG"
+    """
+    from django_countries import countries
+
+    code_name = {k: v.upper() for k, v in dict(countries).items()}
+    name_code = {v: k for k, v in code_name.items()}
+    return code_name, name_code
+
+
+def django_validate_email(email: str) -> str | None:
+    """
+    Django validate email
+
+    Parameters
+    ----------
+    email : str
+        email to check
+
+    Returns
+    -------
+    str | None
+        email or none
+    """
+    from django.core.validators import validate_email
+
+    try:
+        validate_email(email)
+        return email
+    except Exception as e:
+        return
+
+
+def django_validate_url(url: str) -> str | None:
+    """
+    Django validate url
+
+    Parameters
+    ----------
+    url : str
+        url to validate
+
+    Returns
+    -------
+    str | None
+        url or None
+    """
+    from django.core.validators import URLValidator
+
+    validator = URLValidator()
+    try:
+        validator(url)
+        return url
+    except Exception as e:
+        return
+
+
+def django_validate_phone(phone: str, region=None) -> str | None:
+    """
+    Django validate phone
+
+    Parameters
+    ----------
+    phone : str
+        phone to check
+    region : _type_, optional
+        phone region, by default None
+
+    Returns
+    -------
+    str | None
+        parsed phone or none
+    """
+    from phonenumber_field.phonenumber import PhoneNumber
+
+    try:
+        return PhoneNumber.from_string(phone).as_e164
+    except:
+        try:
+            return PhoneNumber.from_string(phone, region=region).as_e164
+        except Exception as e:
+            logger.error(e)
+            return
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/kerberos_.py

@@ -1,6 +1,9 @@
 import logging
 import subprocess
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def keytab_valid(
     keytab_filepath: str,
@@ -29,11 +32,11 @@ def keytab_valid(
     )
     try:
         if output.returncode == 0:
-            logging.info("keytab_valid: True")
+            logger.info("keytab_valid: True")
         else:
-            logging.info("keytab_valid: False")
+            logger.info("keytab_valid: False")
     except Exception as e:
-        logging.error(f"keytab_valid: False ({e})")
+        logger.error(f"keytab_valid: False ({e})")
     return output
 
 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/mysql_connector_python_.py

@@ -3,6 +3,9 @@ from typing import Optional, Union
 
 import mysql.connector
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_connection(
     host: str,
@@ -43,9 +46,9 @@ def create_connection(
             password=password,
             port=port,
         )
-        logging.info("MySQL database connected ...")
+        logger.info("MySQL database connected ...")
     except mysql.connector.errors.Error as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
     return connection
 
 
@@ -79,12 +82,12 @@ def execute_query(
         cursor.execute(sql_query, data)
         if commit:
             connection.commit()
-            logging.info("MySQL committed ...")
+            logger.info("MySQL committed ...")
             id = cursor.lastrowid
             cursor.close()
             return id
     except mysql.connector.errors.Error as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
 
 
 if __name__ == "__main__":

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/openldap_.py

@@ -1,4 +1,3 @@
-import logging
 import subprocess
 
 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/pandas_.py

@@ -1,3 +1,4 @@
+import logging
 import subprocess
 from itertools import combinations
 
@@ -5,6 +6,9 @@ import numpy as np
 import pandas as pd
 from IPython.display import display
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 # CONFIG
 def config_pandas_display():
@@ -226,7 +230,7 @@ def compare_dataframes(dataframe, dataframe2):
         try:
             pd_df2 = dataframe2[dataframe2[column].notnull()]
         except KeyError:
-            print(f"df2 {column}: does not exist")
+            logger.error(f"df2 {column}: does not exist")
 
         if pd_df2 is not None:
             print(f"df2 non-null {column}: {len(pd_df2)}")
@@ -364,15 +368,56 @@ def split_left_merged_dataframe(dataframe, dataframe2, columns):
     return df_both, df_left
 
 
+## AGGREGATE
+def aggregate_set_without_none(column: pd.Series, nested_set: bool = False) -> set:
+    """
+    Use in pandas groupby agg, create a set without none based on input column
+
+    Parameters
+    ----------
+    column : pd.Series
+        columns to set
+    nested_set : bool, optional
+        nested set within set, by default False
+
+    Returns
+    -------
+    set
+        set without None
+    """
+    if nested_set:
+        output = set()
+        for value in column:
+            if isinstance(value, set):
+                output.update(value)
+            # todo, what if there is valid list / dict / tuple / complex object?
+            elif value:
+                output.add(value)
+        return output
+    else:
+        return {value for value in column if value is not None}
+
+
 # DATA STRUCTURE
-def clean_none(dataframe, clean_variation=True, non_variations=[]):
+def clean_none(
+    dataframe: pd.DataFrame,
+    nan_to_none: bool = True,
+    clean_variation: bool = True,
+    none_variations: list = [],
+) -> pd.DataFrame:
     """
     Return a dataframe from given dataframe with standardized None. Deprecated as of pandas 1.3.0.
 
     Parameters
     ----------
-    dataframe : pandas.DataFrame
-        Dataframe
+    dataframe : pd.DataFrame
+        _description_
+    nan_to_none : bool, optional
+        replace np.NaN with None, by default True
+    clean_variation : bool, optional
+        clean custom variables, by default True
+    none_variations : list, optional
+        list of none variations, by default []
 
     Examples
     --------
@@ -410,12 +455,15 @@ def clean_none(dataframe, clean_variation=True, non_variations=[]):
 
     Returns
     -------
-    df : pandas.DataFrame
+    pd.DataFrame
+        _description_
     """
     df = dataframe.copy()
     df = df.replace(r"^\s*$", np.nan, regex=True)
+    if nan_to_none:
+        df = df.replace(np.NaN, None, inplace=True)
     if clean_variation:
-        df = df.replace(non_variations, np.nan)
+        df = df.replace(none_variations, np.nan)
     df = df.where(pd.notnull(df), None)
     return df
 
@@ -678,7 +726,7 @@ def print_dataframe_overview(dataframe, stats=False):
                 print(f"Maximum: {max(dataframe[column])}")
             display(series_count(dataframe[column]).head(20))
         except:
-            print(f"Unable to get value_counts of {column} ...\n")
+            logger.error(f"Unable to get value_counts of {column} ...\n")
 
 
 def series_count(series):

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/psycopg2_.py

@@ -4,6 +4,9 @@ from typing import Optional, Union
 import pandas as pd
 import psycopg2
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_connection(
     host: str,
@@ -39,9 +42,9 @@ def create_connection(
             f"host={host} port={port} dbname={dbname} user={user} password={password}"
         )
         connection = psycopg2.connect(connection_string)
-        logging.info("PostgreSQL database connected ...")
+        logger.info("PostgreSQL database connected ...")
     except psycopg2.OperationalError as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
     return connection
 
 
@@ -70,9 +73,9 @@ def execute_sql(
         cursor.execute(sql_query, data)
         if commit:
             connection.commit()
-            logging.info("PostgreSQL committed ...")
+            logger.info("PostgreSQL committed ...")
     except psycopg2.OperationalError as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
 
 
 def query_to_pandas(

data_manipulation-0.39/data_manipulation/psycopg_.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Optional, Union
+
+import pandas as pd
+import psycopg
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
+def create_connection(
+    host: str,
+    dbname: str,
+    user: str,
+    password: str,
+    port=5432,
+) -> psycopg.Connection:
+    """
+    Return psycopg connection (https://www.psycopg.org/)
+
+    Parameters
+    ----------
+    host : str
+        Database host address
+    dbname : str
+        Database name
+    user : str
+        User used to authenticate
+    password : str
+        Password used to authenticate
+    port : int, optional
+        Port number, by default 5432
+
+    Returns
+    -------
+    psycopg.Connection
+        Connection object
+    """
+    connection = None
+    try:
+        connection_string = (
+            f"host={host} port={port} dbname={dbname} user={user} password={password}"
+        )
+        connection = psycopg.connect(connection_string)
+        logger.info("PostgreSQL database connected ...")
+    except psycopg.OperationalError as e:
+        logger.error(f"{e}")
+    return connection
+
+
+def execute_sql(
+    connection: psycopg.Connection,
+    sql_query: str,
+    data: Union[dict, tuple],
+    commit=True,
+) -> None:
+    """
+    Execute and commit PostgreSQL query
+
+    Parameters
+    ----------
+    connection : psycopg.Connection
+        _description_
+    sql_query : str
+        SQL query
+    data : Union[dict, tuple]
+        Data
+    commit : bool, optional
+        Make database change persistent, by default True
+
+    Reference
+    ---------
+    When you insert data key-value with dict type, error
+        ProgrammingError: cannot adapt type 'dict' using placeholder '%s' (format: AUTO)
+    """
+    cursor = connection.cursor()
+    try:
+        cursor.execute(sql_query, data)
+        if commit:
+            connection.commit()
+            logger.info("PostgreSQL committed ...")
+    except psycopg.OperationalError as e:
+        logger.error(f"{e}")
+
+
+def query_to_pandas(
+    connection: psycopg.Connection,
+    sql_query: str,
+) -> pd.DataFrame:
+    """
+    Return Pandas dataframe from result of given SQL query
+
+    Parameters
+    ----------
+    connection : psycopg.Connection
+        _description_
+    sql_query : str
+        SQL query
+
+    Examples
+    --------
+    >>> sql_query = '''SELECT * FROM users;'''
+
+    Returns
+    -------
+    pd.DataFrame
+        _description_
+    """
+
+    return pd.read_sql(
+        sql=sql_query,
+        con=connection,
+    )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/pyspark_.py

@@ -2,8 +2,6 @@ import pandas as pd
 import pyspark
 import pyspark.sql.functions as F
 
-spark = pyspark.sql.SparkSession.builder.master("local").getOrCreate()
-
 
 # CONFIG
 def config_spark_local(autoset=True):
@@ -84,7 +82,6 @@ def config_spark_local(autoset=True):
     print(f"spark_default_parallelism: {spark_default_parallelism}")
 
     if autoset:
-        global spark
         spark = (
             pyspark.sql.SparkSession.builder.master("local")
             .config("spark.executor.cores", str(spark_executor_cores))

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation/sqlalchemy_.py

@@ -2,6 +2,9 @@ import logging
 
 import sqlalchemy
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_sqlalchemy_url(
     drivername: str,
@@ -45,7 +48,7 @@ def create_sqlalchemy_url(
     return url
 
 
-def check_connection_valid(
+def create_sqlalchemy_engine(
     drivername: str,
     host: str,
     dbname: str,
@@ -54,7 +57,7 @@ def check_connection_valid(
     port=3306,
 ) -> sqlalchemy.engine.base.Engine:
     """
-    Check mysql connection validity
+    Create SQLalchemy engine
 
     Parameters
     ----------
@@ -87,9 +90,9 @@ def check_connection_valid(
     engine = sqlalchemy.create_engine(url)
     try:
         engine.connect()
-        logging.info("check_connection_valid: True")
+        logger.info("create_sqlalchemy_engine: True")
     except Exception as e:
-        logging.error(f"check_connection_valid: False ({e})")
+        logger.error(f"create_sqlalchemy_engine: False ({e})")
     return engine
 
 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: data-manipulation
-Version: 0.36
+Name: data_manipulation
+Version: 0.39
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -19,6 +19,17 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
+Requires-Dist: beautifulsoup4
+Requires-Dist: boto3
+Requires-Dist: django
+Requires-Dist: flask
+Requires-Dist: geopandas
+Requires-Dist: mysql-connector-python
+Requires-Dist: pandas
+Requires-Dist: psycopy
+Requires-Dist: psycopg2
+Requires-Dist: pyspark
+Requires-Dist: sqlalchemy
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation.egg-info/SOURCES.txt

@@ -8,6 +8,7 @@ data_manipulation/__init__.py
 data_manipulation/_version.py
 data_manipulation/base.py
 data_manipulation/beautifulsoup_.py
+data_manipulation/boto3_.py
 data_manipulation/cryptography_.py
 data_manipulation/django_.py
 data_manipulation/flask_.py
@@ -18,6 +19,7 @@ data_manipulation/openldap_.py
 data_manipulation/pandas_.py
 data_manipulation/prometheus_.py
 data_manipulation/psycopg2_.py
+data_manipulation/psycopg_.py
 data_manipulation/pyspark_.py
 data_manipulation/smtplib_.py
 data_manipulation/sqlalchemy_.py

{data_manipulation-0.36 → data_manipulation-0.39}/data_manipulation.egg-info/requires.txt

@@ -1,10 +1,11 @@
 beautifulsoup4
+boto3
 django
-lxml
-matplotlib
+flask
+geopandas
 mysql-connector-python
 pandas
+psycopy
 psycopg2
 pyspark
-tqdm
-xlsxwriter
+sqlalchemy

{data_manipulation-0.36 → data_manipulation-0.39}/setup.py

@@ -23,15 +23,16 @@ CLASSIFIERS = [
 
 DEPENDENCIES = [
     "beautifulsoup4",
+    "boto3",
     "django",
-    "lxml",
-    "matplotlib",
+    "flask",
+    "geopandas",
     "mysql-connector-python",
     "pandas",
+    "psycopy",
     "psycopg2",
     "pyspark",
-    "tqdm",
-    "xlsxwriter",
+    "sqlalchemy",
 ]
 
 setup(

data_manipulation-0.36/data_manipulation/django_.py

@@ -1,48 +0,0 @@
-def init_django(django_dir: str, project_name: str) -> None:
-    """
-    Setup Django in Jupyter. Reference from https://gist.github.com/codingforentrepreneurs/76e570d759f83d690bf36a8a8fa4cfbe
-
-    Parameters
-    ----------
-    django_dir : str
-        Django project location
-    project_name : str
-        Django project name
-    """
-    import os
-    import sys
-
-    import django
-
-    # django_dir = os.getenv("PWD")
-    # os.chdir(django_dir)
-
-    project_name = project_name or os.environ.get("DJANGO_PROJECT") or None
-    if not project_name:
-        raise Exception(
-            """Set an environment variable: `DJANGO_PROJECT=your_project_name` or call `init_django(your_project_name)`"""
-        )
-
-    sys.path.insert(0, django_dir)
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", f"{project_name}.settings")
-    os.environ["DJANGO_ALLOW_ASYNC_UNSAFE"] = "true"
-    django.setup()
-
-
-def get_django_countries_dict():
-    """
-    Get django_countries package's dictionaries
-        code_name - "SG": "SINGAPORE"
-        name_code - "SINGAPORE": "SG"
-    """
-    from django_countries import countries
-
-    code_name = {k: v.upper() for k, v in dict(countries).items()}
-    name_code = {v: k for k, v in code_name.items()}
-    return code_name, name_code
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()

data_manipulation-0.36/data_manipulation/prometheus_.py

@@ -1,27 +0,0 @@
-import logging
-import subprocess
-
-
-def ldapsearch(
-    search_base: str,
-    ldap_uri: str,
-    bind_dn: str,
-    password: str,
-    filter: str,
-) -> subprocess.CompletedProcess:
-    cmd = (
-        "ldapsearch -b {search_base} -H {ldap_uri} -D {bind_dn} -w {password} {filter}"
-    )
-    output = subprocess.run(
-        cmd,
-        capture_output=True,
-        shell=True,
-        text=True,
-    )
-    return output
-
-
-if __name__ == "__main__":
-    import doctest
-
-    doctest.testmod()