data-manipulation 0.37__tar.gz → 0.40__tar.gz

{data_manipulation-0.37 → data_manipulation-0.40}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: data_manipulation
-Version: 0.37
+Version: 0.40
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -20,15 +20,16 @@ Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
 Requires-Dist: beautifulsoup4
+Requires-Dist: boto3
 Requires-Dist: django
-Requires-Dist: lxml
-Requires-Dist: matplotlib
+Requires-Dist: flask
+Requires-Dist: geopandas
 Requires-Dist: mysql-connector-python
 Requires-Dist: pandas
+Requires-Dist: psycopg
 Requires-Dist: psycopg2
 Requires-Dist: pyspark
-Requires-Dist: tqdm
-Requires-Dist: xlsxwriter
+Requires-Dist: sqlalchemy
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.37 → data_manipulation-0.40}/README.md

@@ -25,16 +25,7 @@ pip install data-manipulation
 
 ## Dependencies
 
-The dependencies will be installed automatically along with this package
-
-- beautifulsoup4
-- django
-- numpy
-- pandas
-- ipython
-- requests
-- psycopg2
-- pyspark
+The dependencies will be installed automatically along with this package, in `setup.py`
 
 ## License
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/__init__.py

@@ -12,6 +12,7 @@ from data_manipulation import (
     pandas_,
     prometheus_,
     psycopg2_,
+    psycopg_,
     pyspark_,
     smtplib_,
     sqlalchemy_,

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/_version.py

@@ -8,11 +8,11 @@ import json
 
 version_json = '''
 {
- "date": "2023-12-30T10:16:23+0800",
+ "date": "2024-05-26T15:48:20+0800",
  "dirty": false,
  "error": null,
- "full-revisionid": "6f824ae7f46ddf5b2f60190bacfc2e4118d7f80f",
- "version": "0.37"
+ "full-revisionid": "805e0463f2f7f444a10afa22b658bc31ea864fed",
+ "version": "0.40"
 }
 '''  # END VERSION_JSON
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/base.py

@@ -1,6 +1,10 @@
+import logging
 import subprocess
 from typing import List
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 # DATA STRUCTURE
 def clean_string(
@@ -336,7 +340,7 @@ def remove_path_file(path: str, keyword: str, n: int = 2) -> None:
     to_delete = get_path_files(path=path, keywords=[keyword])[:-n]
     for file in to_delete:
         os.remove(f"{path}/{file}")
-        print(f"{path}/{file} deleted ...")
+        logger.info(f"{path}/{file} deleted ...")
 
 
 def list_to_file(filepath: str, list_: list, newline: bool = True) -> None:

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/beautifulsoup_.py

@@ -1,6 +1,8 @@
+import logging
 from typing import Optional
 
-from bs4 import BeautifulSoup
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
 
 
 def preprocess(html: str) -> Optional[str]:
@@ -37,9 +39,7 @@ def preprocess(html: str) -> Optional[str]:
     return html
 
 
-def build_soup(
-    url: str, features: str = "lxml", to_preprocess: str = True
-) -> Optional[BeautifulSoup]:
+def build_soup(url: str, features: str = "lxml", to_preprocess: str = True):
     """
     Return Beautifulsoup object from given url
 
@@ -74,7 +74,7 @@ def build_soup(
             soup = BeautifulSoup(response.text, features=features)
         return soup
     else:
-        print(f"response status code: {response.status_code}")
+        logger.error(f"response status code: {response.status_code}")
 
 
 if __name__ == "__main__":

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/boto3_.py

@@ -1,3 +1,9 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
 def send_aws_ses_email(
     sender: str,
     recipient: list,
@@ -29,8 +35,12 @@ def send_aws_ses_email(
 
     Returns
     -------
-    str
+    dict
         aws ses client email response or none
+
+    Reference
+    ---------
+    - https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ses/client/send_raw_email.html
     """
     import os
     from email.mime.application import MIMEApplication
@@ -58,7 +68,7 @@ def send_aws_ses_email(
                 )
                 msg.attach(part)
         except FileNotFoundError:
-            print(f"{attachment} not found")
+            logger.error(f"{attachment} not found")
             return
 
     try:
@@ -69,11 +79,10 @@ def send_aws_ses_email(
                 "Data": msg.as_string(),
             },
         )
+        return response
     except (BotoCoreError, ClientError) as error:
-        print(f"Error: {error}")
+        logger.error(f"Error: {error}")
         return
-    else:
-        return response
 
 
 if __name__ == "__main__":

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/cryptography_.py

@@ -2,6 +2,9 @@ import logging
 
 from cryptography.fernet import Fernet
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def generate_fernet_key(output_directory: str, output_filename: str) -> bytes:
     """
@@ -29,9 +32,9 @@ def generate_fernet_key(output_directory: str, output_filename: str) -> bytes:
             f = open(filepath, "wb")
             f.write(key)
             f.close()
-            logging.info(f"generate_fernet_key: True")
+            logger.info(f"generate_fernet_key: True")
         except Exception as e:
-            logging.error(f"generate_fernet_key: False ({e})")
+            logger.error(f"generate_fernet_key: False ({e})")
     return key
 
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/django_.py

@@ -1,3 +1,9 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
 def init_django(django_dir: str, project_name: str) -> None:
     """
     Setup Django in Jupyter. Reference from https://gist.github.com/codingforentrepreneurs/76e570d759f83d690bf36a8a8fa4cfbe
@@ -113,7 +119,7 @@ def django_validate_phone(phone: str, region=None) -> str | None:
         try:
             return PhoneNumber.from_string(phone, region=region).as_e164
         except Exception as e:
-            print(e)
+            logger.error(e)
             return
 
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/kerberos_.py

@@ -1,6 +1,9 @@
 import logging
 import subprocess
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def keytab_valid(
     keytab_filepath: str,
@@ -29,11 +32,11 @@ def keytab_valid(
     )
     try:
         if output.returncode == 0:
-            logging.info("keytab_valid: True")
+            logger.info("keytab_valid: True")
         else:
-            logging.info("keytab_valid: False")
+            logger.info("keytab_valid: False")
     except Exception as e:
-        logging.error(f"keytab_valid: False ({e})")
+        logger.error(f"keytab_valid: False ({e})")
     return output
 
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/mysql_connector_python_.py

@@ -3,6 +3,9 @@ from typing import Optional, Union
 
 import mysql.connector
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_connection(
     host: str,
@@ -43,9 +46,9 @@ def create_connection(
             password=password,
             port=port,
         )
-        logging.info("MySQL database connected ...")
+        logger.info("MySQL database connected ...")
     except mysql.connector.errors.Error as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
     return connection
 
 
@@ -79,12 +82,12 @@ def execute_query(
         cursor.execute(sql_query, data)
         if commit:
             connection.commit()
-            logging.info("MySQL committed ...")
+            logger.info("MySQL committed ...")
             id = cursor.lastrowid
             cursor.close()
             return id
     except mysql.connector.errors.Error as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
 
 
 if __name__ == "__main__":

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/openldap_.py

@@ -1,4 +1,3 @@
-import logging
 import subprocess
 
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/pandas_.py

@@ -1,3 +1,4 @@
+import logging
 import subprocess
 from itertools import combinations
 
@@ -5,6 +6,9 @@ import numpy as np
 import pandas as pd
 from IPython.display import display
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 # CONFIG
 def config_pandas_display():
@@ -226,7 +230,7 @@ def compare_dataframes(dataframe, dataframe2):
         try:
             pd_df2 = dataframe2[dataframe2[column].notnull()]
         except KeyError:
-            print(f"df2 {column}: does not exist")
+            logger.error(f"df2 {column}: does not exist")
 
         if pd_df2 is not None:
             print(f"df2 non-null {column}: {len(pd_df2)}")
@@ -722,7 +726,7 @@ def print_dataframe_overview(dataframe, stats=False):
                 print(f"Maximum: {max(dataframe[column])}")
             display(series_count(dataframe[column]).head(20))
         except:
-            print(f"Unable to get value_counts of {column} ...\n")
+            logger.error(f"Unable to get value_counts of {column} ...\n")
 
 
 def series_count(series):

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/psycopg2_.py

@@ -4,6 +4,9 @@ from typing import Optional, Union
 import pandas as pd
 import psycopg2
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_connection(
     host: str,
@@ -39,9 +42,9 @@ def create_connection(
             f"host={host} port={port} dbname={dbname} user={user} password={password}"
         )
         connection = psycopg2.connect(connection_string)
-        logging.info("PostgreSQL database connected ...")
+        logger.info("PostgreSQL database connected ...")
     except psycopg2.OperationalError as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
     return connection
 
 
@@ -70,9 +73,9 @@ def execute_sql(
         cursor.execute(sql_query, data)
         if commit:
             connection.commit()
-            logging.info("PostgreSQL committed ...")
+            logger.info("PostgreSQL committed ...")
     except psycopg2.OperationalError as e:
-        logging.error(f"{e}")
+        logger.error(f"{e}")
 
 
 def query_to_pandas(

data_manipulation-0.40/data_manipulation/psycopg_.py

@@ -0,0 +1,119 @@
+import logging
+from typing import Optional, Union
+
+import pandas as pd
+import psycopg
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
+
+def create_connection(
+    host: str,
+    dbname: str,
+    user: str,
+    password: str,
+    port=5432,
+) -> psycopg.Connection:
+    """
+    Return psycopg connection (https://www.psycopg.org/)
+
+    Parameters
+    ----------
+    host : str
+        Database host address
+    dbname : str
+        Database name
+    user : str
+        User used to authenticate
+    password : str
+        Password used to authenticate
+    port : int, optional
+        Port number, by default 5432
+
+    Returns
+    -------
+    psycopg.Connection
+        Connection object
+    """
+    connection = None
+    try:
+        connection_string = (
+            f"host={host} port={port} dbname={dbname} user={user} password={password}"
+        )
+        connection = psycopg.connect(connection_string)
+        logger.info("PostgreSQL database connected ...")
+    except psycopg.OperationalError as e:
+        logger.error(f"{e}")
+    return connection
+
+
+def execute_sql(
+    connection: psycopg.Connection,
+    sql_query: str,
+    data: Union[dict, tuple],
+    commit=True,
+) -> None:
+    """
+    Execute and commit PostgreSQL query
+
+    Parameters
+    ----------
+    connection : psycopg.Connection
+        _description_
+    sql_query : str
+        SQL query
+    data : Union[dict, tuple]
+        Data
+    commit : bool, optional
+        Make database change persistent, by default True
+
+    Reference
+    ---------
+    When you insert data key-value with dict type, error
+        ProgrammingError: cannot adapt type 'dict' using placeholder '%s' (format: AUTO)
+    """
+    cursor = connection.cursor()
+    try:
+        cursor.execute(sql_query, data)
+        if commit:
+            connection.commit()
+            logger.info("PostgreSQL committed ...")
+    except psycopg.OperationalError as e:
+        logger.error(f"{e}")
+
+
+def query_to_pandas(
+    connection: psycopg.Connection,
+    sql_query: str,
+) -> pd.DataFrame:
+    """
+    Return Pandas dataframe from result of given SQL query
+
+    Parameters
+    ----------
+    connection : psycopg.Connection
+        _description_
+    sql_query : str
+        SQL query
+
+    Examples
+    --------
+    >>> sql_query = '''SELECT * FROM users;'''
+
+    Returns
+    -------
+    pd.DataFrame
+        _description_
+    """
+
+    return pd.read_sql(
+        sql=sql_query,
+        con=connection,
+    )
+
+
+if __name__ == "__main__":
+    import doctest
+
+    doctest.testmod()

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/pyspark_.py

@@ -2,8 +2,6 @@ import pandas as pd
 import pyspark
 import pyspark.sql.functions as F
 
-spark = pyspark.sql.SparkSession.builder.master("local").getOrCreate()
-
 
 # CONFIG
 def config_spark_local(autoset=True):
@@ -84,7 +82,6 @@ def config_spark_local(autoset=True):
     print(f"spark_default_parallelism: {spark_default_parallelism}")
 
     if autoset:
-        global spark
         spark = (
             pyspark.sql.SparkSession.builder.master("local")
             .config("spark.executor.cores", str(spark_executor_cores))

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation/sqlalchemy_.py

@@ -2,6 +2,9 @@ import logging
 
 import sqlalchemy
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__file__)
+
 
 def create_sqlalchemy_url(
     drivername: str,
@@ -87,9 +90,9 @@ def create_sqlalchemy_engine(
     engine = sqlalchemy.create_engine(url)
     try:
         engine.connect()
-        logging.info("create_sqlalchemy_engine: True")
+        logger.info("create_sqlalchemy_engine: True")
     except Exception as e:
-        logging.error(f"create_sqlalchemy_engine: False ({e})")
+        logger.error(f"create_sqlalchemy_engine: False ({e})")
     return engine
 
 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
-Name: data-manipulation
-Version: 0.37
+Name: data_manipulation
+Version: 0.40
 Summary: Powerful data manipulation
 Home-page: https://github.com/shawnngtq/data-manipulation
 Author: Shawn Ng
@@ -20,15 +20,16 @@ Classifier: Programming Language :: Cython
 Classifier: Topic :: Scientific/Engineering
 License-File: LICENSE
 Requires-Dist: beautifulsoup4
+Requires-Dist: boto3
 Requires-Dist: django
-Requires-Dist: lxml
-Requires-Dist: matplotlib
+Requires-Dist: flask
+Requires-Dist: geopandas
 Requires-Dist: mysql-connector-python
 Requires-Dist: pandas
+Requires-Dist: psycopg
 Requires-Dist: psycopg2
 Requires-Dist: pyspark
-Requires-Dist: tqdm
-Requires-Dist: xlsxwriter
+Requires-Dist: sqlalchemy
 
 
 Data Manipulation is a Python package providing powerful utility functions. 

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation.egg-info/SOURCES.txt

@@ -19,6 +19,7 @@ data_manipulation/openldap_.py
 data_manipulation/pandas_.py
 data_manipulation/prometheus_.py
 data_manipulation/psycopg2_.py
+data_manipulation/psycopg_.py
 data_manipulation/pyspark_.py
 data_manipulation/smtplib_.py
 data_manipulation/sqlalchemy_.py

{data_manipulation-0.37 → data_manipulation-0.40}/data_manipulation.egg-info/requires.txt

@@ -1,10 +1,11 @@
 beautifulsoup4
+boto3
 django
-lxml
-matplotlib
+flask
+geopandas
 mysql-connector-python
 pandas
+psycopg
 psycopg2
 pyspark
-tqdm
-xlsxwriter
+sqlalchemy

{data_manipulation-0.37 → data_manipulation-0.40}/setup.py

@@ -23,15 +23,16 @@ CLASSIFIERS = [
 
 DEPENDENCIES = [
     "beautifulsoup4",
+    "boto3",
     "django",
-    "lxml",
-    "matplotlib",
+    "flask",
+    "geopandas",
     "mysql-connector-python",
     "pandas",
+    "psycopg",
     "psycopg2",
     "pyspark",
-    "tqdm",
-    "xlsxwriter",
+    "sqlalchemy",
 ]
 
 setup(

data_manipulation-0.37/data_manipulation/prometheus_.py

@@ -1,26 +0,0 @@
-# import logging
-# import subprocess
-
-# def ldapsearch(
-#     search_base: str,
-#     ldap_uri: str,
-#     bind_dn: str,
-#     password: str,
-#     filter: str,
-# ) -> subprocess.CompletedProcess:
-#     cmd = (
-#         "ldapsearch -b {search_base} -H {ldap_uri} -D {bind_dn} -w {password} {filter}"
-#     )
-#     output = subprocess.run(
-#         cmd,
-#         capture_output=True,
-#         shell=True,
-#         text=True,
-#     )
-#     return output
-
-
-# if __name__ == "__main__":
-#     import doctest
-
-#     doctest.testmod()