rgwfuncs 0.0.8__tar.gz → 0.0.10__tar.gz

{rgwfuncs-0.0.8/src/rgwfuncs.egg-info → rgwfuncs-0.0.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rgwfuncs
-Version: 0.0.8
+Version: 0.0.10
 Summary: A functional programming paradigm for mathematical modelling and data science
 Home-page: https://github.com/ryangerardwilson/rgwfunc
 Author: Ryan Gerard Wilson
@@ -1151,6 +1151,30 @@ Perform a right join on two DataFrames.
 
 --------------------------------------------------------------------------------
 
+### 45. `sync_dataframe_to_sqlite_database`
+Processes and saves a DataFrame to an SQLite database, adding a timestamp column and replacing the existing table if needed. Creates the table if it does not exist.
+
+• Parameters:
+  - `db_path` (str): Path to the SQLite database file.
+  - `tablename` (str): The name of the table in the database.
+  - `df` (pd.DataFrame): The DataFrame to be processed and saved.
+
+• Returns:
+  - None
+
+• Example:
+
+    from rgwfuncs import sync_dataframe_to_sqlite_database
+    import pandas as pd
+
+    df = pd.DataFrame({'ID': [1, 2, 3], 'Value': [10, 20, 30]})
+    db_path = 'my_database.db'
+    tablename = 'my_table'
+
+    sync_dataframe_to_sqlite_database(db_path, tablename, df)
+
+--------------------------------------------------------------------------------
+
 ## Additional Info
 
 For more information, refer to each function’s docstring by calling:

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10}/README.md

@@ -1125,6 +1125,30 @@ Perform a right join on two DataFrames.
 
 --------------------------------------------------------------------------------
 
+### 45. `sync_dataframe_to_sqlite_database`
+Processes and saves a DataFrame to an SQLite database, adding a timestamp column and replacing the existing table if needed. Creates the table if it does not exist.
+
+• Parameters:
+  - `db_path` (str): Path to the SQLite database file.
+  - `tablename` (str): The name of the table in the database.
+  - `df` (pd.DataFrame): The DataFrame to be processed and saved.
+
+• Returns:
+  - None
+
+• Example:
+
+    from rgwfuncs import sync_dataframe_to_sqlite_database
+    import pandas as pd
+
+    df = pd.DataFrame({'ID': [1, 2, 3], 'Value': [10, 20, 30]})
+    db_path = 'my_database.db'
+    tablename = 'my_table'
+
+    sync_dataframe_to_sqlite_database(db_path, tablename, df)
+
+--------------------------------------------------------------------------------
+
 ## Additional Info
 
 For more information, refer to each function’s docstring by calling:

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rgwfuncs"
-version = "0.0.8"
+version = "0.0.10"
 authors = [
   { name = "Ryan Gerard Wilson", email = "ryangerardwilson@gmail.com" },
 ]

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = rgwfuncs
-version = 0.0.8
+version = 0.0.10
 author = Ryan Gerard Wilson
 author_email = ryangerardwilson@gmail.com
 description = A functional programming paradigm for mathematical modelling and data science

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10}/src/rgwfuncs/__init__.py

@@ -1,4 +1,4 @@
 # This file is automatically generated
 # Dynamically importing functions from modules
 
-from .df_lib import append_columns, append_percentile_classification_column, append_ranged_classification_column, append_ranged_date_classification_column, append_rows, append_xgb_labels, append_xgb_logistic_regression_predictions, append_xgb_regression_predictions, bag_union_join, bottom_n_unique_values, cascade_sort, delete_rows, docs, drop_duplicates, drop_duplicates_retain_first, drop_duplicates_retain_last, filter_dataframe, filter_indian_mobiles, first_n_rows, from_raw_data, last_n_rows, left_join, limit_dataframe, load_data_from_path, load_data_from_query, load_data_from_sqlite_path, mask_against_dataframe, mask_against_dataframe_converse, numeric_clean, order_columns, print_correlation, print_dataframe, print_memory_usage, print_n_frequency_cascading, print_n_frequency_linear, rename_columns, retain_columns, right_join, send_data_to_email, send_data_to_slack, send_dataframe_via_telegram, top_n_unique_values, union_join, update_rows
+from .df_lib import append_columns, append_percentile_classification_column, append_ranged_classification_column, append_ranged_date_classification_column, append_rows, append_xgb_labels, append_xgb_logistic_regression_predictions, append_xgb_regression_predictions, bag_union_join, bottom_n_unique_values, cascade_sort, delete_rows, docs, drop_duplicates, drop_duplicates_retain_first, drop_duplicates_retain_last, filter_dataframe, filter_indian_mobiles, first_n_rows, from_raw_data, last_n_rows, left_join, limit_dataframe, load_data_from_path, load_data_from_query, load_data_from_sqlite_path, mask_against_dataframe, mask_against_dataframe_converse, numeric_clean, order_columns, print_correlation, print_dataframe, print_memory_usage, print_n_frequency_cascading, print_n_frequency_linear, rename_columns, retain_columns, right_join, send_data_to_email, send_data_to_slack, send_dataframe_via_telegram, sync_dataframe_to_sqlite_database, top_n_unique_values, union_join, update_rows

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10}/src/rgwfuncs/df_lib.py

@@ -1631,7 +1631,15 @@ def union_join(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame:
     if set(df1.columns) != set(df2.columns):
         raise ValueError("Both DataFrames must have the same columns for a union join")
 
-    result_df = pd.concat([df1, df2], ignore_index=True).drop_duplicates()
+    # Drop all-NA columns, if any
+    df1_clean = df1.dropna(axis=1, how='all')
+    df2_clean = df2.dropna(axis=1, how='all')
+
+    # Ensure they still have the same columns after dropping all-NA columns
+    if set(df1_clean.columns) != set(df2_clean.columns):
+        raise ValueError("Both DataFrames must have the same columns after dropping all-NA columns")
+    
+    result_df = pd.concat([df1_clean, df2_clean], ignore_index=True).drop_duplicates()
     return result_df
 
 
@@ -1652,7 +1660,15 @@ def bag_union_join(df1: pd.DataFrame, df2: pd.DataFrame) -> pd.DataFrame:
     if set(df1.columns) != set(df2.columns):
         raise ValueError("Both DataFrames must have the same columns for a bag union join")
 
-    result_df = pd.concat([df1, df2], ignore_index=True)
+    # Drop all-NA columns, if any
+    df1_clean = df1.dropna(axis=1, how='all')
+    df2_clean = df2.dropna(axis=1, how='all')
+
+    # Ensure they still have the same columns after dropping all-NA columns
+    if set(df1_clean.columns) != set(df2_clean.columns):
+        raise ValueError("Both DataFrames must have the same columns after dropping all-NA columns")
+
+    result_df = pd.concat([df1_clean, df2_clean], ignore_index=True)
     return result_df
 
 
@@ -1686,3 +1702,60 @@ def right_join(df1: pd.DataFrame, df2: pd.DataFrame, left_on: str, right_on: str
         A new DataFrame as the result of a right join.
     """
     return df1.merge(df2, how='right', left_on=left_on, right_on=right_on)
+
+def sync_dataframe_to_sqlite_database(db_path: str, tablename: str, df: pd.DataFrame) -> None:
+    """
+    Processes and saves a DataFrame to an SQLite database, adding a timestamp column
+    and replacing the existing table if needed. Creates the table if it does not exist.
+
+    Parameters:
+    - db_path (str): Path to the SQLite database file.
+    - tablename (str): The name of the table in the database.
+    - df (pd.DataFrame): The DataFrame to be processed and saved.
+    """
+    # Step 1: Add a timestamp column to the dataframe
+    df['rgwfuncs_sync_timestamp'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+    # Define a simple mapping from pandas dtypes to SQLite types
+    dtype_mapping = {
+        'int64': 'INTEGER',
+        'float64': 'REAL',
+        'object': 'TEXT',
+        'datetime64[ns]': 'TEXT',  # Dates are stored as text in SQLite
+        'bool': 'INTEGER',  # SQLite does not have a separate Boolean storage class
+    }
+
+    # Helper function to map pandas dtype to SQLite type
+    def map_dtype(dtype):
+        return dtype_mapping.get(str(dtype), 'TEXT')
+
+    # Step 2: Save df in SQLite3 db as '{tablename}_new'
+    with sqlite3.connect(db_path) as conn:
+        new_table_name = f"{tablename}_new"
+
+        # Check if the new table already exists, create if not
+        cursor = conn.cursor()
+        cursor.execute(f"PRAGMA table_info({new_table_name})")
+        if cursor.fetchall() == []:  # Table does not exist
+            # Create a table using the DataFrame's column names and types
+            columns_with_types = ', '.join(
+                f'"{col}" {map_dtype(dtype)}' for col, dtype in zip(df.columns, df.dtypes)
+            )
+            create_table_query = f'CREATE TABLE "{new_table_name}" ({columns_with_types})'
+            conn.execute(create_table_query)
+
+        # Insert data into the new table
+        df.to_sql(new_table_name, conn, if_exists='replace', index=False)
+
+        # Step 3: If '{tablename}_new' is not empty, delete table '{tablename}' (if it exists), and rename '{tablename}_new' to '{tablename}'
+        # Check if the new table is not empty
+        cursor.execute(f"SELECT COUNT(*) FROM {new_table_name}")
+        count = cursor.fetchone()[0]
+
+        if count > 0:
+            # Drop the old table if it exists
+            conn.execute(f"DROP TABLE IF EXISTS {tablename}")
+            # Rename the new table to the old table name
+            conn.execute(f"ALTER TABLE {new_table_name} RENAME TO {tablename}")
+
+

{rgwfuncs-0.0.8 → rgwfuncs-0.0.10/src/rgwfuncs.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rgwfuncs
-Version: 0.0.8
+Version: 0.0.10
 Summary: A functional programming paradigm for mathematical modelling and data science
 Home-page: https://github.com/ryangerardwilson/rgwfunc
 Author: Ryan Gerard Wilson
@@ -1151,6 +1151,30 @@ Perform a right join on two DataFrames.
 
 --------------------------------------------------------------------------------
 
+### 45. `sync_dataframe_to_sqlite_database`
+Processes and saves a DataFrame to an SQLite database, adding a timestamp column and replacing the existing table if needed. Creates the table if it does not exist.
+
+• Parameters:
+  - `db_path` (str): Path to the SQLite database file.
+  - `tablename` (str): The name of the table in the database.
+  - `df` (pd.DataFrame): The DataFrame to be processed and saved.
+
+• Returns:
+  - None
+
+• Example:
+
+    from rgwfuncs import sync_dataframe_to_sqlite_database
+    import pandas as pd
+
+    df = pd.DataFrame({'ID': [1, 2, 3], 'Value': [10, 20, 30]})
+    db_path = 'my_database.db'
+    tablename = 'my_table'
+
+    sync_dataframe_to_sqlite_database(db_path, tablename, df)
+
+--------------------------------------------------------------------------------
+
 ## Additional Info
 
 For more information, refer to each function’s docstring by calling: