rgwfuncs 0.0.35__py3-none-any.whl → 0.0.38__py3-none-any.whl

rgwfuncs/df_lib.py

@@ -384,8 +384,7 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
                     raise ConnectionError(
                         "All attempts to connect to ClickHouse failed.")
 
-    def query_google_big_query(
-            db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
+    def query_google_big_query(db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
         json_file_path = db_preset['json_file_path']
         project_id = db_preset['project_id']
 
@@ -400,6 +399,56 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
 
         return pd.DataFrame(rows, columns=columns)
 
+
+    def query_athena(db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
+
+        def execute_athena_query(athena_client, query: str, database: str, output_bucket: str) -> str:
+            response = athena_client.start_query_execution(
+                QueryString=query,
+                QueryExecutionContext={"Database": database},
+                ResultConfiguration={"OutputLocation": output_bucket}
+            )
+            return response["QueryExecutionId"]
+
+        def wait_for_athena_query_to_complete(athena_client, query_execution_id: str):
+            while True:
+                response = athena_client.get_query_execution(QueryExecutionId=query_execution_id)
+                state = response["QueryExecution"]["Status"]["State"]
+                if state == "SUCCEEDED":
+                    break
+                elif state in ("FAILED", "CANCELLED"):
+                    raise Exception(f"Query failed with state: {state}")
+                time.sleep(1)
+
+        def download_athena_query_results(athena_client, query_execution_id: str) -> pd.DataFrame:
+            paginator = athena_client.get_paginator("get_query_results")
+            result_pages = paginator.paginate(QueryExecutionId=query_execution_id)
+            rows = []
+            columns = []
+            for page in result_pages:
+                if not columns:
+                    columns = [col["Name"] for col in page["ResultSet"]["ResultSetMetadata"]["ColumnInfo"]]
+                rows.extend(page["ResultSet"]["Rows"])
+
+            data = [[col.get("VarCharValue", None) for col in row["Data"]] for row in rows[1:]]
+            return pd.DataFrame(data, columns=columns)
+
+
+        aws_region = db_preset['aws_region']
+        database = db_preset['database']
+        output_bucket = db_preset['output_bucket']
+        
+        athena_client = boto3.client(
+            'athena',
+            region_name=aws_region,
+            aws_access_key_id=db_preset['aws_access_key'],
+            aws_secret_access_key=db_preset['aws_secret_key']
+        )
+
+        query_execution_id = execute_athena_query(athena_client, query, database, output_bucket)
+        wait_for_athena_query_to_complete(athena_client, query_execution_id)
+        return download_athena_query_results(athena_client, query_execution_id)
+
     # Assume the configuration file is located at ~/.rgwfuncsrc
     config_path = os.path.expanduser('~/.rgwfuncsrc')
     with open(config_path, 'r') as f:
@@ -422,6 +471,8 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
         return query_clickhouse(db_preset, query)
     elif db_type == 'google_big_query':
         return query_google_big_query(db_preset, query)
+    elif db_type == 'aws_athena':
+        return query_athena(db_preset, query)
     else:
         raise ValueError(f"Unsupported db_type: {db_type}")
 

{rgwfuncs-0.0.35.dist-info → rgwfuncs-0.0.38.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rgwfuncs
-Version: 0.0.35
+Version: 0.0.38
 Summary: A functional programming paradigm for mathematical modelling and data science
 Home-page: https://github.com/ryangerardwilson/rgwfunc
 Author: Ryan Gerard Wilson
@@ -75,6 +75,15 @@ A `.rgwfuncsrc` file (located at `vi ~/.rgwfuncsrc) is required for MSSQL, CLICK
           "db_type": "google_big_query",
           "json_file_path": "",
           "project_id": ""
+        },
+        {
+          "name": "athena_db1",
+          "db_type": "aws_athena",
+          "aws_access_key": "",
+          "aws_secret_key": "",
+          "aws_region: "",
+          "database": "logs",
+          "output_bucket": "s3://bucket-name"
         }
       ],
       "vm_presets": [
@@ -667,28 +676,49 @@ Drop duplicate rows based on specified columns, retaining the last occurrence.
 
 ### 11. `load_data_from_query`
 
-Load data from a database query into a DataFrame based on a configuration preset.
+Load data from a specified database using a SQL query and return the results in a Pandas DataFrame. The database connection configurations are determined by a preset name specified in a configuration file.
 
-- **Parameters:**
-  - `db_preset_name` (str): Name of the database preset in the configuration file.
-  - `query` (str): The SQL query to execute.
+#### Features
 
-- **Returns:**
-  - `pd.DataFrame`: A DataFrame containing the query result.
+- Multi-Database Support: This function supports different database types, including MSSQL, MySQL, ClickHouse, Google BigQuery, and AWS Athena, based on the configuration preset selected.
+- Configuration-Based: It utilizes a configuration file to store database connection details securely, avoiding hardcoding sensitive information directly into the script.
+- Dynamic Query Execution: Capable of executing custom user-defined SQL queries against the specified database.
+- Automatic Result Loading: Fetches query results and loads them directly into a Pandas DataFrame for further manipulation and analysis.
 
-- **Notes:**
-  - The configuration file is assumed to be located at `~/.rgwfuncsrc`.
+#### Parameters
 
-- **Example:**
+- `db_preset_name` (str): The name of the database preset found in the configuration file. This preset determines which database connection details to use.
+- `query` (str): The SQL query string to be executed on the database.
+
+#### Returns
 
-  from rgwfuncs import load_data_from_query
+- `pd.DataFrame`: Returns a DataFrame that contains the results from the executed SQL query.
+
+#### Configuration Details
+
+- The configuration file is expected to be in JSON format and located at `~/.rgwfuncsrc`.
+- Each preset within the configuration file must include:
+  - `name`: Name of the database preset.
+  - `db_type`: Type of the database (`mssql`, `mysql`, `clickhouse`, `google_big_query`, `aws_athena`).
+  - `credentials`: Necessary credentials such as host, username, password, and potentially others depending on the database type.
+
+#### Example
+
+    from rgwfuncs import load_data_from_query
+
+    # Load data using a preset configuration
+    df = load_data_from_query(
+        db_preset_name="MyDBPreset",
+        query="SELECT * FROM my_table"
+    )
+    print(df)
 
-  df = load_data_from_query(
-      db_preset_name="MyDBPreset",
-      query="SELECT * FROM my_table"
-  )
-  print(df)
+#### Notes
 
+- Security: Ensure that the configuration file (`~/.rgwfuncsrc`) is secure and accessible only to authorized users, as it contains sensitive information.
+- Pre-requisites: Ensure the necessary Python packages are installed for each database type you wish to query. For example, `pymssql` for MSSQL, `mysql-connector-python` for MySQL, and so on.
+- Error Handling: The function raises a `ValueError` if the specified preset name does not exist or if the database type is unsupported. Additional exceptions may arise from network issues or database errors.
+- Environment: For AWS Athena, ensure that AWS credentials are configured properly for the boto3 library to authenticate successfully. Consider using AWS IAM roles or AWS Secrets Manager for better security management.
     
 --------------------------------------------------------------------------------
 

{rgwfuncs-0.0.35.dist-info → rgwfuncs-0.0.38.dist-info}/RECORD

@@ -1,12 +1,12 @@
 rgwfuncs/__init__.py,sha256=CLPRpLtzXxyFHEjS-MrxnhXH0LdS6THjAC5sCHg0m3c,1520
 rgwfuncs/algebra_lib.py,sha256=g-sNkf9Hz4i17uRIgLUYLQlyUu8yROgsoJMujdj0U3Y,21577
-rgwfuncs/df_lib.py,sha256=G_H3PXNVeseX2YLjkkrmO9eXA_7r29swUZlbPBDZjXA,66612
+rgwfuncs/df_lib.py,sha256=0bjredJvn9Al8y2S_qXPKkh3z8adi9v6IBvOUl5Jidw,68974
 rgwfuncs/docs_lib.py,sha256=y3wSAOPO3qsA4HZ7xAtW8HimM8w-c8hjcEzMRLJ96ao,1960
 rgwfuncs/interactive_shell_lib.py,sha256=A7EWsYxAfDev_N0-2GjRvAtp0bAwBPHIczXb8Gu9fzI,1107
 rgwfuncs/str_lib.py,sha256=rtAdRlnSJIu3JhI-tA_A0wCiPK2m-zn5RoGpBxv_g-4,2228
-rgwfuncs-0.0.35.dist-info/LICENSE,sha256=7EI8xVBu6h_7_JlVw-yPhhOZlpY9hP8wal7kHtqKT_E,1074
-rgwfuncs-0.0.35.dist-info/METADATA,sha256=RbKY0PIiS4npO8zhpmWLlXeJP48gpHJg7vXU5hvzuLk,47467
-rgwfuncs-0.0.35.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-rgwfuncs-0.0.35.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
-rgwfuncs-0.0.35.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
-rgwfuncs-0.0.35.dist-info/RECORD,,
+rgwfuncs-0.0.38.dist-info/LICENSE,sha256=7EI8xVBu6h_7_JlVw-yPhhOZlpY9hP8wal7kHtqKT_E,1074
+rgwfuncs-0.0.38.dist-info/METADATA,sha256=A5dRwHKtHe9mv3rptslwDNBZxuE8JF7mGLpcWrF21bE,49805
+rgwfuncs-0.0.38.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+rgwfuncs-0.0.38.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
+rgwfuncs-0.0.38.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
+rgwfuncs-0.0.38.dist-info/RECORD,,