rgwfuncs 0.0.34__py3-none-any.whl → 0.0.36__py3-none-any.whl

rgwfuncs/df_lib.py

@@ -384,8 +384,7 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
                     raise ConnectionError(
                         "All attempts to connect to ClickHouse failed.")
 
-    def query_google_big_query(
-            db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
+    def query_google_big_query(db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
         json_file_path = db_preset['json_file_path']
         project_id = db_preset['project_id']
 
@@ -400,6 +399,56 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
 
         return pd.DataFrame(rows, columns=columns)
 
+
+    def query_athena(db_preset: Dict[str, Any], query: str) -> pd.DataFrame:
+
+        def execute_athena_query(athena_client, query: str, database: str, output_bucket: str) -> str:
+            response = athena_client.start_query_execution(
+                QueryString=query,
+                QueryExecutionContext={"Database": database},
+                ResultConfiguration={"OutputLocation": output_bucket}
+            )
+            return response["QueryExecutionId"]
+
+        def wait_for_athena_query_to_complete(athena_client, query_execution_id: str):
+            while True:
+                response = athena_client.get_query_execution(QueryExecutionId=query_execution_id)
+                state = response["QueryExecution"]["Status"]["State"]
+                if state == "SUCCEEDED":
+                    break
+                elif state in ("FAILED", "CANCELLED"):
+                    raise Exception(f"Query failed with state: {state}")
+                time.sleep(1)
+
+        def download_athena_query_results(athena_client, query_execution_id: str) -> pd.DataFrame:
+            paginator = athena_client.get_paginator("get_query_results")
+            result_pages = paginator.paginate(QueryExecutionId=query_execution_id)
+            rows = []
+            columns = []
+            for page in result_pages:
+                if not columns:
+                    columns = [col["Name"] for col in page["ResultSet"]["ResultSetMetadata"]["ColumnInfo"]]
+                rows.extend(page["ResultSet"]["Rows"])
+
+            data = [[col.get("VarCharValue", None) for col in row["Data"]] for row in rows[1:]]
+            return pd.DataFrame(data, columns=columns)
+
+
+        aws_region = db_preset['region']
+        database = db_preset['database']
+        output_bucket = db_preset['output_bucket']
+        
+        athena_client = boto3.client(
+            'athena',
+            region_name=aws_region,
+            aws_access_key_id=db_preset['aws_access_key'],
+            aws_secret_access_key=db_preset['aws_secret_key']
+        )
+
+        query_execution_id = execute_athena_query(athena_client, query, database, output_bucket)
+        wait_for_athena_query_to_complete(athena_client, query_execution_id)
+        return download_athena_query_results(athena_client, query_execution_id)
+
     # Assume the configuration file is located at ~/.rgwfuncsrc
     config_path = os.path.expanduser('~/.rgwfuncsrc')
     with open(config_path, 'r') as f:
@@ -422,6 +471,8 @@ def load_data_from_query(db_preset_name: str, query: str) -> pd.DataFrame:
         return query_clickhouse(db_preset, query)
     elif db_type == 'google_big_query':
         return query_google_big_query(db_preset, query)
+    elif db_type == 'athena':
+        return query_athena(db_preset, query)
     else:
         raise ValueError(f"Unsupported db_type: {db_type}")
 

{rgwfuncs-0.0.34.dist-info → rgwfuncs-0.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: rgwfuncs
-Version: 0.0.34
+Version: 0.0.36
 Summary: A functional programming paradigm for mathematical modelling and data science
 Home-page: https://github.com/ryangerardwilson/rgwfunc
 Author: Ryan Gerard Wilson
@@ -135,7 +135,7 @@ To display all docstrings, use:
 
 --------------------------------------------------------------------------------
 
-## Documentation Access Functions
+## Documentation Access
 
 ### 1. docs
 Print a list of available function names in alphabetical order. If a filter is provided, print the docstrings of functions containing the term.
@@ -150,15 +150,13 @@ Print a list of available function names in alphabetical order. If a filter is p
 
 --------------------------------------------------------------------------------
 
-## Interactive Shell Function
+## Interactive Shell
 
 This section includes functions that facilitate launching an interactive Python shell to inspect and modify local variables within the user's environment.
 
 ### 1. `interactive_shell`
 
-#### Usage
-
-Launches an interactive prompt for inspecting and modifying local variables, making all methods in the rgwfuncs library available by default. This REPL (Read-Eval-Print Loop) environment supports command history and autocompletion, making it easier to interact with your Python code.
+Launches an interactive prompt for inspecting and modifying local variables, making all methods in the rgwfuncs library available by default. This REPL (Read-Eval-Print Loop) environment supports command history and autocompletion, making it easier to interact with your Python code. This function is particularly useful for debugging purposes when you want real-time interaction with your program's execution environment.
 
 • Parameters:
   - `local_vars` (dict, optional): A dictionary of local variables to be accessible within the interactive shell. If not provided, defaults to an empty dictionary.
@@ -180,18 +178,10 @@ Launches an interactive prompt for inspecting and modifying local variables, mak
         'city': ['New York', 'Los Angeles', 'Chicago', 'San Francisco', 'Boston']
     })
 
-    # Function to retrieve the first n rows of a DataFrame
-    def first_n_rows(df, n):
-        return df.head(n).to_dict('records')
-
     # Launch the interactive shell with local variables
     interactive_shell(locals())
 
-- Once in the interactive shell, you are greeted with a welcome message. You can access the variables defined in the local scope where `interactive_shell(locals())` was called, including any imported modules such as `pandas` (accessed as `pd`) and `numpy` (accessed as `np`). This means you can directly use these modules in the interactive session. Type `exit()` to quit the shell.
-
-#### Interactive Shell Example Sessions
-
-1. DataFrame Inspection with Pandas:
+Subsequently, in the interactive shell you can use any library in your python file, as well as all rgwfuncs methods (even if they are not imported). Notice, that while pandas and numpy are available in the shell as a result of importing them in the above script, the rgwfuncs method `first_n_rows` was not imported - yet is available for use.
 
     Welcome to the rgwfuncs interactive shell.
     >>> pirst_n_rows(df, 2)
@@ -208,23 +198,10 @@ Launches an interactive prompt for inspecting and modifying local variables, mak
     2  3  Charlie  35        Chicago
     3  4    David  28  San Francisco
     4  5      Eva  22         Boston
-
-2. Array Operations with NumPy:
-
-    Welcome to the rgwfuncs interactive shell.
     >>> arr = np.array([1, 2, 3, 4, 5])
     >>> arr
     array([1, 2, 3, 4, 5])
 
-These examples illustrate how you can use functions and variables within the interactive shell, handle errors with meaningful suggestions, and perform operations using external libraries like pandas and numpy.
-
-#### Key Features
-
-- Autocompletion: Uses the `rlcompleter` library to provide tab-completion of variables and functions names, enhancing ease of use.
-- History Support: Utilizes `readline` for command-line history, allowing you to navigate through previous commands using the arrow keys.
-
-This function is particularly useful for debugging purposes when you want real-time interaction with your program's execution environment.
-
 --------------------------------------------------------------------------------
 
 ## Algebra Based Functions
@@ -690,28 +667,49 @@ Drop duplicate rows based on specified columns, retaining the last occurrence.
 
 ### 11. `load_data_from_query`
 
-Load data from a database query into a DataFrame based on a configuration preset.
+Load data from a specified database using a SQL query and return the results in a Pandas DataFrame. The database connection configurations are determined by a preset name specified in a configuration file.
 
-- **Parameters:**
-  - `db_preset_name` (str): Name of the database preset in the configuration file.
-  - `query` (str): The SQL query to execute.
+#### Features
 
-- **Returns:**
-  - `pd.DataFrame`: A DataFrame containing the query result.
+- Multi-Database Support: This function supports different database types, including MSSQL, MySQL, ClickHouse, Google BigQuery, and AWS Athena, based on the configuration preset selected.
+- Configuration-Based: It utilizes a configuration file to store database connection details securely, avoiding hardcoding sensitive information directly into the script.
+- Dynamic Query Execution: Capable of executing custom user-defined SQL queries against the specified database.
+- Automatic Result Loading: Fetches query results and loads them directly into a Pandas DataFrame for further manipulation and analysis.
 
-- **Notes:**
-  - The configuration file is assumed to be located at `~/.rgwfuncsrc`.
+#### Parameters
 
-- **Example:**
+- `db_preset_name` (str): The name of the database preset found in the configuration file. This preset determines which database connection details to use.
+- `query` (str): The SQL query string to be executed on the database.
+
+#### Returns
+
+- `pd.DataFrame`: Returns a DataFrame that contains the results from the executed SQL query.
+
+#### Configuration Details
 
-  from rgwfuncs import load_data_from_query
+- The configuration file is expected to be in JSON format and located at `~/.rgwfuncsrc`.
+- Each preset within the configuration file must include:
+  - `name`: Name of the database preset.
+  - `db_type`: Type of the database (`mssql`, `mysql`, `clickhouse`, `google_big_query`, `aws_athena`).
+  - `credentials`: Necessary credentials such as host, username, password, and potentially others depending on the database type.
+
+#### Example
+
+    from rgwfuncs import load_data_from_query
+
+    # Load data using a preset configuration
+    df = load_data_from_query(
+        db_preset_name="MyDBPreset",
+        query="SELECT * FROM my_table"
+    )
+    print(df)
 
-  df = load_data_from_query(
-      db_preset_name="MyDBPreset",
-      query="SELECT * FROM my_table"
-  )
-  print(df)
+#### Notes
 
+- Security: Ensure that the configuration file (`~/.rgwfuncsrc`) is secure and accessible only to authorized users, as it contains sensitive information.
+- Pre-requisites: Ensure the necessary Python packages are installed for each database type you wish to query. For example, `pymssql` for MSSQL, `mysql-connector-python` for MySQL, and so on.
+- Error Handling: The function raises a `ValueError` if the specified preset name does not exist or if the database type is unsupported. Additional exceptions may arise from network issues or database errors.
+- Environment: For AWS Athena, ensure that AWS credentials are configured properly for the boto3 library to authenticate successfully. Consider using AWS IAM roles or AWS Secrets Manager for better security management.
     
 --------------------------------------------------------------------------------
 

{rgwfuncs-0.0.34.dist-info → rgwfuncs-0.0.36.dist-info}/RECORD

@@ -1,12 +1,12 @@
 rgwfuncs/__init__.py,sha256=CLPRpLtzXxyFHEjS-MrxnhXH0LdS6THjAC5sCHg0m3c,1520
 rgwfuncs/algebra_lib.py,sha256=g-sNkf9Hz4i17uRIgLUYLQlyUu8yROgsoJMujdj0U3Y,21577
-rgwfuncs/df_lib.py,sha256=G_H3PXNVeseX2YLjkkrmO9eXA_7r29swUZlbPBDZjXA,66612
+rgwfuncs/df_lib.py,sha256=VYs_2avxVODnxBAuoZ9eQ8dDY1rcNMGmbgwcRkDv_VA,68966
 rgwfuncs/docs_lib.py,sha256=y3wSAOPO3qsA4HZ7xAtW8HimM8w-c8hjcEzMRLJ96ao,1960
 rgwfuncs/interactive_shell_lib.py,sha256=A7EWsYxAfDev_N0-2GjRvAtp0bAwBPHIczXb8Gu9fzI,1107
 rgwfuncs/str_lib.py,sha256=rtAdRlnSJIu3JhI-tA_A0wCiPK2m-zn5RoGpBxv_g-4,2228
-rgwfuncs-0.0.34.dist-info/LICENSE,sha256=7EI8xVBu6h_7_JlVw-yPhhOZlpY9hP8wal7kHtqKT_E,1074
-rgwfuncs-0.0.34.dist-info/METADATA,sha256=JuG4wIxpjCaw7ST1u2lQHD9fBAUC9lC68X1E_4OztuM,48334
-rgwfuncs-0.0.34.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-rgwfuncs-0.0.34.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
-rgwfuncs-0.0.34.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
-rgwfuncs-0.0.34.dist-info/RECORD,,
+rgwfuncs-0.0.36.dist-info/LICENSE,sha256=7EI8xVBu6h_7_JlVw-yPhhOZlpY9hP8wal7kHtqKT_E,1074
+rgwfuncs-0.0.36.dist-info/METADATA,sha256=jU298sn67VvVuv85s4crL4B_833DaxkuHMA0c7Cyf88,49550
+rgwfuncs-0.0.36.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+rgwfuncs-0.0.36.dist-info/entry_points.txt,sha256=j-c5IOPIQ0252EaOV6j6STio56sbXl2C4ym_fQ0lXx0,43
+rgwfuncs-0.0.36.dist-info/top_level.txt,sha256=aGuVIzWsKiV1f2gCb6mynx0zx5ma0B1EwPGFKVEMTi4,9
+rgwfuncs-0.0.36.dist-info/RECORD,,