pydataframer-databricks 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

pydataframer_databricks/connectors.py

@@ -33,7 +33,7 @@ class DatabricksConnector:
     
     Examples
     --------
-    >>> databricks_connector = DatabricksConnector(dbutils)
+    >>> databricks_connector = DatabricksConnector(dbutils, scope="dataframer")
     >>> df = databricks_connector.fetch_sample_data(
     ...     num_items_to_select=25,
     ...     table_name="samples.bakehouse.media_customer_reviews"
@@ -41,7 +41,7 @@ class DatabricksConnector:
     >>> df.head()
     """
     
-    def __init__(self, dbutils):
+    def __init__(self, dbutils, scope):
         """
         Initialize the Databricks connector.
         
@@ -49,8 +49,11 @@ class DatabricksConnector:
         ----------
         dbutils : DBUtils
             The dbutils object from your Databricks notebook context.
+        scope : str
+            The Databricks secret scope name containing connection credentials.
         """
         self.dbutils = dbutils
+        self.scope = scope
     
     def get_connection(self):
         """
@@ -64,14 +67,14 @@ class DatabricksConnector:
         from databricks import sql
         from databricks.sdk.core import Config, oauth_service_principal
 
-        server_hostname = self.dbutils.secrets.get("dataframer", "DATABRICKS_SERVER_HOSTNAME")
-        http_path = self.dbutils.secrets.get("dataframer", "DATABRICKS_HTTP_PATH")
+        server_hostname = self.dbutils.secrets.get(self.scope, "DATABRICKS_SERVER_HOSTNAME")
+        http_path = self.dbutils.secrets.get(self.scope, "DATABRICKS_HTTP_PATH")
 
         def credential_provider():
             config = Config(
                 host=f"https://{server_hostname}",
-                client_id=self.dbutils.secrets.get("dataframer", "DATABRICKS_CLIENT_ID"),
-                client_secret=self.dbutils.secrets.get("dataframer", "DATABRICKS_CLIENT_SECRET"),
+                client_id=self.dbutils.secrets.get(self.scope, "DATABRICKS_CLIENT_ID"),
+                client_secret=self.dbutils.secrets.get(self.scope, "DATABRICKS_CLIENT_SECRET"),
             )
             return oauth_service_principal(config)
 
@@ -100,7 +103,7 @@ class DatabricksConnector:
         
         Examples
         --------
-        >>> databricks_connector = DatabricksConnector(dbutils)
+        >>> databricks_connector = DatabricksConnector(dbutils, scope="dataframer")
         >>> df = databricks_connector.fetch_sample_data(
         ...     num_items_to_select=25,
         ...     table_name="samples.bakehouse.media_customer_reviews"
@@ -146,7 +149,7 @@ class DatabricksConnector:
         
         Examples
         --------
-        >>> databricks_connector = DatabricksConnector(dbutils)
+        >>> databricks_connector = DatabricksConnector(dbutils, scope="dataframer")
         >>> with open("samples.zip", "rb") as f:
         ...     databricks_connector.load_generated_data(
         ...         table_name="my_catalog.my_schema.my_table",
@@ -200,42 +203,37 @@ class DatabricksConnector:
                 raise ValueError(f"Unsupported file_type: {file_type}. Supported: CSV, JSON, JSONL for SINGLE_FILE datasets")
             
             with self.get_connection() as connection:
-                cursor = connection.cursor()
-
-                columns_sql = ", ".join(
-                    f"`{col}` STRING" for col in pandas_df.columns
-                )
-
-                try:
-                    cursor.execute(f"""
-                        CREATE OR REPLACE TABLE {table_name} (
-                            {columns_sql}
-                        )
-                    """)
-                except Exception as e:
-                    error_msg = f"Failed to create table `{table_name}`"
-                    print(f"{error_msg}: {str(e)}")
-                    print("Verify table name format (catalog.schema.table), permissions, and warehouse is running")
-                    cursor.close()
-                    raise RuntimeError(f"{error_msg}: {str(e)}") from e
-
-                insert_sql = f"""
-                    INSERT INTO {table_name}
-                    VALUES ({", ".join(["?"] * len(pandas_df.columns))})
-                """
-
-                try:
-                    cursor.executemany(
-                        insert_sql,
-                        pandas_df.values.tolist()
+                with connection.cursor() as cursor:
+                    columns_sql = ", ".join(
+                        f"`{col}` STRING" for col in pandas_df.columns
                     )
-                except Exception as e:
-                    error_msg = f"Failed to insert data into table `{table_name}`"
-                    print(f"{error_msg}: {str(e)} | Rows attempted: {len(pandas_df)}")
-                    cursor.close()
-                    raise RuntimeError(f"{error_msg}: {str(e)}") from e
 
-                cursor.close()
+                    try:
+                        cursor.execute(f"""
+                            CREATE OR REPLACE TABLE {table_name} (
+                                {columns_sql}
+                            )
+                        """)
+                    except Exception as e:
+                        error_msg = f"Failed to create table `{table_name}`"
+                        print(f"{error_msg}: {str(e)}")
+                        print("Verify table name format (catalog.schema.table), permissions, and warehouse is running")
+                        raise RuntimeError(f"{error_msg}: {str(e)}") from e
+
+                    insert_sql = f"""
+                        INSERT INTO {table_name}
+                        VALUES ({", ".join(["?"] * len(pandas_df.columns))})
+                    """
+
+                    try:
+                        cursor.executemany(
+                            insert_sql,
+                            pandas_df.values.tolist()
+                        )
+                    except Exception as e:
+                        error_msg = f"Failed to insert data into table `{table_name}`"
+                        print(f"{error_msg}: {str(e)} | Rows attempted: {len(pandas_df)}")
+                        raise RuntimeError(f"{error_msg}: {str(e)}") from e
 
             print(f"✅ Table `{table_name}` saved successfully using Databricks SQL")
         

pydataframer_databricks-0.1.2.dist-info/METADATA

@@ -0,0 +1,46 @@
+Metadata-Version: 2.4
+Name: pydataframer-databricks
+Version: 0.1.2
+Summary: Databricks connector for Dataframer
+Author-email: Dataframer <info@dataframer.ai>
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: databricks-sdk>=0.81.0
+Requires-Dist: databricks-sql-connector>=4.2.4
+Requires-Dist: pandas>=2.0.0
+Provides-Extra: dev
+Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
+Requires-Dist: pytest>=7.4.0; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# pydataframer-databricks
+
+Databricks connector for Dataframer
+
+## Installation
+
+```bash
+pip install pydataframer-databricks
+```
+
+## Usage
+
+```python
+from pydataframer_databricks import DatabricksConnector, DatasetType, FileType
+
+databricks_connector = DatabricksConnector(dbutils, scope="dataframer")
+
+# Fetch sample data
+df = databricks_connector.fetch_sample_data(
+    num_items_to_select=100,
+    table_name="catalog.schema.table"
+)
+
+# Load generated data
+databricks_connector.load_generated_data(
+    table_name="catalog.schema.table",
+    downloaded_zip=downloaded_zip,
+    dataset_type=DatasetType.SINGLE_FILE,
+    file_type=FileType.CSV
+)
+```

pydataframer_databricks-0.1.2.dist-info/RECORD

@@ -0,0 +1,5 @@
+pydataframer_databricks/__init__.py,sha256=piRrFtKpGGc2ctFDnMNblp5Whp6froRKXNeYkHnrw_o,214
+pydataframer_databricks/connectors.py,sha256=ZzRBuv-RGJeKoGaAtUTov_52yq2Tq_YKfOs5vFSQ9Zw,9389
+pydataframer_databricks-0.1.2.dist-info/METADATA,sha256=VN0LL1dH5UL0f9eWJnf15gSA-kvyVoXemJGkgbRa6BI,1113
+pydataframer_databricks-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+pydataframer_databricks-0.1.2.dist-info/RECORD,,

pydataframer_databricks-0.1.0.dist-info/METADATA

@@ -1,43 +0,0 @@
-Metadata-Version: 2.4
-Name: pydataframer-databricks
-Version: 0.1.0
-Summary: Databricks connector for Dataframer
-Author-email: Dataframer <info@dataframer.ai>
-License: MIT
-Requires-Python: >=3.9
-Requires-Dist: databricks-sdk>=0.81.0
-Requires-Dist: databricks-sql-connector>=4.2.4
-Requires-Dist: pandas>=2.0.0
-Provides-Extra: dev
-Requires-Dist: pytest-cov>=4.1.0; extra == 'dev'
-Requires-Dist: pytest>=7.4.0; extra == 'dev'
-Description-Content-Type: text/markdown
-
-# pydataframer-databricks
-
-Databricks connector package for Dataframer, providing seamless integration with Databricks SQL and data operations.
-
-## Installation
-
-```bash
-pip install pydataframer-databricks
-```
-
-## Building
-
-Requires [uv](https://docs.astral.sh/uv/) installed in your environment.
-
-```bash
-uv build
-```
-
-## Development
-
-```bash
-# Install with dev dependencies
-uv pip install -e ".[dev]"
-
-# Run tests
-pytest
-```
-

pydataframer_databricks-0.1.0.dist-info/RECORD

@@ -1,5 +0,0 @@
-pydataframer_databricks/__init__.py,sha256=piRrFtKpGGc2ctFDnMNblp5Whp6froRKXNeYkHnrw_o,214
-pydataframer_databricks/connectors.py,sha256=E4RlU30ADp0V27tuHWOai-7CM1YvmTInS_YonUpWMds,9191
-pydataframer_databricks-0.1.0.dist-info/METADATA,sha256=L-5a9ThsJYq_CTeAI22Zlo269NoXH3jRtyFOW_dPxAQ,891
-pydataframer_databricks-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-pydataframer_databricks-0.1.0.dist-info/RECORD,,