pydataframer-databricks 0.1.1__py3-none-any.whl → 0.1.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pydataframer_databricks/connectors.py +29 -34
- {pydataframer_databricks-0.1.1.dist-info → pydataframer_databricks-0.1.2.dist-info}/METADATA +1 -1
- pydataframer_databricks-0.1.2.dist-info/RECORD +5 -0
- pydataframer_databricks-0.1.1.dist-info/RECORD +0 -5
- {pydataframer_databricks-0.1.1.dist-info → pydataframer_databricks-0.1.2.dist-info}/WHEEL +0 -0
|
@@ -203,42 +203,37 @@ class DatabricksConnector:
|
|
|
203
203
|
raise ValueError(f"Unsupported file_type: {file_type}. Supported: CSV, JSON, JSONL for SINGLE_FILE datasets")
|
|
204
204
|
|
|
205
205
|
with self.get_connection() as connection:
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
f"`{col}` STRING" for col in pandas_df.columns
|
|
210
|
-
)
|
|
211
|
-
|
|
212
|
-
try:
|
|
213
|
-
cursor.execute(f"""
|
|
214
|
-
CREATE OR REPLACE TABLE {table_name} (
|
|
215
|
-
{columns_sql}
|
|
216
|
-
)
|
|
217
|
-
""")
|
|
218
|
-
except Exception as e:
|
|
219
|
-
error_msg = f"Failed to create table `{table_name}`"
|
|
220
|
-
print(f"{error_msg}: {str(e)}")
|
|
221
|
-
print("Verify table name format (catalog.schema.table), permissions, and warehouse is running")
|
|
222
|
-
cursor.close()
|
|
223
|
-
raise RuntimeError(f"{error_msg}: {str(e)}") from e
|
|
224
|
-
|
|
225
|
-
insert_sql = f"""
|
|
226
|
-
INSERT INTO {table_name}
|
|
227
|
-
VALUES ({", ".join(["?"] * len(pandas_df.columns))})
|
|
228
|
-
"""
|
|
229
|
-
|
|
230
|
-
try:
|
|
231
|
-
cursor.executemany(
|
|
232
|
-
insert_sql,
|
|
233
|
-
pandas_df.values.tolist()
|
|
206
|
+
with connection.cursor() as cursor:
|
|
207
|
+
columns_sql = ", ".join(
|
|
208
|
+
f"`{col}` STRING" for col in pandas_df.columns
|
|
234
209
|
)
|
|
235
|
-
except Exception as e:
|
|
236
|
-
error_msg = f"Failed to insert data into table `{table_name}`"
|
|
237
|
-
print(f"{error_msg}: {str(e)} | Rows attempted: {len(pandas_df)}")
|
|
238
|
-
cursor.close()
|
|
239
|
-
raise RuntimeError(f"{error_msg}: {str(e)}") from e
|
|
240
210
|
|
|
241
|
-
|
|
211
|
+
try:
|
|
212
|
+
cursor.execute(f"""
|
|
213
|
+
CREATE OR REPLACE TABLE {table_name} (
|
|
214
|
+
{columns_sql}
|
|
215
|
+
)
|
|
216
|
+
""")
|
|
217
|
+
except Exception as e:
|
|
218
|
+
error_msg = f"Failed to create table `{table_name}`"
|
|
219
|
+
print(f"{error_msg}: {str(e)}")
|
|
220
|
+
print("Verify table name format (catalog.schema.table), permissions, and warehouse is running")
|
|
221
|
+
raise RuntimeError(f"{error_msg}: {str(e)}") from e
|
|
222
|
+
|
|
223
|
+
insert_sql = f"""
|
|
224
|
+
INSERT INTO {table_name}
|
|
225
|
+
VALUES ({", ".join(["?"] * len(pandas_df.columns))})
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
try:
|
|
229
|
+
cursor.executemany(
|
|
230
|
+
insert_sql,
|
|
231
|
+
pandas_df.values.tolist()
|
|
232
|
+
)
|
|
233
|
+
except Exception as e:
|
|
234
|
+
error_msg = f"Failed to insert data into table `{table_name}`"
|
|
235
|
+
print(f"{error_msg}: {str(e)} | Rows attempted: {len(pandas_df)}")
|
|
236
|
+
raise RuntimeError(f"{error_msg}: {str(e)}") from e
|
|
242
237
|
|
|
243
238
|
print(f"✅ Table `{table_name}` saved successfully using Databricks SQL")
|
|
244
239
|
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
pydataframer_databricks/__init__.py,sha256=piRrFtKpGGc2ctFDnMNblp5Whp6froRKXNeYkHnrw_o,214
|
|
2
|
+
pydataframer_databricks/connectors.py,sha256=ZzRBuv-RGJeKoGaAtUTov_52yq2Tq_YKfOs5vFSQ9Zw,9389
|
|
3
|
+
pydataframer_databricks-0.1.2.dist-info/METADATA,sha256=VN0LL1dH5UL0f9eWJnf15gSA-kvyVoXemJGkgbRa6BI,1113
|
|
4
|
+
pydataframer_databricks-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
5
|
+
pydataframer_databricks-0.1.2.dist-info/RECORD,,
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
pydataframer_databricks/__init__.py,sha256=piRrFtKpGGc2ctFDnMNblp5Whp6froRKXNeYkHnrw_o,214
|
|
2
|
-
pydataframer_databricks/connectors.py,sha256=efzhUrls_ns0_rwSjXi4n8w_Adcez7cJt5VJT7qHc2Q,9377
|
|
3
|
-
pydataframer_databricks-0.1.1.dist-info/METADATA,sha256=7g23DdCJraqZQkWWemWlA3s8jmGM2nCmoDmWNZi6560,1113
|
|
4
|
-
pydataframer_databricks-0.1.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
|
|
5
|
-
pydataframer_databricks-0.1.1.dist-info/RECORD,,
|
|
File without changes
|