sciduckdb 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/PKG-INFO +2 -2
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/src/sciduckdb/connection.py +53 -34
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/.gitignore +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/README.md +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/pyproject.toml +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/src/sciduckdb/__init__.py +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/src/sciduckdb/codec.py +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/src/sciduckdb/datetimes.py +0 -0
- {sciduckdb-0.2.0 → sciduckdb-0.2.2}/src/sciduckdb/row_lists.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: sciduckdb
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: A thin DuckDB layer for managing versioned scientific data
|
|
5
5
|
Author: SciStack Contributors
|
|
6
6
|
License-Expression: MIT
|
|
@@ -21,7 +21,7 @@ Requires-Python: >=3.10
|
|
|
21
21
|
Requires-Dist: duckdb>=0.9.0
|
|
22
22
|
Requires-Dist: numpy>=1.20
|
|
23
23
|
Requires-Dist: pandas>=1.3
|
|
24
|
-
Requires-Dist: scicanonicalhash==0.2.
|
|
24
|
+
Requires-Dist: scicanonicalhash==0.2.2
|
|
25
25
|
Provides-Extra: dev
|
|
26
26
|
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
27
27
|
Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
@@ -234,52 +234,75 @@ class DuckConnection:
|
|
|
234
234
|
self._recover_from_autocommit_failure()
|
|
235
235
|
raise
|
|
236
236
|
|
|
237
|
-
def
|
|
238
|
-
"""Insert
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
``
|
|
248
|
-
the
|
|
249
|
-
(the register/insert/unregister run on the same connection).
|
|
237
|
+
def insert_frame(self, table: str, frame, *, conflict=None) -> int:
|
|
238
|
+
"""Insert every row of *frame* (a pandas DataFrame or a pyarrow Table)
|
|
239
|
+
into *table* with one ``INSERT ... SELECT``, under the lock.
|
|
240
|
+
|
|
241
|
+
THE one way to insert a frame: register, insert, unregister on this
|
|
242
|
+
connection, with the same contention log and autocommit-failure
|
|
243
|
+
recovery as :meth:`execute`. Before 2026-11-02 scidb's bulk Arrow
|
|
244
|
+
inserts (``RecordStore``) reached past the connection to the raw
|
|
245
|
+
``duckdb`` handle and skipped both (design review F14). Columns are
|
|
246
|
+
the frame's, by name. ``conflict`` (column names) appends
|
|
247
|
+
``ON CONFLICT (...) DO NOTHING``. Safe inside an open transaction.
|
|
248
|
+
Returns the number of rows handed in.
|
|
250
249
|
"""
|
|
251
|
-
|
|
252
|
-
if
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
250
|
+
columns = list(getattr(frame, "column_names", None) or frame.columns)
|
|
251
|
+
n_rows = frame.num_rows if hasattr(frame, "num_rows") else len(frame)
|
|
252
|
+
if n_rows == 0:
|
|
253
|
+
return 0
|
|
254
|
+
view = "_insert_frame"
|
|
256
255
|
col_str = ", ".join(f'"{c}"' for c in columns)
|
|
257
|
-
sql = f'INSERT INTO "{table}" ({col_str}) SELECT
|
|
258
|
-
if
|
|
259
|
-
conflict_str = ", ".join(f'"{c}"' for c in
|
|
256
|
+
sql = f'INSERT INTO "{table}" ({col_str}) SELECT {col_str} FROM {view}'
|
|
257
|
+
if conflict:
|
|
258
|
+
conflict_str = ", ".join(f'"{c}"' for c in conflict)
|
|
260
259
|
sql += f" ON CONFLICT ({conflict_str}) DO NOTHING"
|
|
261
260
|
thread = threading.get_ident()
|
|
262
261
|
wait_start = time.monotonic()
|
|
263
262
|
with self._lock:
|
|
264
263
|
waited = time.monotonic() - wait_start
|
|
265
264
|
foreign_tx = self._tx_owner is not None and self._tx_owner != thread
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
)
|
|
271
|
-
self.con.register("_bulk_insert_df", df)
|
|
265
|
+
self._log_contended("insert_frame", thread, waited, foreign_tx, sql,
|
|
266
|
+
detail=f"table={table} rows={n_rows} ")
|
|
267
|
+
t0 = time.perf_counter()
|
|
268
|
+
self.con.register(view, frame)
|
|
272
269
|
try:
|
|
273
270
|
self.con.execute(sql)
|
|
274
271
|
except Exception:
|
|
275
272
|
logger.exception(
|
|
276
|
-
"
|
|
277
|
-
|
|
273
|
+
"insert_frame FAILED thread=%d tx_owner=%s foreign_tx=%s "
|
|
274
|
+
"table=%s rows=%d",
|
|
275
|
+
thread, self._tx_owner, foreign_tx, table, n_rows,
|
|
278
276
|
)
|
|
279
277
|
self._recover_from_autocommit_failure()
|
|
280
278
|
raise
|
|
281
279
|
finally:
|
|
282
|
-
self._unregister_frame(
|
|
280
|
+
self._unregister_frame(view)
|
|
281
|
+
logger.debug(
|
|
282
|
+
"[insert_frame] table=%s rows=%d ms=%.1f", table, n_rows,
|
|
283
|
+
(time.perf_counter() - t0) * 1000.0,
|
|
284
|
+
)
|
|
285
|
+
return n_rows
|
|
286
|
+
|
|
287
|
+
def bulk_insert(self, table: str, columns, rows, conflict_cols=None) -> None:
|
|
288
|
+
"""Insert many rows with a single vectorized ``INSERT ... SELECT``.
|
|
289
|
+
|
|
290
|
+
DuckDB's ``executemany`` re-runs the prepared single-row statement once
|
|
291
|
+
per row; against a PRIMARY-KEY / indexed table that is pathologically
|
|
292
|
+
slow (per-row index probe + maintenance), e.g. ~497s for 8k rows into
|
|
293
|
+
``_record``. This builds the rows into one DataFrame and inserts it
|
|
294
|
+
through :meth:`insert_frame` (sub-second for the same data).
|
|
295
|
+
|
|
296
|
+
``rows`` is an iterable of value tuples in ``columns`` order. When
|
|
297
|
+
``conflict_cols`` is given, appends ``ON CONFLICT (...) DO NOTHING`` so
|
|
298
|
+
the insert stays idempotent. Safe to call inside an open transaction.
|
|
299
|
+
"""
|
|
300
|
+
rows = list(rows)
|
|
301
|
+
if not rows:
|
|
302
|
+
return
|
|
303
|
+
self.insert_frame(
|
|
304
|
+
table, pd.DataFrame(rows, columns=list(columns)), conflict=conflict_cols
|
|
305
|
+
)
|
|
283
306
|
|
|
284
307
|
def bulk_update(self, table: str, key_cols, set_cols, rows) -> int:
|
|
285
308
|
"""Update many rows with a single vectorized ``UPDATE ... FROM``.
|
|
@@ -675,10 +698,6 @@ class DuckConnection:
|
|
|
675
698
|
# Direct query access
|
|
676
699
|
# ------------------------------------------------------------------
|
|
677
700
|
|
|
678
|
-
def query(self, sql: str, params=None) -> pd.DataFrame:
|
|
679
|
-
"""Execute arbitrary SQL and return a DataFrame."""
|
|
680
|
-
return self.fetchdf(sql, params)
|
|
681
|
-
|
|
682
701
|
# ------------------------------------------------------------------
|
|
683
702
|
# Context manager / cleanup
|
|
684
703
|
# ------------------------------------------------------------------
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|