sciduckdb 0.2.0__tar.gz → 0.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: sciduckdb
3
- Version: 0.2.0
3
+ Version: 0.2.2
4
4
  Summary: A thin DuckDB layer for managing versioned scientific data
5
5
  Author: SciStack Contributors
6
6
  License-Expression: MIT
@@ -21,7 +21,7 @@ Requires-Python: >=3.10
21
21
  Requires-Dist: duckdb>=0.9.0
22
22
  Requires-Dist: numpy>=1.20
23
23
  Requires-Dist: pandas>=1.3
24
- Requires-Dist: scicanonicalhash==0.2.0
24
+ Requires-Dist: scicanonicalhash==0.2.2
25
25
  Provides-Extra: dev
26
26
  Requires-Dist: mypy>=1.0; extra == 'dev'
27
27
  Requires-Dist: pytest-cov>=4.0; extra == 'dev'
@@ -234,52 +234,75 @@ class DuckConnection:
234
234
  self._recover_from_autocommit_failure()
235
235
  raise
236
236
 
237
- def bulk_insert(self, table: str, columns, rows, conflict_cols=None) -> None:
238
- """Insert many rows with a single vectorized ``INSERT ... SELECT``.
239
-
240
- DuckDB's ``executemany`` re-runs the prepared single-row statement once
241
- per row; against a PRIMARY-KEY / indexed table that is pathologically
242
- slow (per-row index probe + maintenance), e.g. ~497s for 8k rows into
243
- ``_record``. This registers the rows as one DataFrame and issues a single
244
- bulk insert instead (sub-second for the same data).
245
-
246
- ``rows`` is an iterable of value tuples in ``columns`` order. When
247
- ``conflict_cols`` is given, appends ``ON CONFLICT (...) DO NOTHING`` so
248
- the insert stays idempotent. Safe to call inside an open transaction
249
- (the register/insert/unregister run on the same connection).
237
+ def insert_frame(self, table: str, frame, *, conflict=None) -> int:
238
+ """Insert every row of *frame* (a pandas DataFrame or a pyarrow Table)
239
+ into *table* with one ``INSERT ... SELECT``, under the lock.
240
+
241
+ THE one way to insert a frame: register, insert, unregister on this
242
+ connection, with the same contention log and autocommit-failure
243
+ recovery as :meth:`execute`. Before 2026-11-02 scidb's bulk Arrow
244
+ inserts (``RecordStore``) reached past the connection to the raw
245
+ ``duckdb`` handle and skipped both (design review F14). Columns are
246
+ the frame's, by name. ``conflict`` (column names) appends
247
+ ``ON CONFLICT (...) DO NOTHING``. Safe inside an open transaction.
248
+ Returns the number of rows handed in.
250
249
  """
251
- rows = list(rows)
252
- if not rows:
253
- return
254
- columns = list(columns)
255
- df = pd.DataFrame(rows, columns=columns)
250
+ columns = list(getattr(frame, "column_names", None) or frame.columns)
251
+ n_rows = frame.num_rows if hasattr(frame, "num_rows") else len(frame)
252
+ if n_rows == 0:
253
+ return 0
254
+ view = "_insert_frame"
256
255
  col_str = ", ".join(f'"{c}"' for c in columns)
257
- sql = f'INSERT INTO "{table}" ({col_str}) SELECT * FROM _bulk_insert_df'
258
- if conflict_cols:
259
- conflict_str = ", ".join(f'"{c}"' for c in conflict_cols)
256
+ sql = f'INSERT INTO "{table}" ({col_str}) SELECT {col_str} FROM {view}'
257
+ if conflict:
258
+ conflict_str = ", ".join(f'"{c}"' for c in conflict)
260
259
  sql += f" ON CONFLICT ({conflict_str}) DO NOTHING"
261
260
  thread = threading.get_ident()
262
261
  wait_start = time.monotonic()
263
262
  with self._lock:
264
263
  waited = time.monotonic() - wait_start
265
264
  foreign_tx = self._tx_owner is not None and self._tx_owner != thread
266
- logger.debug(
267
- "bulk_insert thread=%d waited=%.4fs tx_owner=%s foreign_tx=%s "
268
- "table=%s rows=%d",
269
- thread, waited, self._tx_owner, foreign_tx, table, len(rows),
270
- )
271
- self.con.register("_bulk_insert_df", df)
265
+ self._log_contended("insert_frame", thread, waited, foreign_tx, sql,
266
+ detail=f"table={table} rows={n_rows} ")
267
+ t0 = time.perf_counter()
268
+ self.con.register(view, frame)
272
269
  try:
273
270
  self.con.execute(sql)
274
271
  except Exception:
275
272
  logger.exception(
276
- "bulk_insert FAILED thread=%d tx_owner=%s foreign_tx=%s table=%s",
277
- thread, self._tx_owner, foreign_tx, table,
273
+ "insert_frame FAILED thread=%d tx_owner=%s foreign_tx=%s "
274
+ "table=%s rows=%d",
275
+ thread, self._tx_owner, foreign_tx, table, n_rows,
278
276
  )
279
277
  self._recover_from_autocommit_failure()
280
278
  raise
281
279
  finally:
282
- self._unregister_frame("_bulk_insert_df")
280
+ self._unregister_frame(view)
281
+ logger.debug(
282
+ "[insert_frame] table=%s rows=%d ms=%.1f", table, n_rows,
283
+ (time.perf_counter() - t0) * 1000.0,
284
+ )
285
+ return n_rows
286
+
287
+ def bulk_insert(self, table: str, columns, rows, conflict_cols=None) -> None:
288
+ """Insert many rows with a single vectorized ``INSERT ... SELECT``.
289
+
290
+ DuckDB's ``executemany`` re-runs the prepared single-row statement once
291
+ per row; against a PRIMARY-KEY / indexed table that is pathologically
292
+ slow (per-row index probe + maintenance), e.g. ~497s for 8k rows into
293
+ ``_record``. This builds the rows into one DataFrame and inserts it
294
+ through :meth:`insert_frame` (sub-second for the same data).
295
+
296
+ ``rows`` is an iterable of value tuples in ``columns`` order. When
297
+ ``conflict_cols`` is given, appends ``ON CONFLICT (...) DO NOTHING`` so
298
+ the insert stays idempotent. Safe to call inside an open transaction.
299
+ """
300
+ rows = list(rows)
301
+ if not rows:
302
+ return
303
+ self.insert_frame(
304
+ table, pd.DataFrame(rows, columns=list(columns)), conflict=conflict_cols
305
+ )
283
306
 
284
307
  def bulk_update(self, table: str, key_cols, set_cols, rows) -> int:
285
308
  """Update many rows with a single vectorized ``UPDATE ... FROM``.
@@ -675,10 +698,6 @@ class DuckConnection:
675
698
  # Direct query access
676
699
  # ------------------------------------------------------------------
677
700
 
678
- def query(self, sql: str, params=None) -> pd.DataFrame:
679
- """Execute arbitrary SQL and return a DataFrame."""
680
- return self.fetchdf(sql, params)
681
-
682
701
  # ------------------------------------------------------------------
683
702
  # Context manager / cleanup
684
703
  # ------------------------------------------------------------------
File without changes
File without changes
File without changes