arrowbricks 5.2.1__tar.gz → 5.2.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/PKG-INFO +1 -1
  2. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/pyproject.toml +1 -1
  3. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.lock +1 -1
  4. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/Cargo.toml +1 -1
  5. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/error.rs +9 -0
  6. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/thrift_rpc.rs +26 -3
  7. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client.rs +11 -0
  8. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/thrift_exec.rs +68 -47
  9. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_thrift.rs +101 -1
  10. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/LICENSE +0 -0
  11. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/README.md +0 -0
  12. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/.gitignore +0 -0
  13. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/README.md +0 -0
  14. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/duckdb_query.py +0 -0
  15. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/examples/fastapi_sse.py +0 -0
  16. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/proptest-regressions/json_convert.txt +0 -0
  17. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/rustfmt.toml +0 -0
  18. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/alloc.rs +0 -0
  19. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/arrow_ffi.rs +0 -0
  20. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/call_context.rs +0 -0
  21. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/download.rs +0 -0
  22. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/model.rs +0 -0
  23. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/sea.rs +0 -0
  24. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/client/volume.rs +0 -0
  25. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/heartbeat.rs +0 -0
  26. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/json_convert.rs +0 -0
  27. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/lib.rs +0 -0
  28. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/ndjson.rs +0 -0
  29. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/reorder.rs +0 -0
  30. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/sea.rs +0 -0
  31. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/stats.rs +0 -0
  32. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline/test_support.rs +0 -0
  33. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/pipeline.rs +0 -0
  34. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/src/thrift.rs +0 -0
  35. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/common/mod.rs +0 -0
  36. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_pipeline.rs +0 -0
  37. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests/wiremock_volume_files.rs +0 -0
  38. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/conftest.py +0 -0
  39. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_ipc_stream.py +0 -0
  40. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_parameters.py +0 -0
  41. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_stream_ndjson_lines.py +0 -0
  42. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_streaming.py +0 -0
  43. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_thrift.py +0 -0
  44. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_token_provider.py +0 -0
  45. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/test_volume_files.py +0 -0
  46. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/rust/arrowbricks_core/tests_py/thrift_mock.py +0 -0
  47. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/__init__.py +0 -0
  48. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/_core.pyi +0 -0
  49. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/_streaming.py +0 -0
  50. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/client.py +0 -0
  51. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/cursor.py +0 -0
  52. {arrowbricks-5.2.1 → arrowbricks-5.2.2}/src/arrowbricks/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: arrowbricks
3
- Version: 5.2.1
3
+ Version: 5.2.2
4
4
  Requires-Dist: arro3-core>=0.8 ; extra == 'arro3'
5
5
  Provides-Extra: arro3
6
6
  License-File: LICENSE
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "arrowbricks"
3
- version = "5.2.1"
3
+ version = "5.2.2"
4
4
  description = "Runs SQL against a Databricks SQL warehouse via the Statement Execution API and hands you the result as Arrow -- a DB-API-ish Cursor (fetchone/fetchmany/fetchall/fetchall_arrow) or NDJSON streaming. Rust/PyO3 core throughout -- zero required runtime dependencies."
5
5
  readme = "README.md"
6
6
  license = "MIT"
@@ -177,7 +177,7 @@ dependencies = [
177
177
 
178
178
  [[package]]
179
179
  name = "arrowbricks_core"
180
- version = "5.2.1"
180
+ version = "5.2.2"
181
181
  dependencies = [
182
182
  "arrow-array",
183
183
  "arrow-buffer",
@@ -1,6 +1,6 @@
1
1
  [package]
2
2
  name = "arrowbricks_core"
3
- version = "5.2.1"
3
+ version = "5.2.2"
4
4
  edition = "2024"
5
5
  readme = "README.md"
6
6
 
@@ -130,6 +130,15 @@ impl ApiError {
130
130
  }
131
131
  }
132
132
 
133
+ /// The server rejected a request because the session it names no longer
134
+ /// exists (closed, or invalidated by a warehouse stop/restart). Observed
135
+ /// against a real warehouse: `HTTP 400` with a Thrift exception body
136
+ /// `INVALID_STATE: Invalid SessionHandle: SessionHandle [..]`. Nothing ran,
137
+ /// so the request is safe to repeat on a fresh session.
138
+ pub(crate) fn is_stale_session(&self) -> bool {
139
+ self.message.contains("Invalid SessionHandle")
140
+ }
141
+
133
142
  /// `idempotent` only gates the 5xx case -- 401/403/408/429 mean the
134
143
  /// request was rejected before any processing started (auth failure,
135
144
  /// rate limit, client-side timeout), safe to retry regardless of method.
@@ -106,20 +106,23 @@ impl DbClient {
106
106
  /// `thrift_session_pool`'s doc comment): `None` means the caller must
107
107
  /// open its own throwaway session for this one call (Thrift has no
108
108
  /// session-less submission mode to fall back to).
109
+ ///
110
+ /// The `bool` is `true` for a session taken from the idle pool (it may have
111
+ /// gone stale since it was last used) and `false` for one opened just now.
109
112
  pub(crate) async fn thrift_checkout_session(
110
113
  &self,
111
114
  catalog: Option<&str>,
112
115
  schema: Option<&str>,
113
- ) -> Option<thrift::SessionHandle> {
116
+ ) -> Option<(thrift::SessionHandle, bool)> {
114
117
  let key = (catalog.map(str::to_string), schema.map(str::to_string));
115
118
  if let Some(handle) = self.thrift_session_pool.take(&key) {
116
- return Some(handle);
119
+ return Some((handle, true));
117
120
  }
118
121
  let reservation = self.thrift_session_pool.reserve_guard(&key)?;
119
122
  // Dropped (releasing the slot) on error or if this future is cancelled.
120
123
  let handle = self.thrift_open_session_raw(catalog, schema).await.ok()?;
121
124
  reservation.defuse();
122
- Some(handle)
125
+ Some((handle, false))
123
126
  }
124
127
 
125
128
  pub(crate) fn thrift_checkin_session(
@@ -133,6 +136,26 @@ impl DbClient {
133
136
  self.thrift_session_pool.checkin(key, session, keep);
134
137
  }
135
138
 
139
+ /// Forgets (and closes, best effort) every idle session for this key --
140
+ /// called when one of them turned out to be stale.
141
+ pub(crate) fn thrift_discard_idle_sessions(
142
+ self: &std::sync::Arc<Self>,
143
+ catalog: Option<&str>,
144
+ schema: Option<&str>,
145
+ ) {
146
+ let key = (catalog.map(str::to_string), schema.map(str::to_string));
147
+ let stale = self.thrift_session_pool.drain_key(&key);
148
+ if stale.is_empty() {
149
+ return;
150
+ }
151
+ let client = self.clone();
152
+ pyo3_async_runtimes::tokio::get_runtime().spawn(super::in_call_context(async move {
153
+ for session in stale {
154
+ client.thrift_close_session_raw(&session).await;
155
+ }
156
+ }));
157
+ }
158
+
136
159
  /// Best-effort close of every currently-idle pooled Thrift session --
137
160
  /// same contract as `close_all_sessions` (SEA).
138
161
  pub async fn close_all_thrift_sessions(&self) {
@@ -400,6 +400,17 @@ impl<T> Pool<T> {
400
400
  items
401
401
  }
402
402
 
403
+ /// Takes the idle items of one key (and frees their reservations) -- used
404
+ /// when one of them proved stale, since the rest of that key's idle
405
+ /// sessions went stale with it (a warehouse restart invalidates them all).
406
+ fn drain_key(&self, key: &SessionKey) -> Vec<T> {
407
+ let items = self.idle.lock().unwrap().remove(key).unwrap_or_default();
408
+ if let Some(count) = self.total.lock().unwrap().get_mut(key) {
409
+ *count = count.saturating_sub(items.len());
410
+ }
411
+ items
412
+ }
413
+
403
414
  /// `reserve` as an RAII guard: the slot is released again if the guard
404
415
  /// is dropped without `defuse()` -- i.e. when the caller's future is
405
416
  /// cancelled (timeout, `asyncio.wait_for`) while it is still creating
@@ -375,58 +375,79 @@ pub(crate) async fn submit_thrift_and_start_fetch(
375
375
  stats.add_warehouse_wait_s(warehouse_t0.elapsed().as_secs_f64());
376
376
 
377
377
  let submit_t0 = Instant::now();
378
- let pooled = client.thrift_checkout_session(catalog, schema).await;
379
- let (session, from_pool) = match pooled {
380
- Some(s) => (s, true),
381
- None => match client.thrift_open_session_raw(catalog, schema).await {
382
- Ok(s) => (s, false),
383
- Err(e) => {
384
- report_submit_error(&client, "thrift", submit_t0.elapsed().as_secs_f64(), &stats);
385
- return Err(e);
378
+ // A session taken from the idle pool may have gone stale since its last
379
+ // use (the warehouse auto-stopped and restarted, say). The server then
380
+ // rejects the statement before running it, so it is safe to retry once on
381
+ // a freshly opened session. See `ApiError::is_stale_session`.
382
+ let mut retried = false;
383
+ let (session, from_pool, ready) = loop {
384
+ // On the retry the idle pool for this key was just emptied, so this
385
+ // opens (and pools) a fresh session rather than reusing a stale one.
386
+ let pooled = client.thrift_checkout_session(catalog, schema).await;
387
+ let (session, from_pool, reused) = match pooled {
388
+ Some((s, reused)) => (s, true, reused),
389
+ None => match client.thrift_open_session_raw(catalog, schema).await {
390
+ Ok(s) => (s, false, false),
391
+ Err(e) => {
392
+ report_submit_error(&client, "thrift", submit_t0.elapsed().as_secs_f64(), &stats);
393
+ return Err(e);
394
+ }
395
+ },
396
+ };
397
+
398
+ let mut session_guard = ThriftSessionOnDrop {
399
+ client: &client,
400
+ catalog,
401
+ schema,
402
+ session: Some(session.clone()),
403
+ from_pool,
404
+ };
405
+ let ready = submit_and_await_thrift_statement(&client, &session, statement, parameters.as_ref(), &stats).await;
406
+ session_guard.session = None;
407
+ drop(session_guard);
408
+
409
+ // Exactly one of these arms ever touches `session` -- a pooled session
410
+ // is checked in right away (safe: that only makes it available for a
411
+ // *different* operation, see `execute_lazy_thrift`'s own doc comment);
412
+ // a throwaway session that failed to even reach a terminal state is
413
+ // closed right here, since it'll never be handed to
414
+ // `drive_thrift_fetch_loop`; a throwaway session that succeeded is
415
+ // carried forward for that function to close once the fetch loop is
416
+ // actually done with it.
417
+ if from_pool {
418
+ let keep = ready.is_ok();
419
+ client.thrift_checkin_session(catalog, schema, session.clone(), keep);
420
+ if !keep {
421
+ // A discarded session is only forgotten by the pool; close it
422
+ // server-side too instead of leaving it to the idle TTL.
423
+ let client = client.clone();
424
+ let discarded = session.clone();
425
+ pyo3_async_runtimes::tokio::get_runtime().spawn(in_call_context(async move {
426
+ client.thrift_close_session_raw(&discarded).await;
427
+ }));
386
428
  }
387
- },
388
- };
429
+ } else if ready.is_err() {
430
+ client.thrift_close_session_raw(&session).await;
431
+ }
389
432
 
390
- let mut session_guard = ThriftSessionOnDrop {
391
- client: &client,
392
- catalog,
393
- schema,
394
- session: Some(session.clone()),
395
- from_pool,
433
+ if let Err(e) = &ready
434
+ && reused
435
+ && !retried
436
+ && e.is_stale_session()
437
+ {
438
+ // Its idle siblings went stale with it.
439
+ client.thrift_discard_idle_sessions(catalog, schema);
440
+ stats.retry_count.fetch_add(1, Ordering::Relaxed);
441
+ retried = true;
442
+ continue;
443
+ }
444
+ break (session, from_pool, ready);
396
445
  };
397
- let ready = submit_and_await_thrift_statement(&client, &session, statement, parameters.as_ref(), &stats).await;
398
- session_guard.session = None;
399
- drop(session_guard);
400
446
  let submit_to_ready_s = submit_t0.elapsed().as_secs_f64();
401
-
402
- // Exactly one of these two arms ever touches `session` -- a pooled
403
- // session is checked in right away (safe: that only makes it available
404
- // for a *different* operation, see `execute_lazy_thrift`'s own doc
405
- // comment); a throwaway session that failed to even reach a terminal
406
- // state is closed right here, since it'll never be handed to
407
- // `drive_thrift_fetch_loop`; a throwaway session that succeeded is
408
- // carried forward for that function to close once the fetch loop is
409
- // actually done with it.
410
- let throwaway_session = if from_pool {
411
- let keep = ready.is_ok();
412
- client.thrift_checkin_session(catalog, schema, session.clone(), keep);
413
- if !keep {
414
- // A discarded session is only forgotten by the pool; close it
415
- // server-side too instead of leaving it to the idle TTL.
416
- let client = client.clone();
417
- pyo3_async_runtimes::tokio::get_runtime().spawn(in_call_context(async move {
418
- client.thrift_close_session_raw(&session).await;
419
- }));
420
- }
421
- None
447
+ let throwaway_session = if !from_pool && ready.is_ok() {
448
+ Some(session)
422
449
  } else {
423
- match &ready {
424
- Ok(_) => Some(session),
425
- Err(_) => {
426
- client.thrift_close_session_raw(&session).await;
427
- None
428
- }
429
- }
450
+ None
430
451
  };
431
452
  let ready = match ready {
432
453
  Ok(r) => r,
@@ -2340,7 +2340,7 @@ async fn thrift_failed_statement_closes_its_discarded_pooled_session() {
2340
2340
  );
2341
2341
  }
2342
2342
 
2343
- // ============================================================================
2343
+ // =====================================================================
2344
2344
  // Call context (5.2.1) -- see the matching section of `wiremock_pipeline.rs`.
2345
2345
  // Here the token requests come from the heartbeat-wrapped submit and the
2346
2346
  // Thrift fetch loop (`FetchResults`, `CloseOperation`), all spawned tasks.
@@ -2426,3 +2426,103 @@ async fn thrift_streamed_query_requests_every_token_in_the_callers_call_context(
2426
2426
  // + one FetchResults per batch
2427
2427
  token.assert_all_in_context(4 + N_BATCHES as usize);
2428
2428
  }
2429
+
2430
+ // ---- issue #14: a stale pooled session is replaced, once ----
2431
+
2432
+ /// Mounts counting `OpenSession`/`ExecuteStatement` mocks. `ExecuteStatement`
2433
+ /// call number `fail_on` (0-based) answers `HTTP 400` with `fail_body`, the
2434
+ /// shape a real warehouse returns for a session it no longer knows
2435
+ /// (`INVALID_STATE: Invalid SessionHandle: ...`, reproduced live).
2436
+ async fn mount_counted_submit(
2437
+ server: &MockServer,
2438
+ fail_on: usize,
2439
+ fail_body: &'static str,
2440
+ ) -> (Arc<AtomicUsize>, Arc<AtomicUsize>) {
2441
+ let opens = Arc::new(AtomicUsize::new(0));
2442
+ let opens_for_mock = opens.clone();
2443
+ Mock::given(method("POST"))
2444
+ .and(path(thrift_path()))
2445
+ .and(IsThriftRpc("OpenSession"))
2446
+ .respond_with(move |_: &Request| {
2447
+ let n = opens_for_mock.fetch_add(1, Ordering::SeqCst);
2448
+ ResponseTemplate::new(200).set_body_raw(
2449
+ build_open_session_resp(format!("s{n}").as_bytes(), b"sec"),
2450
+ "application/x-thrift",
2451
+ )
2452
+ })
2453
+ .mount(server)
2454
+ .await;
2455
+ let schema = test_schema();
2456
+ let executes = Arc::new(AtomicUsize::new(0));
2457
+ let executes_for_mock = executes.clone();
2458
+ Mock::given(method("POST"))
2459
+ .and(path(thrift_path()))
2460
+ .and(IsThriftRpc("ExecuteStatement"))
2461
+ .respond_with(move |_: &Request| {
2462
+ let n = executes_for_mock.fetch_add(1, Ordering::SeqCst);
2463
+ if n == fail_on {
2464
+ ResponseTemplate::new(400).set_body_string(fail_body)
2465
+ } else {
2466
+ ResponseTemplate::new(200).set_body_raw(
2467
+ small_execute_statement_success(format!("op-{n}").into_bytes(), &schema),
2468
+ "application/x-thrift",
2469
+ )
2470
+ }
2471
+ })
2472
+ .mount(server)
2473
+ .await;
2474
+ mount_close_operation_ok(server).await;
2475
+ mount_close_session_counter(server).await;
2476
+ (opens, executes)
2477
+ }
2478
+
2479
+ #[tokio::test]
2480
+ async fn thrift_retries_once_on_a_fresh_session_when_a_pooled_one_went_stale() {
2481
+ let server = MockServer::start().await;
2482
+ mount_warehouse_running(&server).await;
2483
+ let (opens, executes) = mount_counted_submit(
2484
+ &server,
2485
+ 1,
2486
+ "INVALID_STATE: Invalid SessionHandle: SessionHandle [01f1c49c-2f22-1b64-af6b-5d6be1eb8de4].",
2487
+ )
2488
+ .await;
2489
+ let client = thrift_client(&server);
2490
+
2491
+ let mut first = execute_lazy_thrift(client.clone(), "SELECT 1", None, None, None)
2492
+ .await
2493
+ .unwrap();
2494
+ first.fetchall_arrow().await.unwrap();
2495
+ assert_eq!(opens.load(Ordering::SeqCst), 1, "first query opens the pooled session");
2496
+
2497
+ // Second query reuses that session; the server says it is gone.
2498
+ let mut second = execute_lazy_thrift(client.clone(), "SELECT 1", None, None, None)
2499
+ .await
2500
+ .expect("a stale pooled session must be replaced, not surfaced");
2501
+ second.fetchall_arrow().await.unwrap();
2502
+ assert_eq!(
2503
+ opens.load(Ordering::SeqCst),
2504
+ 2,
2505
+ "exactly one fresh session for the retry"
2506
+ );
2507
+ assert_eq!(executes.load(Ordering::SeqCst), 3, "one failed attempt, one retry");
2508
+ }
2509
+
2510
+ #[tokio::test]
2511
+ async fn thrift_does_not_retry_a_rejected_statement_that_is_not_a_stale_session() {
2512
+ let server = MockServer::start().await;
2513
+ mount_warehouse_running(&server).await;
2514
+ let (opens, executes) = mount_counted_submit(&server, 1, "PARSE_SYNTAX_ERROR: bad sql").await;
2515
+ let client = thrift_client(&server);
2516
+
2517
+ let mut first = execute_lazy_thrift(client.clone(), "SELECT 1", None, None, None)
2518
+ .await
2519
+ .unwrap();
2520
+ first.fetchall_arrow().await.unwrap();
2521
+ assert!(
2522
+ execute_lazy_thrift(client.clone(), "bad", None, None, None)
2523
+ .await
2524
+ .is_err()
2525
+ );
2526
+ assert_eq!(executes.load(Ordering::SeqCst), 2, "an ordinary error is not retried");
2527
+ assert_eq!(opens.load(Ordering::SeqCst), 1);
2528
+ }
File without changes
File without changes