chalk-remote-call-python 1.9.1__tar.gz → 1.9.2__tar.gz

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.9.1
+Version: 1.9.2
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai
@@ -108,6 +108,16 @@ python -m chalk_remote_call --handler my_handler.handler
 | `--on-shutdown` | | | Dotted path to a shutdown function |
 | `--log-level` | | `INFO` | `DEBUG`, `INFO`, `WARNING`, or `ERROR` |
 
+The CLI writes Python logs to stderr as one JSON object per line, with separate
+`timestamp`, `severity`, `logger`, and `message` fields. Exceptions include an
+`exception.stacktrace` field. Calls to `serve()` from Python retain the embedding
+application's logging configuration.
+
+Missing optional StatsD endpoints or a metrics-bus topic do not emit warnings.
+Failures initializing configured metrics sinks still warn. Routine customer
+StatsD configuration messages are emitted at DEBUG by the shared metrics library;
+set `RUST_LOG=info,chalk_metrics::metrics::config=debug` to include them.
+
 ### Environment variables
 
 | Variable | Description |
@@ -212,3 +222,23 @@ To check the Rust code independently:
 ```bash
 PYO3_PYTHON=python3 cargo check --manifest-path chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml
 ```
+
+## Remote exception transport
+
+Handler failures retain their original exception type, message, traceback,
+exception chains, and exception groups. Runtime dispatch frames are excluded
+from the user traceback; full diagnostics are retained in server logs.
+
+All call modes carry a JSON payload with `chalk_remote_error: 1` and an
+`exception` object containing the `ChalkException` fields `kind`, `message`,
+`stacktrace`, and `internal_stacktrace`. Direct calls carry the payload as the
+unescaped gRPC status message; queued calls carry it in the error message. The
+local async service additionally populates `ChalkError.exception`. This permits
+existing queue transports to forward structured errors without a protobuf
+migration. Clients must decode the versioned payload, not parse gRPC debug text
+or classify user errors by their message. Unknown versions remain opaque.
+
+Payloads are bounded for gRPC header limits. Truncation is marked explicitly,
+with complete tracebacks kept in server logs. Exception objects and locals are
+never pickled or sent. Chalk SDK-generated handlers identify their dispatch
+frames, and notebook source maps identify the deployed snapshot's lines.

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/README.md

@@ -70,6 +70,16 @@ python -m chalk_remote_call --handler my_handler.handler
 | `--on-shutdown` | | | Dotted path to a shutdown function |
 | `--log-level` | | `INFO` | `DEBUG`, `INFO`, `WARNING`, or `ERROR` |
 
+The CLI writes Python logs to stderr as one JSON object per line, with separate
+`timestamp`, `severity`, `logger`, and `message` fields. Exceptions include an
+`exception.stacktrace` field. Calls to `serve()` from Python retain the embedding
+application's logging configuration.
+
+Missing optional StatsD endpoints or a metrics-bus topic do not emit warnings.
+Failures initializing configured metrics sinks still warn. Routine customer
+StatsD configuration messages are emitted at DEBUG by the shared metrics library;
+set `RUST_LOG=info,chalk_metrics::metrics::config=debug` to include them.
+
 ### Environment variables
 
 | Variable | Description |
@@ -174,3 +184,23 @@ To check the Rust code independently:
 ```bash
 PYO3_PYTHON=python3 cargo check --manifest-path chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml
 ```
+
+## Remote exception transport
+
+Handler failures retain their original exception type, message, traceback,
+exception chains, and exception groups. Runtime dispatch frames are excluded
+from the user traceback; full diagnostics are retained in server logs.
+
+All call modes carry a JSON payload with `chalk_remote_error: 1` and an
+`exception` object containing the `ChalkException` fields `kind`, `message`,
+`stacktrace`, and `internal_stacktrace`. Direct calls carry the payload as the
+unescaped gRPC status message; queued calls carry it in the error message. The
+local async service additionally populates `ChalkError.exception`. This permits
+existing queue transports to forward structured errors without a protobuf
+migration. Clients must decode the versioned payload, not parse gRPC debug text
+or classify user errors by their message. Unknown versions remain opaque.
+
+Payloads are bounded for gRPC header limits. Truncation is marked explicitly,
+with complete tracebacks kept in server logs. Exception objects and locals are
+never pickled or sent. Chalk SDK-generated handlers identify their dispatch
+frames, and notebook source maps identify the deployed snapshot's lines.

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/Cargo.lock

@@ -484,7 +484,7 @@ dependencies = [
 [[package]]
 name = "chalk-core"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "anyhow",
  "envconfig",
@@ -499,7 +499,7 @@ dependencies = [
 [[package]]
 name = "chalk-protos"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "prost 0.11.9",
  "prost-build",
@@ -547,6 +547,7 @@ dependencies = [
  "prost 0.13.5",
  "pyo3",
  "redis",
+ "serde_json",
  "tokio",
  "tokio-stream",
  "tonic 0.12.3",
@@ -560,7 +561,7 @@ dependencies = [
 [[package]]
 name = "chalk_metric_definitions"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "chalk-protos",
  "metrics-generator",
@@ -570,7 +571,7 @@ dependencies = [
 [[package]]
 name = "chalk_metrics"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "async-trait",
  "chalk-core",
@@ -604,7 +605,7 @@ dependencies = [
 [[package]]
 name = "chalk_statsd"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "chalk-core",
  "chrono",
@@ -1867,7 +1868,7 @@ dependencies = [
 [[package]]
 name = "metrics-generator"
 version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=783e9ce6b2c#783e9ce6b2c17563ab3302e5d11f8a664ed00232"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=a0f8932ec10d3b8340af3e98cf63b800d60b4926#a0f8932ec10d3b8340af3e98cf63b800d60b4926"
 dependencies = [
  "codegen",
  "serde",

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml

@@ -21,6 +21,7 @@ arrow = { version = "54", default-features = false, features = ["ipc"] }
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 bytes = "1"
+serde_json = "1"
 uuid = { version = "1", features = ["v4"] }
 dashmap = "5"
 futures-util = "0.3"
@@ -34,8 +35,8 @@ futures-util = "0.3"
 # chalk-private commit, mirroring how chalkcompute_remote_client depends on
 # chalk_remote_call_client. chalk-queue declares `redis = "1.0.2"`; with no
 # `[patch.crates-io]` here it builds against vanilla crates.io redis (it only
-# uses standard APIs). Same rev across all three so cargo reuses one checkout.
+# uses standard APIs). Metrics and StatsD share a revision so their client types agree.
 [target.'cfg(target_os = "linux")'.dependencies]
 chalk-queue = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "9c61f6a938d8f885f138217ac64149e29d7a691e" }
-chalk_metrics = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "783e9ce6b2c" }
-chalk_statsd = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "783e9ce6b2c" }
+chalk_metrics = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "a0f8932ec10d3b8340af3e98cf63b800d60b4926" }
+chalk_statsd = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "a0f8932ec10d3b8340af3e98cf63b800d60b4926" }

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/async_service.rs

@@ -39,7 +39,7 @@ struct CallState {
     /// and `status`/`error` can be updated without blocking emits.
     chunks: Arc<Mutex<ChunkBuffer>>,
     /// Populated only when `status == Failed`; surfaced as a `ChalkError`.
-    error: Option<String>,
+    error: Option<ChalkError>,
     /// When the call reached a terminal status. Used only by the orphan sweeper
     /// to decide when an un-drained terminal entry is safe to reclaim.
     completed_at: Option<Instant>,
@@ -189,7 +189,7 @@ impl AsyncRemoteCallService for AsyncRemoteCallServiceImpl {
                     }
                     Err(e) => {
                         s.status = RemoteCallStatus::Failed;
-                        s.error = Some(e.to_string());
+                        s.error = Some(e.chalk_error());
                         error!(
                             call_id = %id,
                             exception.stacktrace = e.details(),
@@ -246,10 +246,10 @@ impl AsyncRemoteCallService for AsyncRemoteCallServiceImpl {
         drop(buf);
 
         let errors = if status == RemoteCallStatus::Failed {
-            vec![ChalkError {
-                message: error.unwrap_or_else(|| "remote call failed".to_string()),
+            vec![error.unwrap_or_else(|| ChalkError {
+                message: "remote call failed".to_string(),
                 ..Default::default()
-            }]
+            })]
         } else {
             Vec::new()
         };

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/coalesce.rs

@@ -193,8 +193,7 @@ impl CoalescingQueue {
                         chalk_metrics::metrics::tags::Mode::Sync,
                     );
                 }
-                let status =
-                    Status::unknown(format!("Exception raised during handler execution: {e}"));
+                let status = Status::unknown(e.to_string());
                 for tx in response_txs {
                     let _ = tx.send(Err(status.clone())).await;
                 }

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/metrics.rs

@@ -1,6 +1,6 @@
-//! Metrics emission for the self-consuming shim, mirroring the catalog
-//! consumer / fnq-server: build a `chalk_metrics` publishing pipeline at startup
-//! and emit per-call dequeue + processing-latency metrics.
+//! Function-call metrics for synchronous gRPC calls and self-consumed queue work.
+//! A shared `chalk_metrics` publishing pipeline emits call counts and
+//! processing latency, independently of whether Redis queue consumption is enabled.
 //!
 //! The External Function "Function Calls Dequeued" chart reads TimescaleDB
 //! `metrics1`, which is fed only by the chalk_metrics PubSub bus (statsd goes to
@@ -15,8 +15,9 @@ use chalk_metrics::metrics::well_known_metrics::WellKnownMetricName;
 use chalk_metrics::{
     make_pubsub_publisher, BatchConfig, BusPublisher, MetricsPipeline, PublishingMetricsPipeline,
 };
+use chalk_statsd::datadog::DatadogError;
 use chalk_statsd::metrics::MetricsClient;
-use tracing::{info, warn};
+use tracing::{debug, warn};
 
 // Batch tuning — chalk_function_queue's defaults (METRICS_BATCH_* ), inlined.
 const BATCH_MAX_MESSAGES: usize = 1000;
@@ -24,26 +25,27 @@ const BATCH_MAX_BYTES: usize = 10 * 1024 * 1024; // 10 MB
 const BATCH_MAX_LATENCY: Duration = Duration::from_millis(100);
 
 /// Build and start the metrics pipeline from the environment, mirroring the
-/// fnq-server / catalog-consumer init. Returns `None` only if the pipeline
-/// object can't be constructed; the self-consumer then runs without metrics.
+/// fnq-server / catalog-consumer init. Returns `None` if startup fails.
 ///
 /// The bus publisher is what reaches the metrics store (and thus the dashboard);
-/// statsd alone only reaches Datadog. An empty `METRICS_BUS_TOPIC` therefore
-/// means the dashboard won't move — surfaced as a warning, not an error.
+/// StatsD and the bus are optional sinks. Missing configuration is quiet;
+/// failures initializing a configured sink remain warnings.
 pub async fn init_pipeline() -> Option<Arc<PublishingMetricsPipeline>> {
     let environment_id = std::env::var("CHALK_ENVIRONMENT_ID").unwrap_or_default();
 
     let statsd = Arc::new(
         MetricsClient::new_from_default_environment().unwrap_or_else(|e| {
-            warn!(error = %e, "failed to init statsd client; statsd metrics disabled");
+            if !matches!(
+                e.downcast_ref::<DatadogError>(),
+                Some(DatadogError::ConnectionInfoNotFound)
+            ) {
+                warn!(error = %e, "failed to init statsd client; statsd metrics disabled");
+            }
             MetricsClient::new_null()
         }),
     );
 
     let publishers: Vec<Arc<dyn BusPublisher>> = build_bus_publisher().await.into_iter().collect();
-    if publishers.is_empty() {
-        warn!("no metrics bus publisher configured; function call metrics will not reach the dashboard");
-    }
 
     let metrics_pipeline: Arc<PublishingMetricsPipeline> =
         Arc::new(PublishingMetricsPipeline::new_with_env_var_defaults(
@@ -54,8 +56,9 @@ pub async fn init_pipeline() -> Option<Arc<PublishingMetricsPipeline>> {
         ));
     if let Err(e) = metrics_pipeline.start().await {
         warn!(error = %e, "failed to start metrics pipeline; metrics disabled");
+        return None;
     }
-    info!("fnq self-consumer metrics pipeline started");
+    debug!("function-call metrics pipeline started");
     Some(metrics_pipeline)
 }
 

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/python_bridge.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 use std::sync::{Arc, Mutex};
 
+use chalk_remote_call_proto::chalk::common::v1::{ChalkError, ChalkException};
 use chalk_remote_call_proto::chalk::runtime::v1::CallFunctionResponse;
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyDict, PyList};
@@ -321,28 +322,62 @@ fn invoke_process_fn(
 #[derive(Debug)]
 pub struct PythonError {
     details: String,
+    wire_message: String,
 }
 
 impl PythonError {
     fn new(message: String) -> Self {
-        Self { details: message }
+        Self {
+            details: message.clone(),
+            wire_message: message,
+        }
     }
 
     pub fn details(&self) -> &str {
         &self.details
     }
+
+    pub fn chalk_error(&self) -> ChalkError {
+        let exception = serde_json::from_str::<serde_json::Value>(&self.wire_message)
+            .ok()
+            .and_then(|payload| {
+                let exception = payload.get("exception")?;
+                Some(ChalkException {
+                    kind: exception.get("kind")?.as_str()?.to_string(),
+                    message: exception.get("message")?.as_str()?.to_string(),
+                    stacktrace: exception.get("stacktrace")?.as_str()?.to_string(),
+                    internal_stacktrace: exception
+                        .get("internal_stacktrace")?
+                        .as_str()?
+                        .to_string(),
+                })
+            });
+        ChalkError {
+            message: self.wire_message.clone(),
+            exception,
+            ..Default::default()
+        }
+    }
 }
 
 impl std::fmt::Display for PythonError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.details)
+        write!(f, "{}", self.wire_message)
     }
 }
 
 fn into_python_error(py: Python<'_>, err: PyErr) -> PythonError {
     let summary = err.to_string();
     let details = format_python_traceback(py, &err).unwrap_or_else(|_| summary.clone());
-    PythonError { details }
+    let wire_message = py
+        .import("chalk_remote_call.errors")
+        .and_then(|module| module.call_method1("serialize_exception", (err.value(py),)))
+        .and_then(|value| value.extract::<String>())
+        .unwrap_or_else(|_| details.clone());
+    PythonError {
+        details,
+        wire_message,
+    }
 }
 
 fn format_python_traceback(py: Python<'_>, err: &PyErr) -> PyResult<String> {

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/self_consumer.rs

@@ -231,8 +231,6 @@ pub async fn run(
     let conn = ConnectionManager::new(client).await?;
     let queue = FunctionQueue::new(conn.clone(), config.tenant.clone());
     let input_queue = queue.bind(&config.function_name, config.queue_protocol);
-    let legacy_queue = (config.queue_protocol == QueueProtocol::StreamV1)
-        .then(|| queue.bind(&config.function_name, QueueProtocol::ListV1));
     let sem = queue.semaphore(&config.function_name, config.max_concurrent, config.sem_ttl);
     let sem_key = queue.sem_key_for(&config.function_name);
     let live_calls: LiveCalls = Arc::new(DashMap::new());
@@ -242,9 +240,12 @@ pub async fn run(
         Uuid::new_v4()
     );
 
-    // XGROUP CREATE is a startup operation. Keeping it out of every XREAD and
-    // XAUTOCLAIM avoids two avoidable Redis round trips per dispatch window.
-    input_queue.initialize_consumer_group().await?;
+    if config.queue_protocol == QueueProtocol::StreamV1 {
+        // XGROUP CREATE is a startup operation. Keeping it out of every XREAD
+        // and XAUTOCLAIM avoids two avoidable Redis round trips per dispatch
+        // window.
+        input_queue.initialize_consumer_group().await?;
+    }
 
     spawn_renewer(
         input_queue.clone(),
@@ -265,11 +266,12 @@ pub async fn run(
     );
 
     if config.queue_protocol == QueueProtocol::StreamV1 {
+        let legacy_queue = queue.bind(&config.function_name, QueueProtocol::ListV1);
         return run_stream_dispatch_loop(
             &config,
             queue,
             input_queue,
-            legacy_queue,
+            Some(legacy_queue),
             sem,
             conn,
             consumer_name,
@@ -281,13 +283,42 @@ pub async fn run(
         .await;
     }
 
+    run_list_dispatch_loop(
+        &config,
+        queue,
+        input_queue,
+        sem,
+        conn,
+        consumer_name,
+        live_calls,
+        handler,
+        metrics_pipeline,
+        coalescing_queue,
+    )
+    .await
+}
+
+/// Consume the legacy Redis List queue one call at a time. Streams use their
+/// own batched dispatch loop above so they never fall through this path.
+#[allow(clippy::too_many_arguments)]
+async fn run_list_dispatch_loop(
+    config: &SelfConsumerConfig,
+    queue: FunctionQueue,
+    input_queue: BoundFunctionQueue,
+    sem: chalk_queue::DistributedSemaphore,
+    conn: ConnectionManager,
+    consumer_name: String,
+    live_calls: LiveCalls,
+    handler: Arc<PythonHandler>,
+    metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
+    coalescing_queue: Option<Arc<CoalescingQueue>>,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let mut backoff = config.poll_backoff_initial;
     loop {
         // Idle-gate: if there is no work, don't take (then immediately release)
         // a semaphore slot — that churn would race genuine acquirers on other
-        // pods. During a Stream rollout, also observe the legacy List so a new
-        // shim can drain work queued before its revision metadata changed.
-        match pending_depth(&input_queue, legacy_queue.as_ref()).await {
+        // pods.
+        match input_queue.current_depth().await {
             Ok(0) => {
                 tokio::time::sleep(backoff).await;
                 backoff = (backoff * 2).min(config.poll_backoff_max);
@@ -318,14 +349,7 @@ pub async fn run(
             }
         };
 
-        let delivery = match dequeue_next(
-            &input_queue,
-            legacy_queue.as_ref(),
-            &consumer_name,
-            config.sem_ttl,
-        )
-        .await
-        {
+        let delivery = match input_queue.dequeue(&consumer_name).await {
             Ok(Some(delivery)) => delivery,
             // Raced another consumer (or our own idle-gate) to an empty queue:
             // give the slot back and retry.
@@ -344,41 +368,18 @@ pub async fn run(
 
         backoff = config.poll_backoff_initial;
 
-        // Register the token so the pod-level renewer heartbeats this lease
-        // while the call runs.
-        live_calls.insert(lease.token().to_string(), delivery.receipt.clone());
-
-        match &coalescing_queue {
-            // Batching on: route through the shared coalescing queue so this
-            // call batches with inbound gRPC calls (and other self-consumed
-            // work) under the user's batch-size/duration.
-            Some(cq) => {
-                tokio::spawn(run_one_coalesced_call(
-                    queue.clone(),
-                    input_queue.clone(),
-                    cq.clone(),
-                    conn.clone(),
-                    delivery,
-                    lease,
-                    live_calls.clone(),
-                    metrics_pipeline.clone(),
-                ));
-            }
-            // Batching off: stream the handler's output chunk-by-chunk.
-            None => {
-                tokio::spawn(run_one_call(
-                    handler.clone(),
-                    queue.clone(),
-                    input_queue.clone(),
-                    conn.clone(),
-                    delivery,
-                    lease,
-                    live_calls.clone(),
-                    config.flush_timeout,
-                    metrics_pipeline.clone(),
-                ));
-            }
-        }
+        dispatch_delivery(
+            queue.clone(),
+            input_queue.clone(),
+            coalescing_queue.clone(),
+            handler.clone(),
+            conn.clone(),
+            delivery,
+            lease,
+            live_calls.clone(),
+            config.flush_timeout,
+            metrics_pipeline.clone(),
+        );
         // Loop immediately — dispatch as fast as we can acquire + pop. The
         // semaphore (not this loop) is what bounds in-flight work.
     }
@@ -543,43 +544,6 @@ fn dispatch_delivery(
     }
 }
 
-async fn pending_depth(
-    queue: &BoundFunctionQueue,
-    legacy_queue: Option<&BoundFunctionQueue>,
-) -> Result<u64, redis::RedisError> {
-    let selected = queue.current_depth().await?;
-    let legacy = match legacy_queue {
-        Some(queue) => queue.current_depth().await?,
-        None => 0,
-    };
-    Ok(selected + legacy)
-}
-
-async fn dequeue_next(
-    queue: &BoundFunctionQueue,
-    legacy_queue: Option<&BoundFunctionQueue>,
-    consumer_name: &str,
-    reclaim_idle: Duration,
-) -> Result<Option<DequeuedCall>, Box<dyn std::error::Error + Send + Sync>> {
-    // Recovered work wins first. The legacy List is then drained before new
-    // Stream traffic; producers never route new work to both transports.
-    if let Some(reclaimed) = queue
-        .reclaim_stale(consumer_name, reclaim_idle, 1)
-        .await?
-        .into_iter()
-        .next()
-    {
-        return Ok(Some(reclaimed));
-    }
-    if let Some(legacy_queue) = legacy_queue {
-        if let Some(legacy) = legacy_queue.dequeue(consumer_name).await? {
-            info!(function = %queue.function_name(), "draining legacy List item during Stream migration");
-            return Ok(Some(legacy));
-        }
-    }
-    queue.dequeue(consumer_name).await
-}
-
 /// Run a single queued call to completion, write its results to the call's
 /// stream, then deregister + release its lease. Always releases the lease,
 /// even if execution or stream writes fail or panic.
@@ -937,15 +901,13 @@ async fn execute_and_write(
             info!(call_id = %work.call_id, chunks = flushed, "queued call completed");
         }
         Err(e) => {
-            // The formatted Python traceback is available, but the concrete
-            // exception type is not carried separately across this boundary.
             stream_writer::write_error(
                 conn,
                 &stream_key,
                 seq,
                 "RemoteFunctionError",
-                e.details(),
-                None,
+                &e.to_string(),
+                Some(e.details()),
             )
             .await?;
             let summary: String = e.to_string().chars().take(200).collect();

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/server.rs

@@ -80,8 +80,7 @@ pub async fn run_server(
 
     #[cfg(target_os = "linux")]
     if let Some(cfg) = crate::self_consumer::SelfConsumerConfig::from_env() {
-        // Build the chalk_metrics pipeline once (mirrors the fnq-server/cc
-        // main init) so the self-consumer can emit function-call metrics.
+        // Share the synchronous gRPC server's function-call metrics pipeline.
         let consumer_handler = python_handler.clone();
         let consumer_pipeline = pipeline.clone();
         let consumer_coalescing = coalescing_queue.clone();

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk-remote-call-rs/chalk-remote-call-server/src/service.rs

@@ -144,11 +144,7 @@ impl RemoteCallService for RemoteCallServiceImpl {
                             "Python handler error: {}",
                             e.details()
                         );
-                        let _ = error_tx
-                            .send(Err(Status::unknown(format!(
-                                "Exception raised during handler execution: {e}"
-                            ))))
-                            .await;
+                        let _ = error_tx.send(Err(Status::unknown(e.to_string()))).await;
                     }
                 }
             }

chalk_remote_call_python-1.9.2/chalk_remote_call/_version.py

@@ -0,0 +1 @@
+__version__ = "1.9.2"

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk_remote_call/cli.py

@@ -7,6 +7,7 @@ import sys
 
 from chalk_remote_call.handler_loader import load_function, load_handler
 from chalk_remote_call.input_transform import parse_input_args
+from chalk_remote_call.logging import JsonFormatter
 from chalk_remote_call.server import serve
 
 
@@ -99,16 +100,15 @@ def main() -> None:
     )
     args = parser.parse_args()
 
-    logging.basicConfig(
-        level=getattr(logging, args.log_level),
-        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
-    )
+    log_handler = logging.StreamHandler(sys.stderr)
+    log_handler.setFormatter(JsonFormatter())
+    logging.basicConfig(level=getattr(logging, args.log_level), handlers=[log_handler])
 
     # Load handler
     try:
         handler, auto_startup, auto_shutdown = load_handler(args.handler)
-    except Exception as e:
-        print(f"Error loading handler: {e}", file=sys.stderr)
+    except Exception:
+        logging.getLogger(__name__).exception("Error loading handler")
         sys.exit(1)
 
     # Determine startup function
@@ -116,8 +116,8 @@ def main() -> None:
     if args.on_startup:
         try:
             on_startup = load_function(args.on_startup, label="Startup")
-        except Exception as e:
-            print(f"Error loading startup function: {e}", file=sys.stderr)
+        except Exception:
+            logging.getLogger(__name__).exception("Error loading startup function")
             sys.exit(1)
     elif auto_startup is not None:
         on_startup = auto_startup
@@ -127,8 +127,8 @@ def main() -> None:
     if args.on_shutdown:
         try:
             on_shutdown = load_function(args.on_shutdown, label="Shutdown")
-        except Exception as e:
-            print(f"Error loading shutdown function: {e}", file=sys.stderr)
+        except Exception:
+            logging.getLogger(__name__).exception("Error loading shutdown function")
             sys.exit(1)
     elif auto_shutdown is not None:
         on_shutdown = auto_shutdown

chalk_remote_call_python-1.9.2/chalk_remote_call/errors.py

@@ -0,0 +1,102 @@
+"""Versioned, data-only remote exception transport shared by all call modes."""
+
+from __future__ import annotations
+
+import importlib
+import json
+import traceback
+from types import TracebackType
+
+
+def _user_stack(tb: TracebackType | None) -> traceback.StackSummary:
+    frames = []
+    entered_user_code = False
+    while tb is not None:
+        frame = tb.tb_frame
+        module = str(frame.f_globals.get("__name__", ""))
+        runner_frame = not entered_user_code and module in {
+            "asyncio.base_events",
+            "asyncio.runners",
+            "concurrent.futures._base",
+            "concurrent.futures.thread",
+        }
+        if not (
+            runner_frame
+            or module == "chalk_remote_call.servicer"
+            or module == "chalk_remote_call.errors"
+            or module.startswith("chalkcompute.")
+            or frame.f_globals.get("__chalk_generated_handler__") is True
+        ):
+            entered_user_code = True
+            filename, lineno = frame.f_code.co_filename, tb.tb_lineno
+            source_map = frame.f_globals.get("__chalk_source_map__", {})
+            mapped = source_map.get(lineno) if isinstance(source_map, dict) else None
+            if isinstance(mapped, tuple) and len(mapped) == 3:
+                filename, lineno, source = mapped
+            else:
+                source = None
+            frames.append(traceback.FrameSummary(filename, lineno, frame.f_code.co_name, line=source))
+        tb = tb.tb_next
+    return traceback.StackSummary.from_list(frames)
+
+
+def _filter_traceback(summary: traceback.TracebackException, error: BaseException) -> None:
+    summary.stack = _user_stack(error.__traceback__)
+    if summary.__cause__ is not None and error.__cause__ is not None:
+        _filter_traceback(summary.__cause__, error.__cause__)
+    if summary.__context__ is not None and error.__context__ is not None:
+        _filter_traceback(summary.__context__, error.__context__)
+    children = getattr(error, "exceptions", ())
+    for child_summary, child in zip(getattr(summary, "exceptions", None) or (), children, strict=False):
+        _filter_traceback(child_summary, child)
+
+
+def serialize_exception(error: BaseException) -> str:
+    """Keep chains and groups, without pickling exceptions or their locals."""
+    summary = traceback.TracebackException.from_exception(error, capture_locals=False)
+    internal = "".join(summary.format())
+    _filter_traceback(summary, error)
+    payload: dict[str, object] = {
+        "chalk_remote_error": 1,
+        "exception": {
+            "kind": type(error).__name__[:256],
+            "message": _safe_message(error),
+            "stacktrace": "".join(summary.format()),
+            "internal_stacktrace": internal,
+        },
+    }
+    # Include a trace ID when serialization runs with an active trace context.
+    try:
+        trace = importlib.import_module("opentelemetry.trace")
+        context = trace.get_current_span().get_span_context()
+        if context.is_valid:
+            payload["trace_id"] = format(context.trace_id, "032x")
+    except ImportError:
+        pass
+    # gRPC status messages live in headers. Keep the complete diagnostic in
+    # server logs, and bound the wire payload below common 8 KiB header limits.
+    encoded = json.dumps(payload, ensure_ascii=True)
+    exception = payload["exception"]
+    assert isinstance(exception, dict)
+    while _status_header_size(encoded) > 6000:
+        field = max(exception, key=lambda key: _status_header_size(json.dumps(exception[key], ensure_ascii=True)))
+        text = exception[field]
+        keep = len(text) // 4
+        if keep < 20:
+            break
+        exception[field] = text[:keep] + "\n... remote traceback truncated ...\n" + text[-keep:]
+        encoded = json.dumps(payload, ensure_ascii=True)
+    return encoded
+
+
+def _status_header_size(message: str) -> int:
+    # Tonic percent-encodes these ASCII characters in grpc-message. The JSON
+    # serializer already escapes controls and non-ASCII code points.
+    return len(message) + 2 * sum(char in ' "#%<>`?{}' for char in message)
+
+
+def _safe_message(error: BaseException) -> str:
+    try:
+        return str(error)
+    except Exception:
+        return "<exception message unavailable>"

chalk_remote_call_python-1.9.2/chalk_remote_call/logging.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+_RECORD_FIELDS = frozenset(logging.makeLogRecord({}).__dict__) | {"message", "asctime"}
+
+
+class JsonFormatter(logging.Formatter):
+    """Emit one JSON object per line using fields recognized by Chalk's log collector."""
+
+    def format(self, record: logging.LogRecord) -> str:
+        fields: dict[str, Any] = {key: value for key, value in vars(record).items() if key not in _RECORD_FIELDS}
+        fields.update(
+            timestamp=datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(),
+            severity=record.levelname,
+            logger=record.name,
+            message=record.getMessage(),
+        )
+        # Keep tracebacks out of the message and escape their newlines so the collector sees one event.
+        if record.exc_info:
+            fields["exception.stacktrace"] = self.formatException(record.exc_info)
+        elif record.exc_text:
+            fields["exception.stacktrace"] = record.exc_text
+        if record.stack_info:
+            fields["stack_info"] = self.formatStack(record.stack_info)
+        return json.dumps(fields, default=str)

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk_remote_call/servicer.py

@@ -200,34 +200,28 @@ def process_batches(
             raise ValueError(f"Input transformation failed: {e}") from e
 
         with remote_function_invocation_span(context_metadata):
-            try:
-                result = handler(event, context_metadata)
-                # If the handler is async, await the coroutine.
-                if inspect.iscoroutine(result):
-                    result = _run_async(result)
-            except Exception as e:
-                raise RuntimeError(f"Exception raised during handler execution: {e}") from e
+            result = handler(event, context_metadata)
+            # If the handler is async, await the coroutine.
+            if inspect.iscoroutine(result):
+                result = _run_async(result)
 
-            try:
-                if inspect.isgenerator(result):
-                    # Sync generator: encode each yielded value as a separate chunk.
-                    for item in result:
-                        result_batch = _coerce_to_record_batch(item)
-                        append_or_emit(encode_record_batch(result_batch))
-                elif inspect.isasyncgen(result):
-                    # Async generator: stream when possible, otherwise preserve
-                    # the existing list-returning bridge contract.
-                    if emit is not None:
-                        _drain_async_gen(result, emit)
-                    else:
-                        for item in _collect_async_gen(result):
-                            result_batch = _coerce_to_record_batch(item)
-                            results.append(encode_record_batch(result_batch))
-                else:
-                    result_batch = _coerce_to_record_batch(result)
+            if inspect.isgenerator(result):
+                # Sync generator: encode each yielded value as a separate chunk.
+                for item in result:
+                    result_batch = _coerce_to_record_batch(item)
                     append_or_emit(encode_record_batch(result_batch))
-            except Exception as e:
-                raise RuntimeError(f"Failed to encode response: {e}") from e
+            elif inspect.isasyncgen(result):
+                # Async generator: stream when possible, otherwise preserve
+                # the existing list-returning bridge contract.
+                if emit is not None:
+                    _drain_async_gen(result, emit)
+                else:
+                    for item in _collect_async_gen(result):
+                        result_batch = _coerce_to_record_batch(item)
+                        results.append(encode_record_batch(result_batch))
+            else:
+                result_batch = _coerce_to_record_batch(result)
+                append_or_emit(encode_record_batch(result_batch))
 
     return results
 
@@ -300,12 +294,9 @@ def process_batches_coalesced(
     # Dispatch. Contexts are passed as a parallel list — handler can correlate
     # per-caller metadata with each event by index.
     with remote_function_invocation_span(context_metadatas, coalesced_count=len(context_metadatas)):
-        try:
-            result = handler(events, context_metadatas)
-            if inspect.iscoroutine(result):
-                result = _run_async(result)
-        except Exception as e:
-            raise RuntimeError(f"Exception raised during handler execution: {e}") from e
+        result = handler(events, context_metadatas)
+        if inspect.iscoroutine(result):
+            result = _run_async(result)
 
         if not isinstance(result, list):
             raise TypeError(f"Coalesced handler must return a list, got {type(result).__name__}")
@@ -402,12 +393,9 @@ def process_batches_coalesced_combined(
         combined = combined.rename_columns(names)
 
     with remote_function_invocation_span(context_metadatas, coalesced_count=len(context_metadatas)):
-        try:
-            result = handler(combined, offsets, context_metadatas)
-            if inspect.iscoroutine(result):
-                result = _run_async(result)
-        except Exception as e:
-            raise RuntimeError(f"Exception raised during handler execution: {e}") from e
+        result = handler(combined, offsets, context_metadatas)
+        if inspect.iscoroutine(result):
+            result = _run_async(result)
 
         if isinstance(result, pa.RecordBatch):
             result_rb = result

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk_remote_call_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.9.1
+Version: 1.9.2
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai
@@ -108,6 +108,16 @@ python -m chalk_remote_call --handler my_handler.handler
 | `--on-shutdown` | | | Dotted path to a shutdown function |
 | `--log-level` | | `INFO` | `DEBUG`, `INFO`, `WARNING`, or `ERROR` |
 
+The CLI writes Python logs to stderr as one JSON object per line, with separate
+`timestamp`, `severity`, `logger`, and `message` fields. Exceptions include an
+`exception.stacktrace` field. Calls to `serve()` from Python retain the embedding
+application's logging configuration.
+
+Missing optional StatsD endpoints or a metrics-bus topic do not emit warnings.
+Failures initializing configured metrics sinks still warn. Routine customer
+StatsD configuration messages are emitted at DEBUG by the shared metrics library;
+set `RUST_LOG=info,chalk_metrics::metrics::config=debug` to include them.
+
 ### Environment variables
 
 | Variable | Description |
@@ -212,3 +222,23 @@ To check the Rust code independently:
 ```bash
 PYO3_PYTHON=python3 cargo check --manifest-path chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml
 ```
+
+## Remote exception transport
+
+Handler failures retain their original exception type, message, traceback,
+exception chains, and exception groups. Runtime dispatch frames are excluded
+from the user traceback; full diagnostics are retained in server logs.
+
+All call modes carry a JSON payload with `chalk_remote_error: 1` and an
+`exception` object containing the `ChalkException` fields `kind`, `message`,
+`stacktrace`, and `internal_stacktrace`. Direct calls carry the payload as the
+unescaped gRPC status message; queued calls carry it in the error message. The
+local async service additionally populates `ChalkError.exception`. This permits
+existing queue transports to forward structured errors without a protobuf
+migration. Clients must decode the versioned payload, not parse gRPC debug text
+or classify user errors by their message. Unknown versions remain opaque.
+
+Payloads are bounded for gRPC header limits. Truncation is marked explicitly,
+with complete tracebacks kept in server logs. Exception objects and locals are
+never pickled or sent. Chalk SDK-generated handlers identify their dispatch
+frames, and notebook source maps identify the deployed snapshot's lines.

{chalk_remote_call_python-1.9.1 → chalk_remote_call_python-1.9.2}/chalk_remote_call_python.egg-info/SOURCES.txt

@@ -28,8 +28,10 @@ chalk_remote_call/_native.pyi
 chalk_remote_call/_version.py
 chalk_remote_call/arrow_utils.py
 chalk_remote_call/cli.py
+chalk_remote_call/errors.py
 chalk_remote_call/handler_loader.py
 chalk_remote_call/input_transform.py
+chalk_remote_call/logging.py
 chalk_remote_call/server.py
 chalk_remote_call/servicer.py
 chalk_remote_call/tracing.py

chalk_remote_call_python-1.9.1/chalk_remote_call/_version.py

@@ -1 +0,0 @@
-__version__ = "1.9.1"