chalk-remote-call-python 1.8.1__tar.gz → 1.8.3__tar.gz

{chalk_remote_call_python-1.8.1/chalk_remote_call_python.egg-info → chalk_remote_call_python-1.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.8.1
+Version: 1.8.3
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/Cargo.lock

@@ -483,8 +483,8 @@ dependencies = [
 
 [[package]]
 name = "chalk-core"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "anyhow",
  "envconfig",
@@ -498,8 +498,8 @@ dependencies = [
 
 [[package]]
 name = "chalk-protos"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "prost 0.11.9",
  "prost-build",
@@ -510,8 +510,8 @@ dependencies = [
 
 [[package]]
 name = "chalk-queue"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "base64 0.22.1",
  "redis",
@@ -559,8 +559,8 @@ dependencies = [
 
 [[package]]
 name = "chalk_metric_definitions"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "chalk-protos",
  "metrics-generator",
@@ -569,8 +569,8 @@ dependencies = [
 
 [[package]]
 name = "chalk_metrics"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "async-trait",
  "chalk-core",
@@ -603,8 +603,8 @@ dependencies = [
 
 [[package]]
 name = "chalk_statsd"
-version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+version = "0.0.0"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "chalk-core",
  "chrono",
@@ -874,7 +874,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -1472,7 +1472,7 @@ dependencies = [
  "libc",
  "percent-encoding",
  "pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.3",
  "tokio",
  "tower-service",
  "tracing",
@@ -1867,7 +1867,7 @@ dependencies = [
 [[package]]
 name = "metrics-generator"
 version = "0.1.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=44c26fd738fabd7e2223854618d1f9ccb1b08a71#44c26fd738fabd7e2223854618d1f9ccb1b08a71"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=ae8e792f746f77253f682ee0c56b246cfdd88f51#ae8e792f746f77253f682ee0c56b246cfdd88f51"
 dependencies = [
  "codegen",
  "serde",
@@ -2100,7 +2100,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
 dependencies = [
  "libc",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -2761,7 +2761,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys 0.12.1",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -3128,10 +3128,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.3.4",
+ "getrandom 0.4.2",
  "once_cell",
  "rustix 1.1.4",
- "windows-sys 0.52.0",
+ "windows-sys 0.61.2",
 ]
 
 [[package]]
@@ -3607,6 +3607,16 @@ dependencies = [
  "tracing-core",
 ]
 
+[[package]]
+name = "tracing-serde"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
+dependencies = [
+ "serde",
+ "tracing-core",
+]
+
 [[package]]
 name = "tracing-subscriber"
 version = "0.3.23"
@@ -3617,12 +3627,15 @@ dependencies = [
  "nu-ansi-term",
  "once_cell",
  "regex-automata",
+ "serde",
+ "serde_json",
  "sharded-slab",
  "smallvec",
  "thread_local",
  "tracing",
  "tracing-core",
  "tracing-log",
+ "tracing-serde",
 ]
 
 [[package]]

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml

@@ -19,7 +19,7 @@ tokio = { version = "1", features = ["macros", "rt-multi-thread", "sync", "signa
 tokio-stream = "0.1"
 arrow = { version = "54", default-features = false, features = ["ipc"] }
 tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 bytes = "1"
 uuid = { version = "1", features = ["v4"] }
 dashmap = "5"
@@ -36,6 +36,6 @@ futures-util = "0.3"
 # `[patch.crates-io]` here it builds against vanilla crates.io redis (it only
 # uses standard APIs). Same rev across all three so cargo reuses one checkout.
 [target.'cfg(target_os = "linux")'.dependencies]
-chalk-queue = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "44c26fd738fabd7e2223854618d1f9ccb1b08a71" }
-chalk_metrics = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "44c26fd738fabd7e2223854618d1f9ccb1b08a71" }
-chalk_statsd = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "44c26fd738fabd7e2223854618d1f9ccb1b08a71" }
+chalk-queue = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "ae8e792f746f77253f682ee0c56b246cfdd88f51" }
+chalk_metrics = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "ae8e792f746f77253f682ee0c56b246cfdd88f51" }
+chalk_statsd = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "ae8e792f746f77253f682ee0c56b246cfdd88f51" }

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/async_service.rs

@@ -10,7 +10,7 @@ use chalk_remote_call_proto::chalk::runtime::v1::{
     PollRemoteCallResponse, PurgeQueueRequest, PurgeQueueResponse, RemoteCallStatus,
 };
 use tonic::{Request, Response, Status};
-use tracing::{debug, error, info, instrument};
+use tracing::{debug, error, info, instrument, warn};
 
 use crate::python_bridge::{ChunkBuffer, PythonHandler};
 use crate::service::extract_metadata;
@@ -87,10 +87,15 @@ pub struct AsyncRemoteCallServiceImpl {
     /// the per-call dispatch does not satisfy, so async enqueue is refused
     /// rather than silently calling the handler with the wrong shape.
     batching_enabled: bool,
+    is_remote_function: bool,
 }
 
 impl AsyncRemoteCallServiceImpl {
-    pub fn new(python_handler: Arc<PythonHandler>, batching_enabled: bool) -> Self {
+    pub fn new(
+        python_handler: Arc<PythonHandler>,
+        batching_enabled: bool,
+        is_remote_function: bool,
+    ) -> Self {
         let registry: Registry = Arc::new(Mutex::new(HashMap::new()));
         // Runs inside the server's tokio runtime (the service is constructed
         // within `run_server`); reclaims orphaned terminal entries.
@@ -100,6 +105,7 @@ impl AsyncRemoteCallServiceImpl {
             registry,
             next_id: AtomicU64::new(1),
             batching_enabled,
+            is_remote_function,
         }
     }
 }
@@ -125,6 +131,13 @@ impl AsyncRemoteCallService for AsyncRemoteCallServiceImpl {
         let req = request.into_inner();
         let function_name = req.name;
 
+        if self.is_remote_function {
+            warn!(
+                function_name = %function_name,
+                "this enqueue is not fronted by the fnq server (CHALK_REMOTE_FUNCTION is set to true)"
+            );
+        }
+
         // Only inline feather bytes are supported today; the object-store
         // variant is reserved for large payloads and not wired up here.
         let ipc_bytes = match req.args.and_then(|a| a.args) {
@@ -177,7 +190,12 @@ impl AsyncRemoteCallService for AsyncRemoteCallServiceImpl {
                     Err(e) => {
                         s.status = RemoteCallStatus::Failed;
                         s.error = Some(e.to_string());
-                        error!(call_id = %id, error = %e, "async call failed");
+                        error!(
+                            call_id = %id,
+                            exception.stacktrace = e.details(),
+                            "async call failed: {}",
+                            e.details()
+                        );
                     }
                 }
             }

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/coalesce.rs

@@ -13,6 +13,8 @@ use tokio::task::JoinHandle;
 use tonic::Status;
 use tracing::error;
 
+#[cfg(target_os = "linux")]
+use chalk_metrics::PublishingMetricsPipeline;
 use chalk_remote_call_proto::chalk::runtime::v1::CallFunctionResponse;
 
 use crate::python_bridge::{CallerInput, PythonHandler};
@@ -35,10 +37,17 @@ pub struct CoalescingQueue {
     max_size: usize,
     max_duration: Duration,
     state: Mutex<QueueState>,
+    #[cfg(target_os = "linux")]
+    pipeline: Option<Arc<PublishingMetricsPipeline>>,
 }
 
 impl CoalescingQueue {
-    pub fn new(handler: Arc<PythonHandler>, max_size: usize, max_duration_ms: u64) -> Arc<Self> {
+    pub fn new(
+        handler: Arc<PythonHandler>,
+        max_size: usize,
+        max_duration_ms: u64,
+        #[cfg(target_os = "linux")] pipeline: Option<Arc<PublishingMetricsPipeline>>,
+    ) -> Arc<Self> {
         Arc::new(Self {
             handler,
             max_size,
@@ -47,6 +56,8 @@ impl CoalescingQueue {
                 items: Vec::new(),
                 flush_task: None,
             }),
+            #[cfg(target_os = "linux")]
+            pipeline,
         })
     }
 
@@ -90,17 +101,33 @@ impl CoalescingQueue {
         // Dispatch off the queue's critical section — don't block further
         // submissions while Python runs.
         let handler = Arc::clone(&self.handler);
+        #[cfg(target_os = "linux")]
+        let pipeline = self.pipeline.clone();
         tokio::spawn(async move {
-            Self::dispatch(handler, items).await;
+            Self::dispatch(
+                handler,
+                items,
+                #[cfg(target_os = "linux")]
+                pipeline,
+            )
+            .await;
         });
     }
 
-    async fn dispatch(handler: Arc<PythonHandler>, items: Vec<BufferedCall>) {
+    async fn dispatch(
+        handler: Arc<PythonHandler>,
+        items: Vec<BufferedCall>,
+        #[cfg(target_os = "linux")] pipeline: Option<Arc<PublishingMetricsPipeline>>,
+    ) {
         // Split each BufferedCall into its input (for the bridge) and its
         // response channel (for routing the reply back).
         let n = items.len();
         let mut callers: Vec<CallerInput> = Vec::with_capacity(n);
         let mut response_txs = Vec::with_capacity(n);
+        #[cfg(target_os = "linux")]
+        let function_name = items[0].input.function_name.clone();
+        #[cfg(target_os = "linux")]
+        let started = std::time::Instant::now();
         for item in items {
             callers.push(item.input);
             response_txs.push(item.response_tx);
@@ -116,11 +143,31 @@ impl CoalescingQueue {
                         n
                     );
                     error!("{}", msg);
+                    #[cfg(target_os = "linux")]
+                    if let Some(p) = &pipeline {
+                        crate::metrics::record_completed_call(
+                            p,
+                            &function_name,
+                            /* success = */ false,
+                            started.elapsed(),
+                            chalk_metrics::metrics::tags::Mode::Sync,
+                        );
+                    }
                     for tx in &response_txs {
                         let _ = tx.send(Err(Status::unknown(msg.clone()))).await;
                     }
                     return;
                 }
+                #[cfg(target_os = "linux")]
+                if let Some(p) = &pipeline {
+                    crate::metrics::record_completed_call(
+                        p,
+                        &function_name,
+                        /* success = */ true,
+                        started.elapsed(),
+                        chalk_metrics::metrics::tags::Mode::Sync,
+                    );
+                }
                 for (tx, chunk) in response_txs.into_iter().zip(per_caller_bytes) {
                     let msg = CallFunctionResponse {
                         feather_stream: chunk.into(),
@@ -129,9 +176,23 @@ impl CoalescingQueue {
                 }
             }
             Err(e) => {
-                error!("Coalesced handler error: {}", e);
+                error!(
+                    exception.stacktrace = e.details(),
+                    "Coalesced handler error: {}",
+                    e.details()
+                );
                 // IPC decode errors are already caught by the service's
                 // pre-validation; anything reaching here is a handler error.
+                #[cfg(target_os = "linux")]
+                if let Some(p) = &pipeline {
+                    crate::metrics::record_completed_call(
+                        p,
+                        &function_name,
+                        /* success = */ false,
+                        started.elapsed(),
+                        chalk_metrics::metrics::tags::Mode::Sync,
+                    );
+                }
                 let status =
                     Status::unknown(format!("Exception raised during handler execution: {e}"));
                 for tx in response_txs {

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/lib.rs

@@ -44,8 +44,12 @@ fn start_server(
     max_batching_size: usize,
     max_buffer_duration_ms: u64,
 ) -> PyResult<()> {
-    // Initialize tracing (respects RUST_LOG env var, defaults to info)
+    // Kubernetes consumes stdout/stderr as a machine-readable log stream. Keep
+    // event fields flattened so collectors can read `level`, `target`,
+    // `message`, and fields such as `error` without parsing terminal output.
     let _ = tracing_subscriber::fmt()
+        .json()
+        .flatten_event(true)
         .with_env_filter(
             EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")),
         )
@@ -53,16 +57,6 @@ fn start_server(
 
     let python_handler = Arc::new(PythonHandler::new(handler, process_fn, arg_names));
 
-    let coalescing_queue = if max_batching_size > 0 {
-        Some(CoalescingQueue::new(
-            Arc::clone(&python_handler),
-            max_batching_size,
-            max_buffer_duration_ms,
-        ))
-    } else {
-        None
-    };
-
     // Release the GIL while the Rust server runs
     py.detach(|| {
         let rt = tokio::runtime::Builder::new_multi_thread()
@@ -75,12 +69,35 @@ fn start_server(
                 ))
             })?;
 
-        rt.block_on(server::run_server(
-            host,
-            port,
-            python_handler,
-            coalescing_queue,
-        ))
+        rt.block_on(async {
+            // Linux-only: the self-consumer + its metrics pipeline depend on
+            // chalk-queue / chalk_metrics, since these don't compile on macos
+            // and windows. Built inside the runtime since it needs to `.await`.
+            #[cfg(target_os = "linux")]
+            let pipeline = crate::metrics::init_pipeline().await;
+
+            let coalescing_queue = if max_batching_size > 0 {
+                Some(CoalescingQueue::new(
+                    Arc::clone(&python_handler),
+                    max_batching_size,
+                    max_buffer_duration_ms,
+                    #[cfg(target_os = "linux")]
+                    pipeline.clone(),
+                ))
+            } else {
+                None
+            };
+
+            server::run_server(
+                host,
+                port,
+                python_handler,
+                coalescing_queue,
+                #[cfg(target_os = "linux")]
+                pipeline,
+            )
+            .await
+        })
         .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(format!("Server error: {e}")))
     })
 }

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/metrics.rs

@@ -10,7 +10,7 @@
 use std::sync::Arc;
 use std::time::Duration;
 
-use chalk_metrics::metrics::tags::{FunctionName, Status as MetricStatus, TagValue};
+use chalk_metrics::metrics::tags::{FunctionName, Mode, Status as MetricStatus, TagValue};
 use chalk_metrics::metrics::well_known_metrics::WellKnownMetricName;
 use chalk_metrics::{
     make_pubsub_publisher, BatchConfig, BusPublisher, MetricsPipeline, PublishingMetricsPipeline,
@@ -30,7 +30,7 @@ const BATCH_MAX_LATENCY: Duration = Duration::from_millis(100);
 /// The bus publisher is what reaches the metrics store (and thus the dashboard);
 /// statsd alone only reaches Datadog. An empty `METRICS_BUS_TOPIC` therefore
 /// means the dashboard won't move — surfaced as a warning, not an error.
-pub async fn init_pipeline() -> Option<Arc<dyn MetricsPipeline>> {
+pub async fn init_pipeline() -> Option<Arc<PublishingMetricsPipeline>> {
     let environment_id = std::env::var("CHALK_ENVIRONMENT_ID").unwrap_or_default();
 
     let statsd = Arc::new(
@@ -45,7 +45,7 @@ pub async fn init_pipeline() -> Option<Arc<dyn MetricsPipeline>> {
         warn!("no metrics bus publisher configured; function call metrics will not reach the dashboard");
     }
 
-    let metrics_pipeline: Arc<dyn MetricsPipeline> =
+    let metrics_pipeline: Arc<PublishingMetricsPipeline> =
         Arc::new(PublishingMetricsPipeline::new_with_env_var_defaults(
             environment_id,
             String::new(),
@@ -111,10 +111,11 @@ async fn build_bus_publisher() -> Option<Arc<dyn BusPublisher>> {
 /// carry the scaling-group revision tag used by the legacy catalog consumer;
 /// the dashboard filters by function name.
 pub fn record_completed_call(
-    pipeline: &Arc<dyn MetricsPipeline>,
+    pipeline: &Arc<PublishingMetricsPipeline>,
     function_name: &str,
     success: bool,
     elapsed: Duration,
+    mode: Mode,
 ) {
     let status = if success {
         MetricStatus::Success
@@ -124,6 +125,7 @@ pub fn record_completed_call(
     let tags = vec![
         TagValue::FunctionName(FunctionName(function_name.to_string())),
         TagValue::Status(status.clone()),
+        TagValue::Mode(mode.clone()),
     ];
 
     if let Err(e) = pipeline.count(WellKnownMetricName::FunctionCallDequeued, 1.0, tags.clone()) {

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/python_bridge.rs

@@ -74,7 +74,7 @@ impl PythonHandler {
             })
         })
         .await
-        .map_err(|e| PythonError(format!("Task join error: {e}")))?
+        .map_err(|e| PythonError::new(format!("Task join error: {e}")))?
     }
 
     /// Non-batched streaming dispatch. Calls `process_batches(..., emit)` so
@@ -147,7 +147,7 @@ impl PythonHandler {
             })
         })
         .await
-        .map_err(|e| PythonError(format!("Task join error: {e}")))?
+        .map_err(|e| PythonError::new(format!("Task join error: {e}")))?
     }
 }
 
@@ -305,13 +305,13 @@ fn invoke_process_fn(
     let result_list: &Bound<'_, PyList> = result
         .bind(py)
         .cast::<PyList>()
-        .map_err(|e| PythonError(format!("bridge must return a list, got: {e}")))?;
+        .map_err(|e| PythonError::new(format!("bridge must return a list, got: {e}")))?;
 
     let mut out = Vec::with_capacity(result_list.len());
     for item in result_list.iter() {
-        let bytes_ref: &Bound<'_, PyBytes> = item
-            .cast::<PyBytes>()
-            .map_err(|e| PythonError(format!("bridge must return list[bytes], got item: {e}")))?;
+        let bytes_ref: &Bound<'_, PyBytes> = item.cast::<PyBytes>().map_err(|e| {
+            PythonError::new(format!("bridge must return list[bytes], got item: {e}"))
+        })?;
         out.push(bytes_ref.as_bytes().to_vec());
     }
     Ok(out)
@@ -319,14 +319,38 @@ fn invoke_process_fn(
 
 /// Error type for Python handler invocations.
 #[derive(Debug)]
-pub struct PythonError(pub String);
+pub struct PythonError {
+    details: String,
+}
+
+impl PythonError {
+    fn new(message: String) -> Self {
+        Self { details: message }
+    }
+
+    pub fn details(&self) -> &str {
+        &self.details
+    }
+}
 
 impl std::fmt::Display for PythonError {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.0)
+        write!(f, "{}", self.details)
     }
 }
 
 fn into_python_error(py: Python<'_>, err: PyErr) -> PythonError {
-    PythonError(format!("{}", err.value(py)))
+    let summary = err.to_string();
+    let details = format_python_traceback(py, &err).unwrap_or_else(|_| summary.clone());
+    PythonError { details }
+}
+
+fn format_python_traceback(py: Python<'_>, err: &PyErr) -> PyResult<String> {
+    let traceback = py.import("traceback")?;
+    let formatted = traceback.call_method1(
+        "format_exception",
+        (err.get_type(py), err.value(py), err.traceback(py)),
+    )?;
+    let lines: Vec<String> = formatted.extract()?;
+    Ok(lines.concat().trim_end().to_string())
 }

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/self_consumer.rs

@@ -28,6 +28,7 @@
 //!   caller. The semaphore still admits one slot per call, so it remains the
 //!   producer of the concurrency the coalescer batches.
 
+use std::collections::HashMap;
 use std::panic::AssertUnwindSafe;
 use std::sync::{Arc, Mutex};
 use std::time::{Duration, Instant};
@@ -44,7 +45,8 @@ use tokio::sync::mpsc;
 use tracing::{error, info, warn};
 use uuid::Uuid;
 
-use chalk_metrics::MetricsPipeline;
+use chalk_metrics::metrics::tags::Mode;
+use chalk_metrics::PublishingMetricsPipeline;
 
 use crate::coalesce::{BufferedCall, CoalescingQueue};
 use crate::python_bridge::{CallerInput, ChunkBuffer, PythonHandler};
@@ -186,7 +188,7 @@ type LiveTokens = Arc<DashSet<String>>;
 pub async fn run(
     config: SelfConsumerConfig,
     handler: Arc<PythonHandler>,
-    metrics_pipeline: Option<Arc<dyn MetricsPipeline>>,
+    metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
     coalescing_queue: Option<Arc<CoalescingQueue>>,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let client = Client::open(config.redis_url.clone())?;
@@ -310,7 +312,7 @@ async fn run_one_call(
     lease: SemaphoreLease,
     live_tokens: LiveTokens,
     flush_timeout: Duration,
-    metrics_pipeline: Option<Arc<dyn MetricsPipeline>>,
+    metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
 ) {
     let call_id = work.call_id.clone();
     match AssertUnwindSafe(execute_and_write(
@@ -362,7 +364,7 @@ async fn run_one_coalesced_call(
     work: QueuedCall,
     lease: SemaphoreLease,
     live_tokens: LiveTokens,
-    metrics_pipeline: Option<Arc<dyn MetricsPipeline>>,
+    metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
 ) {
     let call_id = work.call_id.clone();
     if let Err(e) = coalesced_execute_and_write(
@@ -397,13 +399,14 @@ async fn coalesced_execute_and_write(
     queue: &FunctionQueue,
     conn: &mut ConnectionManager,
     work: QueuedCall,
-    metrics_pipeline: Option<&Arc<dyn MetricsPipeline>>,
+    metrics_pipeline: Option<&Arc<PublishingMetricsPipeline>>,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let call_uuid = Uuid::parse_str(&work.call_id)?;
     let stream_key = queue.stream_key_for(call_uuid);
     let ipc_bytes = work.call.feather_bytes()?;
     let function_name = work.call.name.clone();
-    let metadata = work.metadata.otel_headers.clone();
+    let mut metadata = work.metadata.otel_headers.clone();
+    let processing_trace_id = ensure_traceparent(&mut metadata);
     let started = Instant::now();
 
     let _ = queue.update_status(call_uuid, "running").await;
@@ -430,7 +433,13 @@ async fn coalesced_execute_and_write(
     // stream writes below — parity with the streaming path.
     let success = matches!(response, Some(Ok(_)));
     if let Some(p) = metrics_pipeline {
-        crate::metrics::record_completed_call(p, &function_name, success, started.elapsed());
+        crate::metrics::record_completed_call(
+            p,
+            &function_name,
+            success,
+            started.elapsed(),
+            Mode::Async,
+        );
     }
 
     match response {
@@ -438,7 +447,12 @@ async fn coalesced_execute_and_write(
             stream_writer::write_chunk(conn, &stream_key, 1, &resp.feather_stream).await?;
             stream_writer::write_end(conn, &stream_key, 2).await?;
             let _ = queue
-                .complete_status(call_uuid, "completed", "1 chunk(s)")
+                .complete_status(
+                    call_uuid,
+                    "completed",
+                    "1 chunk(s)",
+                    Some(processing_trace_id),
+                )
                 .await;
             info!(call_id = %work.call_id, "coalesced queued call completed");
         }
@@ -455,7 +469,9 @@ async fn coalesced_execute_and_write(
             )
             .await?;
             let summary: String = status.message().chars().take(200).collect();
-            let _ = queue.complete_status(call_uuid, "failed", &summary).await;
+            let _ = queue
+                .complete_status(call_uuid, "failed", &summary, Some(processing_trace_id))
+                .await;
             error!(call_id = %work.call_id, error = %status, "coalesced queued call failed");
         }
         None => {
@@ -465,13 +481,31 @@ async fn coalesced_execute_and_write(
             let msg = "coalesced dispatch produced no response";
             stream_writer::write_error(conn, &stream_key, 1, "RemoteFunctionError", msg, None)
                 .await?;
-            let _ = queue.complete_status(call_uuid, "failed", msg).await;
+            let _ = queue
+                .complete_status(call_uuid, "failed", msg, Some(processing_trace_id))
+                .await;
             error!(call_id = %work.call_id, "{msg}");
         }
     }
     Ok(())
 }
 
+// Insert trace ID into headers if none already present
+fn ensure_traceparent(headers: &mut HashMap<String, String>) -> String {
+    if let Some(tp) = headers.get("traceparent") {
+        if let Some(trace_id) = tp.split('-').nth(1).filter(|s| s.len() == 32) {
+            return trace_id.to_string();
+        }
+    }
+    let trace_id = Uuid::new_v4().simple().to_string();
+    let span_id = &Uuid::new_v4().simple().to_string()[..16];
+    headers.insert(
+        "traceparent".to_string(),
+        format!("00-{trace_id}-{span_id}-01"),
+    );
+    trace_id
+}
+
 /// Decode the payload, invoke the handler, and write the outcome (chunks then
 /// `end`, or `error`) to the call's result stream, updating meta status along
 /// the way.
@@ -481,13 +515,14 @@ async fn execute_and_write(
     conn: &mut ConnectionManager,
     work: QueuedCall,
     flush_timeout: Duration,
-    metrics_pipeline: Option<&Arc<dyn MetricsPipeline>>,
+    metrics_pipeline: Option<&Arc<PublishingMetricsPipeline>>,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let call_uuid = Uuid::parse_str(&work.call_id)?;
     let stream_key = queue.stream_key_for(call_uuid);
     let ipc_bytes = work.call.feather_bytes()?;
     let function_name = work.call.name.clone();
-    let metadata = work.metadata.otel_headers.clone();
+    let mut metadata = work.metadata.otel_headers.clone();
+    let processing_trace_id = ensure_traceparent(&mut metadata);
     let started = Instant::now();
 
     let _ = queue.update_status(call_uuid, "running").await;
@@ -543,7 +578,13 @@ async fn execute_and_write(
     // so every executed call is recorded regardless of downstream Redis write
     // failure. Mirrors the catalog consumer's per-call emission.
     if let Some(p) = metrics_pipeline {
-        crate::metrics::record_completed_call(p, &function_name, result.is_ok(), started.elapsed());
+        crate::metrics::record_completed_call(
+            p,
+            &function_name,
+            result.is_ok(),
+            started.elapsed(),
+            Mode::Async,
+        );
     }
 
     // Flush whatever the handler produced after the loop exits (on the fast path
@@ -558,19 +599,37 @@ async fn execute_and_write(
         Ok(()) => {
             stream_writer::write_end(conn, &stream_key, seq).await?;
             let _ = queue
-                .complete_status(call_uuid, "completed", &format!("{flushed} chunk(s)"))
+                .complete_status(
+                    call_uuid,
+                    "completed",
+                    &format!("{flushed} chunk(s)"),
+                    Some(processing_trace_id),
+                )
                 .await;
             info!(call_id = %work.call_id, chunks = flushed, "queued call completed");
         }
         Err(e) => {
-            // Parity with the enqueue/poll path: only a message string is
-            // available (no structured exc_type / traceback yet), so exc_type is
-            // a fixed placeholder.
-            stream_writer::write_error(conn, &stream_key, seq, "RemoteFunctionError", &e.0, None)
-                .await?;
-            let summary: String = e.0.chars().take(200).collect();
-            let _ = queue.complete_status(call_uuid, "failed", &summary).await;
-            error!(call_id = %work.call_id, error = %e, "queued call failed");
+            // The formatted Python traceback is available, but the concrete
+            // exception type is not carried separately across this boundary.
+            stream_writer::write_error(
+                conn,
+                &stream_key,
+                seq,
+                "RemoteFunctionError",
+                e.details(),
+                None,
+            )
+            .await?;
+            let summary: String = e.to_string().chars().take(200).collect();
+            let _ = queue
+                .complete_status(call_uuid, "failed", &summary, Some(processing_trace_id))
+                .await;
+            error!(
+                call_id = %work.call_id,
+                exception.stacktrace = e.details(),
+                "queued call failed: {}",
+                e.details()
+            );
         }
     }
     Ok(())

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/server.rs

@@ -1,6 +1,8 @@
 use std::net::SocketAddr;
 use std::sync::Arc;
 
+#[cfg(target_os = "linux")]
+use chalk_metrics::PublishingMetricsPipeline;
 use chalk_remote_call_proto::chalk::runtime::v1::async_remote_call_service_server::AsyncRemoteCallServiceServer;
 use chalk_remote_call_proto::chalk::runtime::v1::remote_call_service_server::RemoteCallServiceServer;
 use chalk_remote_call_proto::chalk::runtime::v1::FILE_DESCRIPTOR_SET;
@@ -14,11 +16,14 @@ use crate::service::RemoteCallServiceImpl;
 
 const MAX_MESSAGE_SIZE: usize = 32 * 1024 * 1024; // 32 MB
 
+const ENV_REMOTE_FUNCTION: &str = "CHALK_REMOTE_FUNCTION";
+
 pub async fn run_server(
     host: String,
     port: u16,
     python_handler: Arc<PythonHandler>,
     coalescing_queue: Option<Arc<CoalescingQueue>>,
+    #[cfg(target_os = "linux")] pipeline: Option<Arc<PublishingMetricsPipeline>>,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     // Parse host — ensure IPv6 addresses are bracketed for SocketAddr parsing
     let addr_str = if host.contains(':') && !host.starts_with('[') {
@@ -57,7 +62,14 @@ pub async fn run_server(
     // coalescing flag before `coalescing_queue` is moved into the streaming
     // service below; the async path refuses enqueue when batching is on.
     let batching_enabled = coalescing_queue.is_some();
-    let async_svc = AsyncRemoteCallServiceImpl::new(python_handler.clone(), batching_enabled);
+    // Checks if remote function env var is set or not
+    let is_remote_function: bool = std::env::var(ENV_REMOTE_FUNCTION)
+        .is_ok_and(|v| v.eq_ignore_ascii_case("true") || v == "1");
+    let async_svc = AsyncRemoteCallServiceImpl::new(
+        python_handler.clone(),
+        batching_enabled,
+        is_remote_function,
+    );
 
     // Self-consume the Redis fn queue when this pod was provisioned with a
     // Redis URL (gRPC-only otherwise). The self-consumer shares the gRPC
@@ -65,20 +77,22 @@ pub async fn run_server(
     // inbound gRPC calls feed the same queue so they batch together against the
     // single batch-size/duration the user configured. When batching is off the
     // queue is `None` and the self-consumer dispatches per-call (streaming).
-    //
-    // Linux-only: the self-consumer + its metrics pipeline depend on chalk-queue
-    // / chalk_metrics, since these don't compile on macos and windows.
+
     #[cfg(target_os = "linux")]
     if let Some(cfg) = crate::self_consumer::SelfConsumerConfig::from_env() {
         // Build the chalk_metrics pipeline once (mirrors the fnq-server/cc
         // main init) so the self-consumer can emit function-call metrics.
-        let pipeline = crate::metrics::init_pipeline().await;
         let consumer_handler = python_handler.clone();
+        let consumer_pipeline = pipeline.clone();
         let consumer_coalescing = coalescing_queue.clone();
         tokio::spawn(async move {
-            if let Err(e) =
-                crate::self_consumer::run(cfg, consumer_handler, pipeline, consumer_coalescing)
-                    .await
+            if let Err(e) = crate::self_consumer::run(
+                cfg,
+                consumer_handler,
+                consumer_pipeline,
+                consumer_coalescing,
+            )
+            .await
             {
                 tracing::error!(error = %e, "fnq self-consumer failed to start");
             }
@@ -89,6 +103,8 @@ pub async fn run_server(
     let svc = RemoteCallServiceImpl {
         python_handler,
         coalescing_queue,
+        #[cfg(target_os = "linux")]
+        metrics_pipeline: pipeline,
     };
 
     info!("Server starting on {}", addr);

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3}/chalk-remote-call-rs/chalk-remote-call-server/src/service.rs

@@ -1,6 +1,12 @@
 use std::collections::HashMap;
 use std::sync::Arc;
+#[cfg(target_os = "linux")]
+use std::time::Instant;
 
+#[cfg(target_os = "linux")]
+use chalk_metrics::metrics::tags::Mode;
+#[cfg(target_os = "linux")]
+use chalk_metrics::PublishingMetricsPipeline;
 use chalk_remote_call_proto::chalk::runtime::v1::remote_call_service_server::RemoteCallService;
 use chalk_remote_call_proto::chalk::runtime::v1::{CallFunctionRequest, CallFunctionResponse};
 use tokio_stream::wrappers::ReceiverStream;
@@ -18,6 +24,9 @@ pub struct RemoteCallServiceImpl {
     /// Optional coalescing queue. When `Some`, incoming calls are buffered and
     /// dispatched as a single batched handler invocation.
     pub coalescing_queue: Option<Arc<CoalescingQueue>>,
+    // Linux-gated metrics pipeline
+    #[cfg(target_os = "linux")]
+    pub metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
 }
 
 #[tonic::async_trait]
@@ -90,21 +99,51 @@ impl RemoteCallService for RemoteCallServiceImpl {
 
         // Single-request path.
         let handler = self.python_handler.clone();
+        #[cfg(target_os = "linux")]
+        let metrics_pipeline = self.metrics_pipeline.clone();
         tokio::spawn(async move {
+            #[cfg(target_os = "linux")]
+            let started = Instant::now();
+
             let error_tx = tx.clone();
             match handler
-                .call_streaming(all_bytes, function_name, metadata, peer, tx)
+                .call_streaming(all_bytes, function_name.clone(), metadata, peer, tx)
                 .await
             {
-                Ok(()) => {}
+                Ok(()) => {
+                    #[cfg(target_os = "linux")]
+                    if let Some(p) = &metrics_pipeline {
+                        crate::metrics::record_completed_call(
+                            p,
+                            &function_name,
+                            /* success = */ true,
+                            started.elapsed(),
+                            Mode::Sync,
+                        )
+                    }
+                }
                 Err(e) => {
+                    #[cfg(target_os = "linux")]
+                    if let Some(p) = &metrics_pipeline {
+                        crate::metrics::record_completed_call(
+                            p,
+                            &function_name,
+                            /* success = */ false,
+                            started.elapsed(),
+                            Mode::Sync,
+                        )
+                    }
                     if error_tx.is_closed() {
                         // The only reason blocking_send fails is a dropped
                         // receiver — the client went away mid-stream. Not a
                         // handler failure; nothing to send.
                         debug!("client disconnected during streaming response");
                     } else {
-                        error!("Python handler error: {}", e);
+                        error!(
+                            exception.stacktrace = e.details(),
+                            "Python handler error: {}",
+                            e.details()
+                        );
                         let _ = error_tx
                             .send(Err(Status::unknown(format!(
                                 "Exception raised during handler execution: {e}"

chalk_remote_call_python-1.8.3/chalk_remote_call/_version.py

@@ -0,0 +1 @@
+__version__ = "1.8.3"

{chalk_remote_call_python-1.8.1 → chalk_remote_call_python-1.8.3/chalk_remote_call_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.8.1
+Version: 1.8.3
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai

chalk_remote_call_python-1.8.1/chalk_remote_call/_version.py

@@ -1 +0,0 @@
-__version__ = "1.8.1"