chalk-remote-call-python 1.10.0__tar.gz → 1.10.2__tar.gz

{chalk_remote_call_python-1.10.0/chalk_remote_call_python.egg-info → chalk_remote_call_python-1.10.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.10.0
+Version: 1.10.2
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai

{chalk_remote_call_python-1.10.0 → chalk_remote_call_python-1.10.2}/chalk-remote-call-rs/Cargo.lock

@@ -511,7 +511,7 @@ dependencies = [
 [[package]]
 name = "chalk-queue"
 version = "0.0.0"
-source = "git+https://github.com/chalk-ai/chalk-private.git?rev=c6eed6bd7bdad1f2a7568ef206e570c719c4c64f#c6eed6bd7bdad1f2a7568ef206e570c719c4c64f"
+source = "git+https://github.com/chalk-ai/chalk-private.git?rev=1460144ea659debbaee9dccadcd4d7dbf383ccf8#1460144ea659debbaee9dccadcd4d7dbf383ccf8"
 dependencies = [
  "base64 0.22.1",
  "redis",

{chalk_remote_call_python-1.10.0 → chalk_remote_call_python-1.10.2}/chalk-remote-call-rs/chalk-remote-call-server/Cargo.toml

@@ -37,6 +37,6 @@ futures-util = "0.3"
 # `[patch.crates-io]` here it builds against vanilla crates.io redis (it only
 # uses standard APIs). Metrics and StatsD share a revision so their client types agree.
 [target.'cfg(target_os = "linux")'.dependencies]
-chalk-queue = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "c6eed6bd7bdad1f2a7568ef206e570c719c4c64f" }
+chalk-queue = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "1460144ea659debbaee9dccadcd4d7dbf383ccf8" }
 chalk_metrics = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "a0f8932ec10d3b8340af3e98cf63b800d60b4926" }
 chalk_statsd = { git = "https://github.com/chalk-ai/chalk-private.git", rev = "a0f8932ec10d3b8340af3e98cf63b800d60b4926" }

{chalk_remote_call_python-1.10.0 → chalk_remote_call_python-1.10.2}/chalk-remote-call-rs/chalk-remote-call-server/src/self_consumer.rs

@@ -39,7 +39,7 @@ use std::time::{Duration, Instant};
 
 use chalk_queue::semaphore::renew_many;
 use chalk_queue::stream_writer;
-use chalk_queue::types::QueuedCall;
+use chalk_queue::types::{DeliveryAttempt, QueuedCall};
 use chalk_queue::{
     BoundFunctionQueue, DequeuedCall, FunctionQueue, QueueProtocol, QueueReceipt, SemaphoreLease,
 };
@@ -108,6 +108,7 @@ const PEER: &str = "fnq-self-consumer";
 enum ResultDestination {
     PerCall {
         stream_key: String,
+        result_ttl: Duration,
     },
     Batch {
         stream_key: String,
@@ -125,10 +126,18 @@ impl ResultDestination {
                 batch_result.row_index,
             )
         });
-        Self::from_routing(queue.stream_key_for(call_uuid), batch)
+        Self::from_routing(
+            queue.stream_key_for(call_uuid),
+            Duration::from_secs(work.result_ttl_seconds),
+            batch,
+        )
     }
 
-    fn from_routing(per_call_stream_key: String, batch: Option<(String, String, u64)>) -> Self {
+    fn from_routing(
+        per_call_stream_key: String,
+        result_ttl: Duration,
+        batch: Option<(String, String, u64)>,
+    ) -> Self {
         match batch {
             Some((stream_key, call_id, row_index)) => Self::Batch {
                 stream_key,
@@ -137,6 +146,7 @@ impl ResultDestination {
             },
             None => Self::PerCall {
                 stream_key: per_call_stream_key,
+                result_ttl,
             },
         }
     }
@@ -148,7 +158,7 @@ impl ResultDestination {
         data: &[u8],
     ) -> Result<(), redis::RedisError> {
         match self {
-            Self::PerCall { stream_key } => {
+            Self::PerCall { stream_key, .. } => {
                 stream_writer::write_chunk(conn, stream_key, seq, data).await
             }
             Self::Batch {
@@ -169,7 +179,10 @@ impl ResultDestination {
         seq: u64,
     ) -> Result<(), redis::RedisError> {
         match self {
-            Self::PerCall { stream_key } => stream_writer::write_end(conn, stream_key, seq).await,
+            Self::PerCall {
+                stream_key,
+                result_ttl,
+            } => stream_writer::write_end_with_ttl(conn, stream_key, seq, *result_ttl).await,
             Self::Batch {
                 stream_key,
                 call_id,
@@ -191,9 +204,20 @@ impl ResultDestination {
         traceback: Option<&str>,
     ) -> Result<(), redis::RedisError> {
         match self {
-            Self::PerCall { stream_key } => {
-                stream_writer::write_error(conn, stream_key, seq, exc_type, message, traceback)
-                    .await
+            Self::PerCall {
+                stream_key,
+                result_ttl,
+            } => {
+                stream_writer::write_error_with_ttl(
+                    conn,
+                    stream_key,
+                    seq,
+                    exc_type,
+                    message,
+                    traceback,
+                    *result_ttl,
+                )
+                .await
             }
             Self::Batch {
                 stream_key,
@@ -665,25 +689,83 @@ async fn run_one_call(
     flush_timeout: Duration,
     metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
 ) {
+    let attempt = match input_queue.delivery_attempt(&delivery).await {
+        Ok(attempt) => attempt,
+        Err(error) => {
+            error!(call_id = %delivery.call.call_id, %error, "failed to resolve delivery attempt");
+            finish_delivery(
+                &input_queue,
+                &delivery.receipt,
+                false,
+                &delivery.call.call_id,
+                lease,
+                &live_calls,
+            )
+            .await;
+            return;
+        }
+    };
     let DequeuedCall {
         call: work,
         receipt,
     } = delivery;
     let call_id = work.call_id.clone();
+    let final_attempt = match attempt {
+        DeliveryAttempt::Ready {
+            attempt,
+            max_attempts,
+        } => {
+            info!(%call_id, attempt, max_attempts, "executing queued call attempt");
+            attempt == max_attempts
+        }
+        DeliveryAttempt::Exhausted {
+            attempts,
+            max_attempts,
+        } => {
+            let message = format!(
+                "retry attempts exhausted after {attempts} deliveries ({max_attempts} executions allowed)"
+            );
+            let completed = write_terminal_error(
+                &queue,
+                &mut conn,
+                &work,
+                "RetryAttemptsExhausted",
+                &message,
+                None,
+            )
+            .await
+            .is_ok();
+            finish_delivery(
+                &input_queue,
+                &receipt,
+                completed,
+                &call_id,
+                lease,
+                &live_calls,
+            )
+            .await;
+            return;
+        }
+        DeliveryAttempt::OwnershipLost => {
+            finish_delivery(&input_queue, &receipt, false, &call_id, lease, &live_calls).await;
+            return;
+        }
+    };
     let completed = match AssertUnwindSafe(execute_and_write(
         &handler,
         &queue,
         &mut conn,
-        work,
+        work.clone(),
         flush_timeout,
+        final_attempt,
         metrics_pipeline.as_ref(),
     ))
     .catch_unwind()
     .await
     {
-        Ok(Ok(())) => true,
+        Ok(Ok(completed)) => completed,
         Ok(Err(e)) => {
-            // The call's result stream may be incomplete; the call's 24h stream TTL
+            // The call's result stream may be incomplete; the enqueue safety TTL
             // and the meta status are the backstop for a stuck poller.
             error!(call_id = %call_id, error = %e, "failed to execute/write queued call");
             false
@@ -699,7 +781,20 @@ async fn run_one_call(
                 panic = %panic_message,
                 "panic while executing/writing queued call"
             );
-            false
+            if final_attempt {
+                write_terminal_error(
+                    &queue,
+                    &mut conn,
+                    &work,
+                    "RemoteFunctionPanic",
+                    panic_message,
+                    None,
+                )
+                .await
+                .is_ok()
+            } else {
+                false
+            }
         }
     };
 
@@ -753,24 +848,82 @@ async fn run_one_coalesced_call(
     live_calls: LiveCalls,
     metrics_pipeline: Option<Arc<PublishingMetricsPipeline>>,
 ) {
+    let attempt = match input_queue.delivery_attempt(&delivery).await {
+        Ok(attempt) => attempt,
+        Err(error) => {
+            error!(call_id = %delivery.call.call_id, %error, "failed to resolve delivery attempt");
+            finish_delivery(
+                &input_queue,
+                &delivery.receipt,
+                false,
+                &delivery.call.call_id,
+                lease,
+                &live_calls,
+            )
+            .await;
+            return;
+        }
+    };
     let DequeuedCall {
         call: work,
         receipt,
     } = delivery;
     let call_id = work.call_id.clone();
+    let final_attempt = match attempt {
+        DeliveryAttempt::Ready {
+            attempt,
+            max_attempts,
+        } => {
+            info!(%call_id, attempt, max_attempts, "executing coalesced queued call attempt");
+            attempt == max_attempts
+        }
+        DeliveryAttempt::Exhausted {
+            attempts,
+            max_attempts,
+        } => {
+            let message = format!(
+                "retry attempts exhausted after {attempts} deliveries ({max_attempts} executions allowed)"
+            );
+            let completed = write_terminal_error(
+                &queue,
+                &mut conn,
+                &work,
+                "RetryAttemptsExhausted",
+                &message,
+                None,
+            )
+            .await
+            .is_ok();
+            finish_delivery(
+                &input_queue,
+                &receipt,
+                completed,
+                &call_id,
+                lease,
+                &live_calls,
+            )
+            .await;
+            return;
+        }
+        DeliveryAttempt::OwnershipLost => {
+            finish_delivery(&input_queue, &receipt, false, &call_id, lease, &live_calls).await;
+            return;
+        }
+    };
     let completed = match coalesced_execute_and_write(
         &coalescing_queue,
         &queue,
         &mut conn,
         work,
+        final_attempt,
         metrics_pipeline.as_ref(),
     )
     .await
     {
-        Ok(()) => true,
+        Ok(completed) => completed,
         Err(e) => {
-            // The call's result stream may be incomplete; the 24h stream TTL and
-            // the meta status are the backstop for a stuck poller.
+            // The call's result stream may be incomplete; the enqueue safety TTL
+            // and the meta status are the backstop for a stuck poller.
             error!(call_id = %call_id, error = %e, "failed to execute/write coalesced queued call");
             false
         }
@@ -797,8 +950,9 @@ async fn coalesced_execute_and_write(
     queue: &FunctionQueue,
     conn: &mut ConnectionManager,
     work: QueuedCall,
+    final_attempt: bool,
     metrics_pipeline: Option<&Arc<PublishingMetricsPipeline>>,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
     let call_uuid = Uuid::parse_str(&work.call_id)?;
     let result_destination = ResultDestination::for_call(queue, &work, call_uuid);
     let ipc_bytes = work.call.feather_bytes()?;
@@ -857,6 +1011,10 @@ async fn coalesced_execute_and_write(
             info!(call_id = %work.call_id, "coalesced queued call completed");
         }
         Some(Err(status)) => {
+            if !final_attempt {
+                warn!(call_id = %work.call_id, error = %status, "coalesced queued call failed; leaving delivery pending for retry");
+                return Ok(false);
+            }
             // Parity with the enqueue/poll path: only a message string is
             // available, so exc_type is a fixed placeholder.
             result_destination
@@ -873,6 +1031,10 @@ async fn coalesced_execute_and_write(
             // happen (it always sends one Ok/Err per caller), but treat a
             // missing response as a failure rather than leaving the poller hung.
             let msg = "coalesced dispatch produced no response";
+            if !final_attempt {
+                warn!(call_id = %work.call_id, "{msg}; leaving delivery pending for retry");
+                return Ok(false);
+            }
             result_destination
                 .write_error(conn, 1, "RemoteFunctionError", msg, None)
                 .await?;
@@ -882,7 +1044,7 @@ async fn coalesced_execute_and_write(
             error!(call_id = %work.call_id, "{msg}");
         }
     }
-    Ok(())
+    Ok(true)
 }
 
 // Insert trace ID into headers if none already present
@@ -910,8 +1072,9 @@ async fn execute_and_write(
     conn: &mut ConnectionManager,
     work: QueuedCall,
     flush_timeout: Duration,
+    final_attempt: bool,
     metrics_pipeline: Option<&Arc<PublishingMetricsPipeline>>,
-) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+) -> Result<bool, Box<dyn std::error::Error + Send + Sync>> {
     let call_uuid = Uuid::parse_str(&work.call_id)?;
     let result_destination = ResultDestination::for_call(queue, &work, call_uuid);
     let ipc_bytes = work.call.feather_bytes()?;
@@ -923,13 +1086,9 @@ async fn execute_and_write(
     let _ = queue.update_status(call_uuid, "running").await;
     info!(call_id = %work.call_id, function = %function_name, "running queued call");
 
-    // The handler appends each emitted chunk into `buffer` in real time — the
-    // Python bridge calls the emitter per generator-yield. Stream those chunks
-    // to the result stream *as they appear* rather than holding them until the
-    // handler returns: we wait on the notify (signaled on each append) and flush
-    // immediately when chunks arrive, so a slow generator's early yields reach
-    // the client without waiting for the whole call to finish. A timeout fallback
-    // ensures we make progress if the notification pipeline breaks.
+    // The Python bridge appends each generator yield into `buffer`. Calls
+    // without retries flush incrementally; retry-enabled calls retain chunks
+    // until the attempt succeeds so a later attempt cannot duplicate output.
     let buffer = ChunkBuffer::new().into_shared();
     let handler_fut = handler.call_into_buffer(
         ipc_bytes,
@@ -944,6 +1103,9 @@ async fn execute_and_write(
     // seq 0. `flushed` is the count of chunks already written to the stream.
     let mut seq: u64 = 1;
     let mut flushed: usize = 0;
+    // A retry must not duplicate generator chunks. Retry-enabled calls buffer
+    // output until the handler succeeds or reaches its final attempt.
+    let buffer_until_terminal = work.max_retries.is_some();
 
     // Completion is driven by the handler future resolving, NOT by the buffer
     // emitter being torn down — so a handler that pins its emitter (e.g. via a
@@ -960,10 +1122,12 @@ async fn execute_and_write(
             biased;
             res = &mut handler_fut => break res,
             _ = tokio::time::timeout(flush_timeout, notify_handle.notified()) => {
-                if let Err(e) = flush_new(conn, &result_destination, &buffer, &mut flushed, &mut seq).await {
-                    // Non-fatal: keep draining; the cursor didn't advance past the
-                    // failed chunk, so the next wait retries it without duplicating.
-                    warn!(call_id = %work.call_id, error = %e, "failed to flush partial results; retrying");
+                if !buffer_until_terminal {
+                    if let Err(e) = flush_new(conn, &result_destination, &buffer, &mut flushed, &mut seq).await {
+                        // Non-fatal: keep draining; the cursor didn't advance past the
+                        // failed chunk, so the next wait retries it without duplicating.
+                        warn!(call_id = %work.call_id, error = %e, "failed to flush partial results; retrying");
+                    }
                 }
             }
         }
@@ -982,6 +1146,11 @@ async fn execute_and_write(
         );
     }
 
+    if result.is_err() && !final_attempt {
+        warn!(call_id = %work.call_id, "queued call failed; leaving delivery pending for retry");
+        return Ok(false);
+    }
+
     // Flush whatever the handler produced after the loop exits (on the fast path
     // this is everything). This happens even if the handler panicked mid-execution,
     // so partial output is always preserved. Parity with the enqueue/poll path: a
@@ -1025,6 +1194,34 @@ async fn execute_and_write(
             );
         }
     }
+    Ok(true)
+}
+
+async fn write_terminal_error(
+    queue: &FunctionQueue,
+    conn: &mut ConnectionManager,
+    work: &QueuedCall,
+    exc_type: &str,
+    message: &str,
+    traceback: Option<&str>,
+) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+    let call_uuid = Uuid::parse_str(&work.call_id)?;
+    let result_destination = ResultDestination::for_call(queue, work, call_uuid);
+    result_destination
+        .write_error(conn, 1, exc_type, message, traceback)
+        .await?;
+    let summary: String = message.chars().take(200).collect();
+    let processing_trace_id = work
+        .metadata
+        .otel_headers
+        .get("traceparent")
+        .and_then(|value| value.split('-').nth(1))
+        .filter(|value| value.len() == 32)
+        .map(str::to_string);
+    let _ = queue
+        .complete_status(call_uuid, "failed", &summary, processing_trace_id)
+        .await;
+    error!(call_id = %work.call_id, %message, "queued call reached terminal failure");
     Ok(())
 }
 
@@ -1128,9 +1325,14 @@ mod tests {
     #[test]
     fn result_destination_preserves_per_call_stream() {
         assert_eq!(
-            ResultDestination::from_routing("tenant:gen:call-1:stream".to_string(), None),
+            ResultDestination::from_routing(
+                "tenant:gen:call-1:stream".to_string(),
+                Duration::from_secs(60),
+                None,
+            ),
             ResultDestination::PerCall {
                 stream_key: "tenant:gen:call-1:stream".to_string(),
+                result_ttl: Duration::from_secs(60),
             }
         );
     }
@@ -1140,6 +1342,7 @@ mod tests {
         assert_eq!(
             ResultDestination::from_routing(
                 "unused-per-call-stream".to_string(),
+                Duration::from_secs(60),
                 Some((
                     "tenant:batch:batch-1:results".to_string(),
                     "call-1".to_string(),

chalk_remote_call_python-1.10.2/chalk_remote_call/_version.py

@@ -0,0 +1 @@
+__version__ = "1.10.2"

{chalk_remote_call_python-1.10.0 → chalk_remote_call_python-1.10.2/chalk_remote_call_python.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: chalk-remote-call-python
-Version: 1.10.0
+Version: 1.10.2
 Summary: Chalk remote call Python runtime interface client
 Author: Chalk AI, Inc.
 Project-URL: Homepage, https://chalk.ai

chalk_remote_call_python-1.10.0/chalk_remote_call/_version.py

@@ -1 +0,0 @@
-__version__ = "1.10.0"