nebu 0.1.117__tar.gz → 0.1.119__tar.gz

{nebu-0.1.117/src/nebu.egg-info → nebu-0.1.119}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nebu
-Version: 0.1.117
+Version: 0.1.119
 Summary: A globally distributed container runtime
 Requires-Python: >=3.10.14
 Description-Content-Type: text/markdown

{nebu-0.1.117 → nebu-0.1.119}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "nebu"
-version = "0.1.117"
+version = "0.1.119"
 description = "A globally distributed container runtime"
 readme = "README.md"
 requires-python = ">=3.10.14"

{nebu-0.1.117 → nebu-0.1.119}/src/nebu/processors/consumer.py

@@ -388,7 +388,13 @@ def start_health_check_subprocess() -> Optional[subprocess.Popen]:
     """Start the health check consumer subprocess."""
     global REDIS_HEALTH_STREAM, REDIS_HEALTH_CONSUMER_GROUP
 
+    print(f"[DEBUG] start_health_check_subprocess called")
+    print(f"[DEBUG] REDIS_URL: {REDIS_URL}")
+    print(f"[DEBUG] REDIS_HEALTH_STREAM: {REDIS_HEALTH_STREAM}")
+    print(f"[DEBUG] REDIS_HEALTH_CONSUMER_GROUP: {REDIS_HEALTH_CONSUMER_GROUP}")
+
     if not all([REDIS_URL, REDIS_HEALTH_STREAM, REDIS_HEALTH_CONSUMER_GROUP]):
+        print(f"[DEBUG] Health check not configured - missing required variables")
         logger.warning(
             "[Consumer] Health check stream not configured. Health consumer subprocess not started."
         )
@@ -399,6 +405,11 @@ def start_health_check_subprocess() -> Optional[subprocess.Popen]:
         assert isinstance(REDIS_HEALTH_STREAM, str)
         assert isinstance(REDIS_HEALTH_CONSUMER_GROUP, str)
 
+        print(
+            f"[DEBUG] Starting health check subprocess for stream: {REDIS_HEALTH_STREAM}"
+        )
+        print(f"[DEBUG] Health consumer group: {REDIS_HEALTH_CONSUMER_GROUP}")
+
         # Prepare environment variables for the subprocess
         health_env = os.environ.copy()
         health_env["REDIS_HEALTH_STREAM"] = REDIS_HEALTH_STREAM
@@ -412,6 +423,8 @@ def start_health_check_subprocess() -> Optional[subprocess.Popen]:
             "nebu.processors.consumer_health_worker",
         ]
 
+        print(f"[DEBUG] Health subprocess command: {' '.join(health_cmd)}")
+
         process = subprocess.Popen(
             health_cmd,
             stdout=subprocess.PIPE,
@@ -422,12 +435,17 @@ def start_health_check_subprocess() -> Optional[subprocess.Popen]:
             bufsize=1,  # Line buffered
         )
 
+        print(
+            f"[DEBUG] Health check subprocess started successfully with PID {process.pid}"
+        )
         logger.info(
             f"[Consumer] Health check subprocess started with PID {process.pid}"
         )
         return process
 
     except Exception as e:
+        print(f"[DEBUG] Failed to start health check subprocess: {e}")
+        print(f"[DEBUG] Exception type: {type(e)}")
         logger.error(f"[Consumer] Failed to start health check subprocess: {e}")
         logger.exception("Health Subprocess Start Error Traceback:")
         return None
@@ -435,13 +453,17 @@ def start_health_check_subprocess() -> Optional[subprocess.Popen]:
 
 def monitor_health_subprocess(process: subprocess.Popen) -> None:
     """Monitor the health check subprocess and log its output."""
+    print(f"[DEBUG] monitor_health_subprocess started for PID {process.pid}")
     try:
         # Read output from the subprocess
         if process.stdout:
             for line in iter(process.stdout.readline, ""):
+                print(f"[DEBUG] [HealthSubprocess] {line.strip()}")
                 logger.info(f"[HealthSubprocess] {line.strip()}")
         process.stdout.close() if process.stdout else None
+        print(f"[DEBUG] monitor_health_subprocess finished for PID {process.pid}")
     except Exception as e:
+        print(f"[DEBUG] Error monitoring health subprocess: {e}")
         logger.error(f"[Consumer] Error monitoring health subprocess: {e}")
 
 
@@ -449,33 +471,45 @@ def check_health_subprocess() -> bool:
     """Check if the health subprocess is still running and restart if needed."""
     global health_subprocess
 
+    print(f"[DEBUG] check_health_subprocess called")
+
     if health_subprocess is None:
+        print(f"[DEBUG] health_subprocess is None")
         return False
 
     # Check if process is still running
-    if health_subprocess.poll() is None:
+    poll_result = health_subprocess.poll()
+    print(f"[DEBUG] health_subprocess.poll() returned: {poll_result}")
+
+    if poll_result is None:
+        print(f"[DEBUG] Health subprocess still running (PID {health_subprocess.pid})")
         return True  # Still running
 
     # Process has exited
     exit_code = health_subprocess.returncode
+    print(f"[DEBUG] Health subprocess exited with code {exit_code}")
     logger.warning(
         f"[Consumer] Health subprocess exited with code {exit_code}. Restarting..."
     )
 
     # Start a new health subprocess
+    print(f"[DEBUG] Attempting to restart health subprocess...")
     health_subprocess = start_health_check_subprocess()
 
     if health_subprocess:
+        print(f"[DEBUG] Health subprocess restarted successfully")
         # Start monitoring thread for the new subprocess
         monitor_thread = threading.Thread(
             target=monitor_health_subprocess, args=(health_subprocess,), daemon=True
         )
         monitor_thread.start()
+        print(f"[DEBUG] Monitor thread started for health subprocess")
         logger.info(
             "[Consumer] Health subprocess restarted and monitoring thread started."
         )
         return True
     else:
+        print(f"[DEBUG] Failed to restart health subprocess")
         logger.error("[Consumer] Failed to restart health subprocess.")
         return False
 
@@ -714,6 +748,9 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
     user_id = None
     try:
         payload_str = message_data.get("data")
+        print(f"[DEBUG] Raw message_data: {message_data}")
+        print(f"[DEBUG] Payload string: {payload_str}")
+
         if (
             not payload_str
         ):  # Covers None and empty string, isinstance check is redundant
@@ -722,6 +759,7 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
             )
         try:
             raw_payload = json.loads(payload_str)
+            print(f"[DEBUG] Parsed raw_payload: {json.dumps(raw_payload, indent=2)}")
         except json.JSONDecodeError as json_err:
             raise ValueError(f"Failed to parse JSON payload: {json_err}") from json_err
         if not isinstance(raw_payload, dict):
@@ -734,6 +772,9 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         # --- Extract fields from the *inner* message content for HealthCheck and regular processing ---
         # The actual message content is inside raw_payload["content"]
         inner_content_data = raw_payload.get("content", {})
+        print(
+            f"[DEBUG] Extracted inner_content_data: {json.dumps(inner_content_data, indent=2) if isinstance(inner_content_data, dict) else inner_content_data}"
+        )
 
         # Add debug logging for content structure analysis
         logger.debug(f">> inner_content_data type: {type(inner_content_data)}")
@@ -746,6 +787,7 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
             # we can't reliably get 'kind' or other fields from it.
             # This case is more relevant for non-StreamMessage processors.
             # For HealthChecks, we expect a structured 'content'.
+            print(f"[DEBUG] inner_content_data is not a dict, using defaults")
             logger.warning(
                 f"Received non-dict inner_content_data: {inner_content_data}. HealthCheck might be missed if applicable."
             )
@@ -762,6 +804,8 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                 and "content" in inner_content_data
             )
 
+            print(f"[DEBUG] has_message_structure: {has_message_structure}")
+            print(f"[DEBUG] inner_content_data keys: {list(inner_content_data.keys())}")
             logger.debug(f">> has_message_structure: {has_message_structure}")
 
             if has_message_structure:
@@ -770,6 +814,13 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                 inner_msg_id = inner_content_data.get("id", "")
                 actual_content_to_process = inner_content_data.get("content", {})
                 inner_created_at_str = inner_content_data.get("created_at")
+                print(f"[DEBUG] Using nested structure:")
+                print(f"[DEBUG]   inner_kind: '{inner_kind}'")
+                print(f"[DEBUG]   inner_msg_id: '{inner_msg_id}'")
+                print(
+                    f"[DEBUG]   actual_content_to_process: {actual_content_to_process}"
+                )
+                print(f"[DEBUG]   inner_created_at_str: {inner_created_at_str}")
                 logger.debug(
                     f">> Using nested structure - inner_kind: {inner_kind}, inner_msg_id: {inner_msg_id}"
                 )
@@ -786,6 +837,13 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                 inner_created_at_str = raw_payload.get(
                     "created_at"
                 )  # Get created_at from outer payload
+                print(f"[DEBUG] Using direct structure:")
+                print(f"[DEBUG]   inner_kind: '{inner_kind}' (from outer payload)")
+                print(f"[DEBUG]   inner_msg_id: '{inner_msg_id}' (from outer payload)")
+                print(
+                    f"[DEBUG]   actual_content_to_process: {actual_content_to_process}"
+                )
+                print(f"[DEBUG]   inner_created_at_str: {inner_created_at_str}")
                 logger.debug(
                     f">> Using direct structure - inner_kind: {inner_kind}, inner_msg_id: {inner_msg_id}"
                 )
@@ -810,31 +868,117 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         handle = raw_payload.get("handle")  # from outer
         adapter = raw_payload.get("adapter")  # from outer
         api_key = raw_payload.get("api_key")  # from outer
+
+        print(f"[DEBUG] Extracted outer envelope data:")
+        print(f"[DEBUG]   return_stream: {return_stream}")
+        print(f"[DEBUG]   user_id: {user_id}")
+        print(f"[DEBUG]   orgs: {orgs}")
+        print(f"[DEBUG]   handle: {handle}")
+        print(f"[DEBUG]   adapter: {adapter}")
+        print(f"[DEBUG]   api_key length: {len(api_key) if api_key else 0}")
+
         logger.debug(f">> Extracted API key length: {len(api_key) if api_key else 0}")
 
         # --- Health Check Logic ---
         # Use inner_kind for health check
-        if inner_kind == "HealthCheck":
+        print(f"[DEBUG] Checking if message is HealthCheck. inner_kind: '{inner_kind}'")
+        print(
+            f"[DEBUG] Message kind comparison: '{inner_kind}' == 'HealthCheckRequest' -> {inner_kind == 'HealthCheckRequest'}"
+        )
+
+        if inner_kind == "HealthCheckRequest":
+            print(f"[DEBUG] *** HEALTH CHECK REQUEST MESSAGE DETECTED ***")
+            print(f"[DEBUG] Message ID: {message_id}")
+            print(f"[DEBUG] Inner message ID: {inner_msg_id}")
+            print(f"[DEBUG] Return stream: {return_stream}")
+            print(f"[DEBUG] User ID: {user_id}")
+            print(f"[DEBUG] Content: {actual_content_to_process}")
+
             logger.info(
-                f"Received HealthCheck message {message_id} (inner_id: {inner_msg_id})"
+                f"Received HealthCheckRequest message {message_id} (inner_id: {inner_msg_id})"
             )
-            health_response = {
-                "kind": "StreamResponseMessage",
-                "id": message_id,  # Use outer message_id for response ID
-                "content": {"status": "healthy", "checked_message_id": inner_msg_id},
-                "status": "success",
-                "created_at": datetime.now(timezone.utc).isoformat(),
-                "user_id": user_id,
-            }
-            if return_stream:
-                assert isinstance(return_stream, str)
-                r.xadd(return_stream, {"data": json.dumps(health_response)})
-                logger.info(f"Sent health check response to {return_stream}")
 
-            assert isinstance(REDIS_STREAM, str)
-            assert isinstance(REDIS_CONSUMER_GROUP, str)
-            r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
-            logger.info(f"Acknowledged HealthCheck message {message_id}")
+            # Forward to health stream for health worker subprocess to process
+            if REDIS_HEALTH_STREAM:
+                print(
+                    f"[DEBUG] Forwarding health check to health stream: {REDIS_HEALTH_STREAM}"
+                )
+                try:
+                    # Forward the entire message data to the health stream
+                    health_message_data = {
+                        "data": json.dumps(
+                            {
+                                "kind": inner_kind,
+                                "id": inner_msg_id,
+                                "content": actual_content_to_process,
+                                "created_at": inner_created_at.isoformat(),
+                                "return_stream": return_stream,
+                                "user_id": user_id,
+                                "orgs": orgs,
+                                "handle": handle,
+                                "adapter": adapter,
+                                "api_key": api_key,
+                                "original_message_id": message_id,  # Include original message ID for tracking
+                            }
+                        )
+                    }
+
+                    print(
+                        f"[DEBUG] Health message data to forward: {json.dumps(health_message_data, indent=2)}"
+                    )
+
+                    r.xadd(REDIS_HEALTH_STREAM, health_message_data)  # type: ignore[arg-type]
+                    print(
+                        f"[DEBUG] Health check forwarded successfully to {REDIS_HEALTH_STREAM}"
+                    )
+                    logger.info(
+                        f"Forwarded HealthCheckRequest {message_id} to health stream {REDIS_HEALTH_STREAM}"
+                    )
+
+                except Exception as e:
+                    print(
+                        f"[DEBUG] ERROR forwarding health check to health stream: {e}"
+                    )
+                    logger.error(f"Error forwarding health check to health stream: {e}")
+                    # Fall back to sending error response directly
+                    _send_error_response(
+                        message_id,
+                        f"Failed to forward health check: {e}",
+                        traceback.format_exc(),
+                        return_stream,
+                        user_id,
+                    )
+            else:
+                print(
+                    f"[DEBUG] No health stream configured, cannot forward health check"
+                )
+                logger.warning(
+                    "No health stream configured for health check forwarding"
+                )
+                # Send error response since we can't process health checks
+                _send_error_response(
+                    message_id,
+                    "Health check stream not configured",
+                    "REDIS_HEALTH_STREAM environment variable not set",
+                    return_stream,
+                    user_id,
+                )
+
+            # Acknowledge the original message since we've forwarded it
+            print(f"[DEBUG] Acknowledging original health check message {message_id}")
+            try:
+                assert isinstance(REDIS_STREAM, str)
+                assert isinstance(REDIS_CONSUMER_GROUP, str)
+                r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
+                print(
+                    f"[DEBUG] Health check message {message_id} acknowledged successfully"
+                )
+                logger.info(f"Acknowledged HealthCheckRequest message {message_id}")
+            except Exception as e:
+                print(f"[DEBUG] ERROR acknowledging health check message: {e}")
+                logger.error(f"Error acknowledging health check message: {e}")
+
+            print(f"[DEBUG] *** HEALTH CHECK FORWARDING COMPLETE ***")
             return  # Exit early for health checks
         # --- End Health Check Logic ---
 
@@ -1210,19 +1354,31 @@ logger.info(
 
 # Start the health check consumer subprocess
 if REDIS_HEALTH_STREAM and REDIS_HEALTH_CONSUMER_GROUP:
+    print(f"[DEBUG] === HEALTH SUBPROCESS INITIALIZATION ===")
+    print(f"[DEBUG] REDIS_HEALTH_STREAM is set: {REDIS_HEALTH_STREAM}")
+    print(f"[DEBUG] REDIS_HEALTH_CONSUMER_GROUP is set: {REDIS_HEALTH_CONSUMER_GROUP}")
+
     health_subprocess = start_health_check_subprocess()
     if health_subprocess:
+        print(
+            f"[DEBUG] Health subprocess started successfully, starting monitor thread..."
+        )
         # Start monitoring thread for subprocess output
         monitor_thread = threading.Thread(
             target=monitor_health_subprocess, args=(health_subprocess,), daemon=True
         )
         monitor_thread.start()
+        print(f"[DEBUG] Monitor thread started for health subprocess")
         logger.info(
             f"[Consumer] Health check subprocess for {REDIS_HEALTH_STREAM} started and monitoring thread started."
         )
     else:
+        print(f"[DEBUG] Health subprocess failed to start")
         logger.error("[Consumer] Failed to start health check subprocess.")
 else:
+    print(f"[DEBUG] === HEALTH SUBPROCESS NOT CONFIGURED ===")
+    print(f"[DEBUG] REDIS_HEALTH_STREAM: {REDIS_HEALTH_STREAM}")
+    print(f"[DEBUG] REDIS_HEALTH_CONSUMER_GROUP: {REDIS_HEALTH_CONSUMER_GROUP}")
     logger.warning(
         "[Consumer] Health check stream not configured. Health consumer subprocess not started."
     )
@@ -1232,10 +1388,15 @@ try:
         logger.debug(
             f"[{datetime.now(timezone.utc).isoformat()}] --- Top of main loop ---"
         )  # Added log
+        print(f"[DEBUG] === MAIN LOOP ITERATION START ===")
 
         # --- Check Health Subprocess Status ---
         if health_subprocess:
-            check_health_subprocess()
+            print(f"[DEBUG] Checking health subprocess status...")
+            health_status = check_health_subprocess()
+            print(f"[DEBUG] Health subprocess status check result: {health_status}")
+        else:
+            print(f"[DEBUG] No health subprocess to check")
 
         # --- Check for Code Updates ---
         if not disable_hot_reload:
@@ -1448,6 +1609,11 @@ try:
         logger.debug(
             f"[{datetime.now(timezone.utc).isoformat()}] Calling xreadgroup (block=5000ms)..."
         )  # Added log
+        print(f"[DEBUG] About to call xreadgroup...")
+        print(f"[DEBUG] Stream: {REDIS_STREAM}")
+        print(f"[DEBUG] Consumer group: {REDIS_CONSUMER_GROUP}")
+        print(f"[DEBUG] Consumer name: {consumer_name}")
+
         messages = r.xreadgroup(
             REDIS_CONSUMER_GROUP,
             consumer_name,
@@ -1456,10 +1622,14 @@ try:
             block=5000,  # Use milliseconds for block
         )
 
+        print(f"[DEBUG] xreadgroup returned: {messages}")
+        print(f"[DEBUG] Messages type: {type(messages)}")
+
         if not messages:
             logger.trace(
                 f"[{datetime.now(timezone.utc).isoformat()}] xreadgroup timed out (no new messages)."
             )  # Added log
+            print(f"[DEBUG] No messages received (timeout or empty)")
             # logger.debug("[Consumer] No new messages.") # Reduce verbosity
             continue
         # Removed the else block here
@@ -1475,6 +1645,8 @@ try:
         stream_name_str, stream_messages = typed_messages[0]
         num_msgs = len(stream_messages)
 
+        print(f"[DEBUG] Processing {num_msgs} message(s) from stream {stream_name_str}")
+
         # Log reception and count before processing
         logger.info(
             f"[{datetime.now(timezone.utc).isoformat()}] xreadgroup returned {num_msgs} message(s). Processing..."
@@ -1492,8 +1664,13 @@ try:
             #                          for k, v in msg_data_bytes_dict.items() } # No longer needed
             # print(f"Processing message {message_id_str}") # Reduce verbosity
             # print(f"Message data: {message_data_str_dict}") # Reduce verbosity
+            print(f"[DEBUG] === PROCESSING MESSAGE {message_id_str} ===")
+            print(f"[DEBUG] Message data keys: {list(message_data_str_dict.keys())}")
+
             process_message(message_id_str, message_data_str_dict)
 
+            print(f"[DEBUG] === FINISHED PROCESSING MESSAGE {message_id_str} ===")
+
 except ConnectionError as e:
     logger.error(f"Redis connection error: {e}. Reconnecting in 5s...")
     time.sleep(5)

{nebu-0.1.117 → nebu-0.1.119}/src/nebu/processors/consumer_health_worker.py

@@ -5,13 +5,16 @@ import os
 import socket
 import sys
 import time
-from typing import Dict, List, Optional, Tuple, cast
+from typing import Any, Dict, List, Optional, Tuple, cast
 
 import redis
 import socks
 from redis import ConnectionError, ResponseError
 from redis.exceptions import TimeoutError as RedisTimeoutError
 
+# Assuming these are imported from other modules
+from nebu.processors.models import V1ProcessorHealthResponse
+
 
 def setup_health_logging():
     """Set up logging for the health check worker to write to a dedicated file."""
@@ -48,31 +51,131 @@ def process_health_check_message(
     health_group: str,
 ) -> None:
     """Processes a single health check message."""
+    print(f"[HEALTH DEBUG] === PROCESSING HEALTH CHECK MESSAGE ===")
+    print(f"[HEALTH DEBUG] Message ID: {message_id}")
+    print(f"[HEALTH DEBUG] Message data: {message_data}")
+    print(f"[HEALTH DEBUG] Health stream: {health_stream}")
+    print(f"[HEALTH DEBUG] Health group: {health_group}")
+
     logger.info(f"Processing health check message {message_id}: {message_data}")
 
-    # Parse the message if it contains JSON data
+    health_status = "ok"
+    health_message: Optional[str] = "Health check processed successfully."
+    details: Optional[Dict[str, Any]] = None
+    return_stream: Optional[str] = None
+    original_message_id: Optional[str] = None
+    user_id: Optional[str] = None
+
     try:
         if "data" in message_data:
-            data = json.loads(message_data["data"])
+            print(f"[HEALTH DEBUG] Found 'data' field in message")
+            data_str = message_data["data"]
+            print(f"[HEALTH DEBUG] Raw data string: {data_str}")
+
+            data = json.loads(data_str)
+            print(f"[HEALTH DEBUG] Parsed data: {json.dumps(data, indent=2)}")
             logger.info(f"Health check data: {data}")
+
+            # Extract important fields from the forwarded message
+            return_stream = data.get("return_stream")
+            original_message_id = data.get("original_message_id")
+            user_id = data.get("user_id")
+            inner_kind = data.get("kind")
+            inner_id = data.get("id")
+            content = data.get("content")
+
+            print(f"[HEALTH DEBUG] Extracted fields:")
+            print(f"[HEALTH DEBUG]   return_stream: {return_stream}")
+            print(f"[HEALTH DEBUG]   original_message_id: {original_message_id}")
+            print(f"[HEALTH DEBUG]   user_id: {user_id}")
+            print(f"[HEALTH DEBUG]   inner_kind: {inner_kind}")
+            print(f"[HEALTH DEBUG]   inner_id: {inner_id}")
+            print(f"[HEALTH DEBUG]   content: {content}")
+
+            # Update details with health check info
+            details = {
+                "checked_message_id": inner_id,
+                "original_message_id": original_message_id,
+                "health_stream": health_stream,
+            }
+
+            if content and isinstance(content, dict):
+                details.update({"check_content": content})
+                print(f"[HEALTH DEBUG] Added check_content to details")
+        else:
+            print(f"[HEALTH DEBUG] No 'data' field found in message_data")
+
     except (json.JSONDecodeError, KeyError) as e:
+        print(f"[HEALTH DEBUG] ERROR parsing message data: {e}")
         logger.warning(f"Could not parse health check message data: {e}")
+        health_status = "error"
+        health_message = f"Failed to parse health check message data: {e}"
+        details = {"error": str(e)}
+
+    # Construct the health response using V1ProcessorHealthResponse
+    health_response = V1ProcessorHealthResponse(
+        status=health_status, message=health_message, details=details
+    )
 
-    # You could add more logic here, e.g., update an internal health status,
-    # send a response, perform actual health checks, etc.
+    print(
+        f"[HEALTH DEBUG] Constructed V1ProcessorHealthResponse: {health_response.model_dump_json()}"
+    )
+    logger.info(
+        f"Health response for message {message_id}: {health_response.model_dump_json()}"
+    )
+
+    # Send the response to the return stream in the format expected by the system
+    if return_stream:
+        print(
+            f"[HEALTH DEBUG] Sending V1ProcessorHealthResponse to return_stream: {return_stream}"
+        )
+        try:
+            # Send the V1ProcessorHealthResponse directly (not wrapped in StreamResponseMessage)
+            response_data = health_response.model_dump()
+
+            print(
+                f"[HEALTH DEBUG] Sending V1ProcessorHealthResponse directly: {json.dumps(response_data, indent=2)}"
+            )
+
+            # Send to return stream
+            redis_conn.xadd(
+                return_stream,
+                {"data": json.dumps(response_data)},
+                maxlen=1000,
+                approximate=True,
+            )
+            print(
+                f"[HEALTH DEBUG] V1ProcessorHealthResponse sent successfully to {return_stream}"
+            )
+            logger.info(
+                f"Sent health response for {message_id} to stream: {return_stream}"
+            )
+        except Exception as e_resp_send:
+            print(f"[HEALTH DEBUG] ERROR sending response: {e_resp_send}")
+            logger.error(
+                f"Failed to send health response for {message_id} to stream {return_stream}: {e_resp_send}"
+            )
+    else:
+        print(f"[HEALTH DEBUG] No return_stream specified, not sending response")
 
     # Acknowledge the health check message
+    print(f"[HEALTH DEBUG] Acknowledging message {message_id}")
     try:
         redis_conn.xack(health_stream, health_group, message_id)
+        print(f"[HEALTH DEBUG] Message {message_id} acknowledged successfully")
         logger.info(f"Acknowledged health check message {message_id}")
     except Exception as e_ack:
+        print(f"[HEALTH DEBUG] ERROR acknowledging message: {e_ack}")
         logger.error(
             f"Failed to acknowledge health check message {message_id}: {e_ack}"
         )
 
+    print(f"[HEALTH DEBUG] === FINISHED PROCESSING HEALTH CHECK MESSAGE ===")
+
 
 def main():
     """Main function for the health check consumer subprocess."""
+    print(f"[HEALTH DEBUG] === HEALTH WORKER STARTING ===")
     logger = setup_health_logging()
 
     # Get environment variables
@@ -80,7 +183,13 @@ def main():
     health_stream = os.environ.get("REDIS_HEALTH_STREAM")
     health_group = os.environ.get("REDIS_HEALTH_CONSUMER_GROUP")
 
+    print(f"[HEALTH DEBUG] Environment variables:")
+    print(f"[HEALTH DEBUG]   REDIS_URL: {redis_url}")
+    print(f"[HEALTH DEBUG]   REDIS_HEALTH_STREAM: {health_stream}")
+    print(f"[HEALTH DEBUG]   REDIS_HEALTH_CONSUMER_GROUP: {health_group}")
+
     if not all([redis_url, health_stream, health_group]):
+        print(f"[HEALTH DEBUG] ERROR: Missing required environment variables")
         logger.error(
             "Missing required environment variables: REDIS_URL, REDIS_HEALTH_STREAM, REDIS_HEALTH_CONSUMER_GROUP"
         )
@@ -91,43 +200,64 @@ def main():
     assert isinstance(health_stream, str)
     assert isinstance(health_group, str)
 
+    print(
+        f"[HEALTH DEBUG] Starting health consumer for stream: {health_stream}, group: {health_group}"
+    )
     logger.info(
         f"Starting health consumer for stream: {health_stream}, group: {health_group}"
     )
 
     # Configure SOCKS proxy
+    print(f"[HEALTH DEBUG] Configuring SOCKS proxy...")
     socks.set_default_proxy(socks.SOCKS5, "localhost", 1055)
     socket.socket = socks.socksocket
     logger.info("Configured SOCKS5 proxy for socket connections via localhost:1055")
 
     health_redis_conn: Optional[redis.Redis] = None
     health_consumer_name = f"health-consumer-{os.getpid()}-{socket.gethostname()}"
+    print(f"[HEALTH DEBUG] Health consumer name: {health_consumer_name}")
 
+    print(f"[HEALTH DEBUG] === ENTERING MAIN LOOP ===")
     while True:
         try:
             if health_redis_conn is None:
+                print(f"[HEALTH DEBUG] Connecting to Redis for health stream...")
                 logger.info("Connecting to Redis for health stream...")
                 health_redis_conn = redis.from_url(redis_url, decode_responses=True)
                 health_redis_conn.ping()
+                print(f"[HEALTH DEBUG] Connected to Redis successfully")
                 logger.info("Connected to Redis for health stream.")
 
                 # Create health consumer group if it doesn't exist
+                print(f"[HEALTH DEBUG] Creating/checking consumer group...")
                 try:
                     health_redis_conn.xgroup_create(
                         health_stream, health_group, id="0", mkstream=True
                     )
+                    print(
+                        f"[HEALTH DEBUG] Created consumer group {health_group} for stream {health_stream}"
+                    )
                     logger.info(
                         f"Created consumer group {health_group} for stream {health_stream}"
                     )
                 except ResponseError as e_group:
                     if "BUSYGROUP" in str(e_group):
+                        print(
+                            f"[HEALTH DEBUG] Consumer group {health_group} already exists"
+                        )
                         logger.info(f"Consumer group {health_group} already exists.")
                     else:
+                        print(
+                            f"[HEALTH DEBUG] ERROR creating health consumer group: {e_group}"
+                        )
                         logger.error(f"Error creating health consumer group: {e_group}")
                         time.sleep(5)
                         health_redis_conn = None
                         continue
                 except Exception as e_group_other:
+                    print(
+                        f"[HEALTH DEBUG] UNEXPECTED ERROR creating health consumer group: {e_group_other}"
+                    )
                     logger.error(
                         f"Unexpected error creating health consumer group: {e_group_other}"
                     )
@@ -138,6 +268,7 @@ def main():
             # Read from health stream
             assert health_redis_conn is not None
 
+            print(f"[HEALTH DEBUG] Reading from health stream {health_stream}...")
             health_streams_arg: Dict[str, object] = {health_stream: ">"}
             raw_messages = health_redis_conn.xreadgroup(
                 health_group,
@@ -147,13 +278,21 @@ def main():
                 block=5000,  # Block for 5 seconds
             )
 
+            print(f"[HEALTH DEBUG] xreadgroup returned: {raw_messages}")
+            print(f"[HEALTH DEBUG] Messages type: {type(raw_messages)}")
+
             if raw_messages:
+                print(f"[HEALTH DEBUG] Found messages to process")
                 # Cast to expected type for decode_responses=True
                 messages = cast(
                     List[Tuple[str, List[Tuple[str, Dict[str, str]]]]], raw_messages
                 )
-                for _stream_name, stream_messages in messages:
+                for stream_name, stream_messages in messages:
+                    print(
+                        f"[HEALTH DEBUG] Processing stream: {stream_name} with {len(stream_messages)} message(s)"
+                    )
                     for message_id, message_data in stream_messages:
+                        print(f"[HEALTH DEBUG] Processing message {message_id}")
                         process_health_check_message(
                             message_id,
                             message_data,
@@ -162,6 +301,8 @@ def main():
                             health_stream,
                             health_group,
                         )
+            else:
+                print(f"[HEALTH DEBUG] No messages received (timeout)")
 
         except (ConnectionError, RedisTimeoutError, TimeoutError) as e_conn:
             logger.error(f"Redis connection error: {e_conn}. Reconnecting in 5s...")

{nebu-0.1.117 → nebu-0.1.119}/src/nebu/processors/models.py

@@ -149,3 +149,9 @@ class V1OpenAIStreamResponse(BaseModel):
     content: Any  # Using Any for ChatCompletionResponse
     created_at: int
     user_id: Optional[str] = None
+
+
+class V1ProcessorHealthResponse(BaseModel):
+    status: str
+    message: Optional[str] = None
+    details: Optional[Any] = None

{nebu-0.1.117 → nebu-0.1.119}/src/nebu/processors/processor.py

@@ -1,7 +1,8 @@
 import json
 import threading
 import time
-import uuid
+
+# import uuid # Removed unused import
 from typing import (
     Any,
     Dict,
@@ -23,6 +24,7 @@ from nebu.meta import V1ResourceMetaRequest, V1ResourceReference
 from nebu.processors.models import (
     V1ContainerRequest,
     V1Processor,
+    V1ProcessorHealthResponse,
     V1ProcessorRequest,
     V1Processors,
     V1ProcessorScaleRequest,
@@ -246,7 +248,7 @@ class Processor(Generic[InputType, OutputType]):
 
         # --- Wait for health check if requested ---
         if wait_for_healthy:
-            self.wait_for_health_check()
+            self.wait_for_healthy()
 
     def __call__(
         self,
@@ -559,11 +561,11 @@ class Processor(Generic[InputType, OutputType]):
         else:
             logger.info(f"No active log stream to stop for {self.name}.")
 
-    def wait_for_health_check(
+    def wait_for_healthy(
         self, timeout: float = 3600.0, retry_interval: float = 5.0
     ) -> None:
         """
-        Wait for the processor to respond to health checks.
+        Wait for the processor to respond to health checks using the health endpoint.
 
         Args:
             timeout: Maximum time to wait for health check in seconds
@@ -573,40 +575,25 @@ class Processor(Generic[InputType, OutputType]):
             raise ValueError("Processor not found, cannot perform health check")
 
         logger.info(
-            f"Waiting for processor {self.processor.metadata.name} to be healthy..."
+            f"Waiting for processor {self.processor.metadata.name} to be healthy via health endpoint..."
         )
 
         start_time = time.time()
         while time.time() - start_time < timeout:
             try:
-                # Create a health check message
-                health_check_data = {
-                    "kind": "HealthCheck",
-                    "id": str(uuid.uuid4()),
-                    "content": {},
-                    "created_at": time.time(),
-                }
-
-                # Send health check and wait for response
-                response = self.send(
-                    data=health_check_data,  # type: ignore[arg-type]
-                    wait=True,
-                    timeout=30.0,  # Short timeout for individual health check
+                health_response = self.health()  # Use the new health() method
+                logger.info(
+                    f">>> Health check response: {health_response.model_dump_json()}"
                 )
-                logger.info(f">>> Health check response: {response}")
 
                 # Check if the response indicates health
-                if response and isinstance(response, dict):
-                    status = response.get("status")
-                    if status == "healthy":
-                        logger.info(
-                            f"Processor {self.processor.metadata.name} is healthy!"
-                        )
-                        return
-
-                logger.info(
-                    f"Health check attempt failed, retrying in {retry_interval}s..."
-                )
+                if health_response.status == "ok":  # Check for "ok" status
+                    logger.info(f"Processor {self.processor.metadata.name} is healthy!")
+                    return
+                else:
+                    logger.info(
+                        f"Processor {self.processor.metadata.name} reported status: {health_response.status}. Retrying in {retry_interval}s..."
+                    )
 
             except Exception as e:
                 logger.info(
@@ -619,3 +606,39 @@ class Processor(Generic[InputType, OutputType]):
         raise TimeoutError(
             f"Processor {self.processor.metadata.name} failed to become healthy within {timeout} seconds"
         )
+
+    def health(self) -> V1ProcessorHealthResponse:
+        """
+        Performs a health check on the processor by calling the health endpoint.
+        """
+        if (
+            not self.processor
+            or not self.processor.metadata.name
+            or not self.processor.metadata.namespace
+        ):
+            raise ValueError(
+                "Processor not found or missing metadata (name/namespace), cannot perform health check."
+            )
+
+        health_url = f"{self.orign_host}/v1/processors/{self.processor.metadata.namespace}/{self.processor.metadata.name}/health"
+        logger.debug(f"Calling health check endpoint: {health_url}")
+
+        try:
+            response = requests.get(
+                health_url,
+                headers={"Authorization": f"Bearer {self.api_key}"},
+                timeout=30.0,  # Standard timeout for a health check
+            )
+            response.raise_for_status()  # Raise an exception for HTTP errors
+            health_response_data = response.json()
+            return V1ProcessorHealthResponse.model_validate(health_response_data)
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Health check request to {health_url} failed: {e}")
+            # Optionally, return a V1ProcessorHealthResponse indicating an error
+            # For now, re-raising the exception or a custom one might be better
+            raise RuntimeError(f"Failed to get health status: {e}") from e
+        except Exception as e:
+            logger.error(f"An unexpected error occurred during health check: {e}")
+            raise RuntimeError(
+                f"Unexpected error during health status retrieval: {e}"
+            ) from e

{nebu-0.1.117 → nebu-0.1.119/src/nebu.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nebu
-Version: 0.1.117
+Version: 0.1.119
 Summary: A globally distributed container runtime
 Requires-Python: >=3.10.14
 Description-Content-Type: text/markdown