nebu 0.1.76__py3-none-any.whl → 0.1.78__py3-none-any.whl

nebu/errors.py

@@ -0,0 +1,5 @@
+# Add this near the top with other imports/definitions
+class RetriableError(Exception):
+    """Custom exception for failures during Training resource setup."""
+
+    pass

nebu/processors/consumer.py

@@ -5,16 +5,19 @@ import os
 import socket
 import subprocess
 import sys
+import threading
 import time
 import traceback
 import types
 from datetime import datetime, timezone
-from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast
 
 import redis
 import socks
 from redis import ConnectionError, ResponseError
 
+from nebu.errors import RetriableError
+
 # Define TypeVar for generic models
 T = TypeVar("T")
 
@@ -351,9 +354,22 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
     # --- Subprocess Execution Path ---
     if execution_mode == "subprocess":
         print(f"Processing message {message_id} in subprocess...")
+        process = None  # Initialize process variable
+
+        # Helper function to read and print stream lines
+        def stream_reader(stream: IO[str], prefix: str):
+            try:
+                for line in iter(stream.readline, ""):
+                    print(f"{prefix}: {line.strip()}", flush=True)
+            except Exception as e:
+                print(f"Error reading stream {prefix}: {e}")
+            finally:
+                stream.close()
+
         try:
             worker_cmd = [
                 sys.executable,
+                "-u",  # Force unbuffered stdout/stderr in the subprocess
                 "-m",
                 "nebu.processors.consumer_process_worker",
             ]
@@ -361,52 +377,116 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                 {"message_id": message_id, "message_data": message_data}
             )
 
-            # Run the worker script as a module
-            # Inherit environment variables automatically
-            # Pass message data via stdin
-            result = subprocess.run(
+            # Start the worker process
+            process = subprocess.Popen(
                 worker_cmd,
-                input=process_input,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
                 text=True,
-                capture_output=True,
-                check=True,  # Raise CalledProcessError on non-zero exit
-                env=os.environ.copy(),  # Ensure environment is passed
+                encoding="utf-8",
+                bufsize=1,  # Line buffered
+                env=os.environ.copy(),
             )
-            print(f"Subprocess for {message_id} completed successfully.")
-            if result.stdout:
-                print(f"Subprocess stdout:\n{result.stdout}")
-            # Assume the subprocess handled response sending and acknowledgement
 
-        except subprocess.CalledProcessError as e:
-            print(
-                f"Subprocess for message {message_id} failed with exit code {e.returncode}."
+            # Create threads to read stdout and stderr concurrently
+            stdout_thread = threading.Thread(
+                target=stream_reader,
+                args=(process.stdout, f"[Subprocess STDOUT {message_id[:8]}]"),
+            )
+            stderr_thread = threading.Thread(
+                target=stream_reader,
+                args=(process.stderr, f"[Subprocess STDERR {message_id[:8]}]"),
             )
-            if e.stdout:
-                print(f"Subprocess stdout:\n{e.stdout}")
-            if e.stderr:
-                print(f"Subprocess stderr:\n{e.stderr}")
 
-            # Send a generic error message back, as the subprocess likely failed
-            # before it could send its specific error.
+            stdout_thread.start()
+            stderr_thread.start()
+
+            # Send input data to the subprocess
+            # Ensure process and stdin are valid before writing/closing
+            if process and process.stdin:
+                try:
+                    process.stdin.write(process_input)
+                    process.stdin.close()  # Signal end of input
+                except (BrokenPipeError, OSError) as e:
+                    # Handle cases where the process might have exited early
+                    print(
+                        f"Warning: Failed to write full input to subprocess {message_id}: {e}. It might have exited prematurely."
+                    )
+                    # Continue to wait and check return code
+            else:
+                print(
+                    f"Error: Subprocess stdin stream not available for {message_id}. Cannot send input."
+                )
+                # Handle this case - perhaps terminate and report error?
+                # For now, we'll let it proceed to wait() which will likely show an error code.
+
+            # Wait for the process to finish
+            return_code = (
+                process.wait() if process else -1
+            )  # Handle case where process is None
+
+            # Wait for reader threads to finish consuming remaining output
+            stdout_thread.join()
+            stderr_thread.join()
+
+            if return_code == 0:
+                print(
+                    f"Subprocess for {message_id} completed successfully (return code 0)."
+                )
+                # Assume success handling (ack/response) was done by the worker
+            elif return_code == 3:
+                print(
+                    f"Subprocess for {message_id} reported a retriable error (exit code 3). Message will not be acknowledged."
+                )
+                # Optionally send an error response here, though the worker already did.
+                # _send_error_response(...)
+                # DO NOT Acknowledge the message here, let it be retried.
+            else:
+                print(
+                    f"Subprocess for {message_id} failed with exit code {return_code}."
+                )
+                # Worker likely failed, send generic error and ACK here
+                _send_error_response(
+                    message_id,
+                    f"Subprocess execution failed with exit code {return_code}",
+                    "See consumer logs for subprocess stderr.",  # stderr was already printed
+                    message_data.get("return_stream"),
+                    message_data.get("user_id"),
+                )
+                # CRITICAL: Acknowledge the message here since the subprocess failed
+                try:
+                    assert isinstance(REDIS_STREAM, str)
+                    assert isinstance(REDIS_CONSUMER_GROUP, str)
+                    r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
+                    print(f"Acknowledged failed subprocess message {message_id}")
+                except Exception as e_ack:
+                    print(
+                        f"CRITICAL: Failed to acknowledge failed subprocess message {message_id}: {e_ack}"
+                    )
+
+        except FileNotFoundError:
+            print(
+                "FATAL: Worker script 'nebu.processors.consumer_process_worker' not found. Check PYTHONPATH."
+            )
+            # Send error and ack if possible
             _send_error_response(
                 message_id,
-                f"Subprocess execution failed with exit code {e.returncode}",
-                e.stderr or "No stderr captured",
-                message_data.get(
-                    "return_stream"
-                ),  # Try to get return stream from original data
-                message_data.get("user_id"),  # Try to get user_id from original data
+                "Worker script not found",
+                traceback.format_exc(),
+                message_data.get("return_stream"),
+                message_data.get("user_id"),
             )
-
-            # CRITICAL: Acknowledge the message here since the subprocess failed
             try:
                 assert isinstance(REDIS_STREAM, str)
                 assert isinstance(REDIS_CONSUMER_GROUP, str)
                 r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
-                print(f"Acknowledged failed subprocess message {message_id}")
+                print(
+                    f"Acknowledged message {message_id} after worker script not found failure"
+                )
             except Exception as e_ack:
                 print(
-                    f"CRITICAL: Failed to acknowledge failed subprocess message {message_id}: {e_ack}"
+                    f"CRITICAL: Failed to acknowledge message {message_id} after worker script not found failure: {e_ack}"
                 )
 
         except Exception as e:
@@ -417,7 +497,7 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
             # Also send an error and acknowledge
             _send_error_response(
                 message_id,
-                f"Failed to launch subprocess: {e}",
+                f"Failed to launch/manage subprocess: {e}",
                 traceback.format_exc(),
                 message_data.get("return_stream"),
                 message_data.get("user_id"),
@@ -427,12 +507,44 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                 assert isinstance(REDIS_CONSUMER_GROUP, str)
                 r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
                 print(
-                    f"Acknowledged message {message_id} after subprocess launch failure"
+                    f"Acknowledged message {message_id} after subprocess launch/manage failure"
                 )
             except Exception as e_ack:
                 print(
-                    f"CRITICAL: Failed to acknowledge message {message_id} after subprocess launch failure: {e_ack}"
+                    f"CRITICAL: Failed to acknowledge message {message_id} after subprocess launch/manage failure: {e_ack}"
+                )
+            # Ensure process is terminated if it's still running after an error
+            if process and process.poll() is None:
+                print(
+                    f"Terminating potentially lingering subprocess for {message_id}..."
                 )
+                process.terminate()
+                process.wait(timeout=5)  # Give it a moment to terminate
+                if process.poll() is None:
+                    print(
+                        f"Subprocess for {message_id} did not terminate gracefully, killing."
+                    )
+                    process.kill()
+        finally:
+            # Ensure streams are closed even if threads failed or process is None
+            if process:
+                if process.stdout:
+                    try:
+                        process.stdout.close()
+                    except Exception:
+                        pass  # Ignore errors during cleanup close
+                if process.stderr:
+                    try:
+                        process.stderr.close()
+                    except Exception:
+                        pass  # Ignore errors during cleanup close
+                # Stdin should already be closed, but doesn't hurt to be safe
+                if process.stdin and not process.stdin.closed:
+                    try:
+                        process.stdin.close()
+                    except Exception:
+                        pass
+
         return  # Exit process_message after handling subprocess logic
 
     # --- Inline Execution Path (Original Logic) ---
@@ -447,6 +559,16 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
             None,
             None,
         )  # Pass None for user_id if unavailable here
+        # Acknowledge message with code load failure to prevent reprocessing loop
+        try:
+            assert isinstance(REDIS_STREAM, str)
+            assert isinstance(REDIS_CONSUMER_GROUP, str)
+            r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
+            print(f"Acknowledged message {message_id} due to code load failure.")
+        except Exception as e_ack:
+            print(
+                f"CRITICAL: Failed to acknowledge message {message_id} after code load failure: {e_ack}"
+            )
         return  # Skip processing
 
     return_stream = None
@@ -694,6 +816,15 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         assert isinstance(REDIS_CONSUMER_GROUP, str)
         r.xack(REDIS_STREAM, REDIS_CONSUMER_GROUP, message_id)
 
+    except RetriableError as e:
+        print(f"Retriable error processing message {message_id}: {e}")
+        traceback.print_exc()
+        _send_error_response(
+            message_id, str(e), traceback.format_exc(), return_stream, user_id
+        )
+        # DO NOT Acknowledge the message for retriable errors
+        print(f"Message {message_id} will be retried later.")
+
     except Exception as e:
         print(f"Error processing message {message_id}: {e}")
         traceback.print_exc()

nebu/processors/consumer_process_worker.py

@@ -14,6 +14,8 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, TypeVar, cast
 import redis
 import socks
 
+from nebu.errors import RetriableError
+
 # from redis import ConnectionError, ResponseError # Removed unused imports
 
 # Define TypeVar for generic models
@@ -640,8 +642,24 @@ if __name__ == "__main__":
             print("[Worker] Exiting with status 0.")
             sys.exit(0)
 
+        except RetriableError as e:
+            # --- Handle Retriable Processing Error ---
+            print(f"[Worker] Retriable error processing message {message_id}: {e}")
+            tb = traceback.format_exc()
+            print(tb)
+            # Assert message_id is str before sending error
+            assert isinstance(message_id, str)
+            # Send error response (optional, consider suppressing later if too noisy)
+            _send_error_response(message_id, str(e), tb, return_stream, user_id)
+
+            # DO NOT Acknowledge the message for retriable errors
+
+            # --- 9. Exit with specific code for retriable failure ---
+            print("[Worker] Exiting with status 3 due to retriable error.")
+            sys.exit(3)
+
         except Exception as e:
-            # --- Handle Processing Error ---
+            # --- Handle Non-Retriable Processing Error ---
             print(f"[Worker] Error processing message {message_id}: {e}")
             tb = traceback.format_exc()
             print(tb)

{nebu-0.1.76.dist-info → nebu-0.1.78.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nebu
-Version: 0.1.76
+Version: 0.1.78
 Summary: A globally distributed container runtime
 Requires-Python: >=3.10.14
 Description-Content-Type: text/markdown

{nebu-0.1.76.dist-info → nebu-0.1.78.dist-info}/RECORD

@@ -3,6 +3,7 @@ nebu/auth.py,sha256=N_v6SPFD9HU_UoRDTaouH03g2Hmo9C-xxqInE1FweXE,1471
 nebu/cache.py,sha256=jmluqvWnE9N8uNq6nppXSxEJK7DKWaB79GicaGg9KmY,4718
 nebu/config.py,sha256=C5Jt9Bd0i0HrgzBSVNJ-Ml3KwX_gaYbYYZEtNL2gvJg,7031
 nebu/data.py,sha256=X0aAJYuHNVcTCRHpIDDm546HwMqIZpv40lGrozlL41A,39797
+nebu/errors.py,sha256=bBnK5YQ6qZg4OMY81AN2k03ppefg89FUwF_SHEMlqCA,170
 nebu/meta.py,sha256=CzFHMND9seuewzq9zNNx9WTr6JvrCBExe7BLqDSr7lM,745
 nebu/orign.py,sha256=SkVfHgpadwik58KCZCrjdV5EHY0dhpEhDvijzLxY11Y,2052
 nebu/builders/builder.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -13,8 +14,8 @@ nebu/containers/models.py,sha256=0j6NGy4yto-enRDh_4JH_ZTbHrLdSpuMOqNQPnIrwC4,681
 nebu/containers/server.py,sha256=yFa2Y9PzBn59E1HftKiv0iapPonli2rbGAiU6r-wwe0,2513
 nebu/namespaces/models.py,sha256=EqUOpzhVBhvJw2P92ONDUbIgC31M9jMmcaG5vyOrsWg,497
 nebu/namespaces/namespace.py,sha256=Q_EDH7BgQrTkaDh_l4tbo22qpq-uARfIk8ZPBLjITGY,4967
-nebu/processors/consumer.py,sha256=Fxgl0cuqKpX3UMsig_aw5KBbg1blE4xUY4yNnIvIHuw,37806
-nebu/processors/consumer_process_worker.py,sha256=l5_BSMfqy-n2yK_UC3sm_pimzelaASeMdPxRE97HFwc,30959
+nebu/processors/consumer.py,sha256=nWJmlTwJxfadMEQQwzwOJNPCAkcxqIzOpshMMbnfa-o,43617
+nebu/processors/consumer_process_worker.py,sha256=tF5KU3Rnmzfc3Y0cM8J5nwGg1cJMe-ry0FmMSgGvXrY,31765
 nebu/processors/decorate.py,sha256=mu1o05BjNcbJ4M1so4Xvt7UbslX--B4dsYLgs5h8bEg,54610
 nebu/processors/default.py,sha256=W4slJenG59rvyTlJ7gRp58eFfXcNOTT2Hfi6zzJAobI,365
 nebu/processors/models.py,sha256=y40HoW-MEzDWB2dm_tsYlUy3Nf3s6eiLC0iGO9BoNog,3956
@@ -22,8 +23,8 @@ nebu/processors/processor.py,sha256=OgEK8Fz0ehSe_VFiNsxweVKZIckhgVvQQ11NNffYZqA,
 nebu/processors/remote.py,sha256=TeAIPGEMqnDIb7H1iett26IEZrBlcbPB_-DSm6jcH1E,1285
 nebu/redis/models.py,sha256=coPovAcVXnOU1Xh_fpJL4PO3QctgK9nBe5QYoqEcnxg,1230
 nebu/services/service.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nebu-0.1.76.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nebu-0.1.76.dist-info/METADATA,sha256=j130KtOVQeqJRkv6xgC6CJ0BUSmqUmb8jFyhKAC11jE,1731
-nebu-0.1.76.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
-nebu-0.1.76.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
-nebu-0.1.76.dist-info/RECORD,,
+nebu-0.1.78.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nebu-0.1.78.dist-info/METADATA,sha256=5qScVLcCucIHRKQgNam4OlHIKzDvh6qAKP_ompV6UY4,1731
+nebu-0.1.78.dist-info/WHEEL,sha256=ck4Vq1_RXyvS4Jt6SI0Vz6fyVs4GWg7AINwpsaGEgPE,91
+nebu-0.1.78.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
+nebu-0.1.78.dist-info/RECORD,,