nebu 0.1.123__py3-none-any.whl → 0.1.125__py3-none-any.whl

nebu/processors/processor.py

@@ -10,6 +10,7 @@ from typing import (
     List,
     Optional,
     TypeVar,
+    Union,
     cast,
     get_args,
     get_origin,
@@ -257,7 +258,7 @@ class Processor(Generic[InputType, OutputType]):
         logs: bool = False,
         api_key: Optional[str] = None,
         user_key: Optional[str] = None,
-        timeout: Optional[float] = 600.0,
+        timeout: Optional[float] = 3600,
     ) -> OutputType | Dict[str, Any] | None:
         """
         Allows the Processor instance to be called like a function, sending data.
@@ -278,12 +279,21 @@ class Processor(Generic[InputType, OutputType]):
         logs: bool = False,
         api_key: Optional[str] = None,
         user_key: Optional[str] = None,
-        timeout: Optional[float] = 600.0,
+        timeout: Optional[float] = 3600,
+        poll: bool = False,
+        poll_interval_seconds: float = 2.0,
     ) -> OutputType | Dict[str, Any] | None:
         """
-        Send data to the processor and optionally stream logs in the background.
+        Send data to the processor.
+        If wait=True, the request to the /messages endpoint waits for the processing to complete.
+        If wait=False and poll=True, sends the message and then polls the /return/:message_id endpoint for the result.
+        If wait=False and poll=False, sends the message and returns the initial response (e.g., an acknowledgement).
+        Optionally streams logs in the background if logs=True.
         """
-        print("sending data to processor: ", data)
+        logger.debug(
+            f"Sending data to processor {self.name}: {data}, wait={wait}, poll={poll}, logs={logs}"
+        )
+
         if (
             not self.processor
             or not self.processor.metadata.name
@@ -294,35 +304,168 @@ class Processor(Generic[InputType, OutputType]):
         processor_name = self.processor.metadata.name
         processor_namespace = self.processor.metadata.namespace
 
-        if not api_key:
-            api_key = self.api_key
+        # Determine the API key to use for this send operation
+        current_op_api_key = api_key if api_key is not None else self.api_key
+        if not current_op_api_key:
+            logger.error(
+                f"Processor {processor_name}: API key is missing for the send operation."
+            )
+            raise ValueError("API key not available for sending message.")
 
-        # --- Send Data ---
+        # --- Send Initial Message ---
         messages_url = (
             f"{self.processors_url}/{processor_namespace}/{processor_name}/messages"
         )
+
+        # The 'wait' parameter for V1StreamData dictates if the /messages endpoint itself should block.
+        stream_data_wait_param = wait
+
         stream_data = V1StreamData(
             content=data,
-            wait=wait,
+            wait=stream_data_wait_param,
             user_key=user_key,
         )
+
+        # Timeout for the initial POST request.
+        # If stream_data_wait_param is True, use the overall timeout.
+        # Otherwise (quick ack expected), use a shorter fixed timeout.
+        initial_request_timeout = timeout if stream_data_wait_param else 30.0
+
+        logger.debug(
+            f"Processor {processor_name}: Posting to {messages_url} with stream_data_wait={stream_data_wait_param}, initial_timeout={initial_request_timeout}"
+        )
         response = requests.post(
             messages_url,
             json=stream_data.model_dump(mode="json", exclude_none=True),
-            headers={"Authorization": f"Bearer {api_key}"},
-            timeout=timeout,
+            headers={"Authorization": f"Bearer {current_op_api_key}"},
+            timeout=initial_request_timeout,
         )
         response.raise_for_status()
         raw_response_json = response.json()
-
-        print(f">>> Raw response JSON: {raw_response_json}")
+        logger.debug(
+            f"Processor {processor_name}: Initial response JSON: {raw_response_json}"
+        )
 
         if "error" in raw_response_json:
-            print("error in raw_response_json")
+            logger.error(
+                f"Processor {processor_name}: Error in initial response: {raw_response_json['error']}"
+            )
             raise Exception(raw_response_json["error"])
 
-        raw_content = raw_response_json.get("content")
-        print(f">>> Raw content: {raw_content}")
+        # Initialize raw_content. This will hold the final data payload.
+        raw_content: Optional[Union[Dict[str, Any], List[Any], str]] = None
+
+        # --- Handle Response: Polling or Direct Content ---
+        # Poll only if poll=True AND the initial request was configured not to wait (wait=False).
+        if poll and not stream_data_wait_param:
+            message_id = raw_response_json.get("message_id")
+            if not message_id or not isinstance(message_id, str):
+                logger.error(
+                    f"Processor {processor_name}: Polling requested but 'message_id' (string) not found in initial response. Response: {raw_response_json}"
+                )
+                raise ValueError(
+                    "Polling failed: 'message_id' (string) missing or invalid in initial server response."
+                )
+
+            # Polling URL using self.orign_host for consistency
+            polling_url = f"{self.orign_host}/v1/processors/{processor_namespace}/{processor_name}/return/{message_id}"
+
+            logger.info(
+                f"Processor {processor_name}: Polling for message_id {message_id} at {polling_url}. Overall timeout: {timeout}s, Interval: {poll_interval_seconds}s."
+            )
+            polling_start_time = time.time()
+
+            while True:
+                current_time = time.time()
+                if (
+                    timeout is not None
+                    and (current_time - polling_start_time) > timeout
+                ):
+                    logger.warning(
+                        f"Processor {processor_name}: Polling for message_id {message_id} timed out after {timeout} seconds."
+                    )
+                    raise TimeoutError(
+                        f"Polling for message_id {message_id} timed out after {timeout} seconds."
+                    )
+
+                individual_poll_timeout = max(10.0, poll_interval_seconds * 2)
+                logger.debug(
+                    f"Processor {processor_name}: Making polling attempt for {message_id}, attempt timeout: {individual_poll_timeout}s"
+                )
+                try:
+                    poll_response = requests.post(
+                        polling_url,
+                        headers={"Authorization": f"Bearer {current_op_api_key}"},
+                        timeout=individual_poll_timeout,
+                        json={},  # Send an empty JSON body for POST
+                    )
+
+                    if poll_response.status_code == 200:
+                        logger.info(
+                            f"Processor {processor_name}: Successfully retrieved message {message_id} via polling. Status: 200."
+                        )
+                        try:
+                            polled_data = poll_response.json()
+                            if isinstance(polled_data, (dict, list, str)):
+                                raw_content = polled_data
+                            else:
+                                logger.warning(
+                                    f"Processor {processor_name}: Polled data for {message_id} is of unexpected type: {type(polled_data)}. Content: {polled_data}"
+                                )
+                                raw_content = polled_data
+                        except json.JSONDecodeError:
+                            logger.error(
+                                f"Processor {processor_name}: Failed to decode JSON from polling response for {message_id}. Response text: {poll_response.text[:200]}..."
+                            )
+                            raise ValueError(
+                                f"Polling for {message_id} returned non-JSON response with status 200."
+                            )
+                        break  # Exit polling loop
+
+                    elif poll_response.status_code == 404:
+                        logger.debug(
+                            f"Processor {processor_name}: Message {message_id} not yet ready (404). Retrying in {poll_interval_seconds}s..."
+                        )
+                    elif poll_response.status_code == 202:
+                        logger.debug(
+                            f"Processor {processor_name}: Message {message_id} processing (202). Retrying in {poll_interval_seconds}s..."
+                        )
+                    else:
+                        logger.error(
+                            f"Processor {processor_name}: Polling for message_id {message_id} received unexpected status {poll_response.status_code}. Response: {poll_response.text[:500]}"
+                        )
+                        poll_response.raise_for_status()
+
+                except requests.exceptions.Timeout:
+                    logger.warning(
+                        f"Processor {processor_name}: Polling request for message_id {message_id} timed out. Retrying if overall timeout not exceeded..."
+                    )
+                except requests.exceptions.HTTPError as e:
+                    logger.error(
+                        f"Processor {processor_name}: Polling for message_id {message_id} failed with HTTPError: {e}. Response: {e.response.text[:500] if e.response else 'No response text'}"
+                    )
+                    raise
+                except requests.exceptions.RequestException as e:
+                    logger.error(
+                        f"Processor {processor_name}: Polling for message_id {message_id} failed with RequestException: {e}"
+                    )
+                    raise
+
+                if timeout is not None and (time.time() - polling_start_time) > timeout:
+                    logger.warning(
+                        f"Processor {processor_name}: Polling for {message_id} timed out after {timeout}s (checked before sleep)."
+                    )
+                    raise TimeoutError(
+                        f"Polling for message_id {message_id} timed out after {timeout} seconds."
+                    )
+
+                time.sleep(poll_interval_seconds)
+        else:
+            # Handles: wait=True (polling skipped, server waited) OR (wait=False AND poll=False) (fire-and-forget)
+            raw_content = raw_response_json.get("content")
+            logger.debug(
+                f"Processor {processor_name}: Not polling. Raw content from initial response: {str(raw_content)[:200]}..."
+            )
 
         # --- Fetch Logs (if requested and not already running) ---
         if logs:
@@ -332,7 +475,11 @@ class Processor(Generic[InputType, OutputType]):
                 )
                 self._log_thread = threading.Thread(
                     target=_fetch_and_print_logs,
-                    args=(log_url, self.api_key, processor_name),  # Pass processor_name
+                    args=(
+                        log_url,
+                        self.api_key,
+                        processor_name,
+                    ),  # Use self.api_key for logs
                     daemon=True,
                 )
                 try:
@@ -348,41 +495,60 @@ class Processor(Generic[InputType, OutputType]):
             else:
                 logger.info(f"Log fetching is already running for {processor_name}.")
 
-        # Attempt to parse into OutputType if conditions are met
-        print(f">>> wait: {wait}")
-        print(f">>> self.output_model_cls: {self.output_model_cls}")
-        print(">>> type(self.output_model_cls): ", type(self.output_model_cls))
-        print(
-            f">>> isinstance(self.output_model_cls, type): {isinstance(self.output_model_cls, type)}"
+        # --- Attempt to parse into OutputType if conditions are met ---
+        logger.debug(
+            f"Processor {processor_name}: Attempting to parse result. output_model_cls: {self.output_model_cls}, raw_content type: {type(raw_content)}"
         )
-        print(f">>> isinstance(raw_content, dict): {isinstance(raw_content, dict)}")
+
+        # Attempt to parse if the operation was intended to yield full content (either by waiting or polling),
+        # and raw_content is a dictionary, and output_model_cls is a Pydantic model.
+        should_attempt_parse = (
+            wait or poll
+        )  # True if client expects full content back from this method call
+
         if (
-            wait
-            and self.output_model_cls
+            should_attempt_parse
+            and self.output_model_cls is not None
             and isinstance(self.output_model_cls, type)
-            and issubclass(self.output_model_cls, BaseModel)  # type: ignore
             and isinstance(raw_content, dict)
         ):
-            print(f">>> raw_content: {raw_content}")
+            logger.debug(
+                f"Processor {processor_name}: Valid conditions for parsing. Raw content (dict): {str(raw_content)[:200]}..."
+            )
             try:
                 parsed_model = self.output_model_cls.model_validate(raw_content)
-                print(f">>> parsed_model: {parsed_model}")
+                logger.debug(
+                    f"Processor {processor_name}: Successfully parsed to {self.output_model_cls.__name__}. Parsed model: {str(parsed_model)[:200]}..."
+                )
                 parsed_output: OutputType = cast(OutputType, parsed_model)
-                print(f">>> parsed_output: {parsed_output}")
                 return parsed_output
             except Exception as e:
-                print(f">>> error: {e}")
                 model_name = getattr(
                     self.output_model_cls, "__name__", str(self.output_model_cls)
                 )
                 logger.error(
                     f"Processor {processor_name}: Failed to parse 'content' field into output type {model_name}. "
-                    f"Error: {e}. Returning raw JSON response."
+                    f"Error: {e}. Raw content was: {str(raw_content)[:500]}. Returning raw content instead."
+                )
+                return raw_content  # type: ignore
+        else:
+            if (
+                not isinstance(raw_content, dict)
+                and should_attempt_parse
+                and self.output_model_cls
+            ):
+                logger.debug(
+                    f"Processor {processor_name}: Skipping Pydantic parsing because raw_content is not a dict (type: {type(raw_content)})."
+                )
+            elif not (should_attempt_parse and self.output_model_cls):
+                logger.debug(
+                    f"Processor {processor_name}: Skipping Pydantic parsing due to conditions not met (should_attempt_parse: {should_attempt_parse}, output_model_cls: {self.output_model_cls is not None})."
                 )
-                return raw_content
-        # Fallback logic using self.schema_ has been removed.
 
-        return raw_content
+        logger.debug(
+            f"Processor {processor_name}: Returning raw_content (type: {type(raw_content)}): {str(raw_content)[:200]}..."
+        )
+        return raw_content  # type: ignore
 
     def scale(self, replicas: int) -> Dict[str, Any]:
         """

{nebu-0.1.123.dist-info → nebu-0.1.125.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nebu
-Version: 0.1.123
+Version: 0.1.125
 Summary: A globally distributed container runtime
 Requires-Python: >=3.10.14
 Description-Content-Type: text/markdown

{nebu-0.1.123.dist-info → nebu-0.1.125.dist-info}/RECORD

@@ -19,11 +19,11 @@ nebu/processors/consumer_process_worker.py,sha256=h--eNFKaLbUayxn88mB8oGGdrU2liE
 nebu/processors/decorate.py,sha256=5p9pQrk_H8-Fj0UjsgSVCYx7Jk7KFuhMZtNhkKvpmkQ,61306
 nebu/processors/default.py,sha256=cy4ETMdbdRGkrvbYec1o60h7mGDlGN5JsuUph0ENtDU,364
 nebu/processors/models.py,sha256=8-TmKha2_QAnPlXcZxYjrCSPDCX7FFcMDMcHK77jK0U,4223
-nebu/processors/processor.py,sha256=wos5BkNZUVaY5ifT7yeDLbJkdfeloJ7hRuK9P18LU7U,26155
+nebu/processors/processor.py,sha256=KS9zSjSj9KJT8ktcdiLTfXTfh3VvNCl7H0t2jnpH9GY,35250
 nebu/redis/models.py,sha256=coPovAcVXnOU1Xh_fpJL4PO3QctgK9nBe5QYoqEcnxg,1230
 nebu/services/service.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nebu-0.1.123.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nebu-0.1.123.dist-info/METADATA,sha256=8L9L8vNoFhK8A4U-iWvXP9DIgeVXOW0MV324qcOhsV0,1798
-nebu-0.1.123.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nebu-0.1.123.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
-nebu-0.1.123.dist-info/RECORD,,
+nebu-0.1.125.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nebu-0.1.125.dist-info/METADATA,sha256=tGSLm0cFTlfFS5lBPfqDzEX8tGWIgqlD1KfSGkISKqw,1798
+nebu-0.1.125.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nebu-0.1.125.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
+nebu-0.1.125.dist-info/RECORD,,