nebu 0.1.105__py3-none-any.whl → 0.1.108__py3-none-any.whl

nebu/data.py

@@ -59,7 +59,7 @@ def rclone_copy(
         source_dir,
         destination,
         f"--transfers={transfers}",
-        "--progress",
+        # "--progress",
     ]
 
     if dry_run:
@@ -1171,7 +1171,7 @@ class RcloneBucket(StorageBucket):
             "--modify-window=2s",
             "--log-level=DEBUG" if self.verbose else "--log-level=INFO",
             "--log-format=date,time,level,message",
-            "--progress",  # Add progress display
+            # "--progress",  # Add progress display
         ]
         if dry_run:
             rc_args.append("--dry-run")
@@ -1286,7 +1286,7 @@ class RcloneBucket(StorageBucket):
         rc_args = [
             "--log-level=DEBUG" if self.verbose else "--log-level=INFO",
             "--log-format=date,time,level,message",
-            "--progress",  # Add progress display
+            # "--progress",  # Add progress display
         ]
 
         # Set environment variables for AWS credentials if they exist

nebu/processors/consumer.py

@@ -366,6 +366,8 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
     global target_function, imported_module, local_namespace
     global execution_mode, r, REDIS_STREAM, REDIS_CONSUMER_GROUP
 
+    print(f">>> Processing message {message_id}")
+
     # --- Subprocess Execution Path ---
     if execution_mode == "subprocess":
         logger.info(f"Processing message {message_id} in subprocess...")
@@ -610,8 +612,15 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         logger.debug(f">> Raw payload: {raw_payload}")
 
         # --- Extract fields from the *inner* message content for HealthCheck and regular processing ---
-        # The actual message content is inside raw_payload[\"content\"]
+        # The actual message content is inside raw_payload["content"]
         inner_content_data = raw_payload.get("content", {})
+
+        # Add debug logging for content structure analysis
+        logger.debug(f">> inner_content_data type: {type(inner_content_data)}")
+        logger.debug(
+            f">> inner_content_data keys (if dict): {list(inner_content_data.keys()) if isinstance(inner_content_data, dict) else 'N/A'}"
+        )
+
         if not isinstance(inner_content_data, dict):
             # If content is not a dict (e.g. already a primitive from a non-Message processor)
             # we can't reliably get 'kind' or other fields from it.
@@ -625,11 +634,44 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
             actual_content_to_process = inner_content_data  # Use it as is
             inner_created_at_str = None
         else:
-            inner_kind = inner_content_data.get("kind", "")
-            inner_msg_id = inner_content_data.get("id", "")
-            # The 'content' field of the inner_content_data is what the user function expects
-            actual_content_to_process = inner_content_data.get("content", {})
-            inner_created_at_str = inner_content_data.get("created_at")
+            # Check if this looks like a nested message structure (has kind/id/content fields)
+            # vs direct content data
+            has_message_structure = (
+                "kind" in inner_content_data
+                and "id" in inner_content_data
+                and "content" in inner_content_data
+            )
+
+            logger.debug(f">> has_message_structure: {has_message_structure}")
+
+            if has_message_structure:
+                # Nested message structure: extract from inner message
+                inner_kind = inner_content_data.get("kind", "")
+                inner_msg_id = inner_content_data.get("id", "")
+                actual_content_to_process = inner_content_data.get("content", {})
+                inner_created_at_str = inner_content_data.get("created_at")
+                logger.debug(
+                    f">> Using nested structure - inner_kind: {inner_kind}, inner_msg_id: {inner_msg_id}"
+                )
+                logger.debug(
+                    f">> actual_content_to_process keys: {list(actual_content_to_process.keys())}"
+                )
+            else:
+                # Direct content structure: the content data is directly in inner_content_data
+                inner_kind = raw_payload.get("kind", "")  # Get kind from outer payload
+                inner_msg_id = raw_payload.get("id", "")  # Get id from outer payload
+                actual_content_to_process = (
+                    inner_content_data  # Use inner_content_data directly
+                )
+                inner_created_at_str = raw_payload.get(
+                    "created_at"
+                )  # Get created_at from outer payload
+                logger.debug(
+                    f">> Using direct structure - inner_kind: {inner_kind}, inner_msg_id: {inner_msg_id}"
+                )
+                logger.debug(
+                    f">> actual_content_to_process keys: {list(actual_content_to_process.keys())}"
+                )
 
         # Attempt to parse inner_created_at, fallback to now()
         try:
@@ -685,12 +727,24 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         if isinstance(actual_content_to_process, str):
             try:
                 content_for_validation = json.loads(actual_content_to_process)
+                logger.debug(
+                    f">> Parsed JSON string content_for_validation keys: {list(content_for_validation.keys()) if isinstance(content_for_validation, dict) else 'N/A'}"
+                )
             except json.JSONDecodeError:
                 content_for_validation = (
                     actual_content_to_process  # Keep as string if not valid JSON
                 )
+                logger.debug(
+                    f">> Failed to parse JSON, keeping as string: {type(actual_content_to_process)}"
+                )
         else:
             content_for_validation = actual_content_to_process
+            logger.debug(
+                f">> Using actual_content_to_process directly: {type(content_for_validation)}"
+            )
+            logger.debug(
+                f">> content_for_validation keys: {list(content_for_validation.keys())}"
+            )
 
         # --- Construct Input Object using Imported Types ---
         input_obj: Any = None
@@ -736,9 +790,16 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
 
                 if content_model_class:
                     try:
+                        logger.debug(
+                            f">> Attempting to validate content with {content_model_class.__name__}"
+                        )
+                        logger.debug(
+                            f">> Content being validated: {json.dumps(content_for_validation, indent=2) if isinstance(content_for_validation, dict) else str(content_for_validation)}"
+                        )
                         content_model = content_model_class.model_validate(
                             content_for_validation
                         )
+                        logger.debug(f">> Successfully validated content model")
                         # print(f"Validated content model: {content_model}")
                         input_obj = message_class(
                             kind=inner_kind,
@@ -752,10 +813,16 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                             adapter=adapter,
                             api_key=api_key,
                         )
+                        logger.debug(
+                            f">> Successfully created Message object with validated content"
+                        )
                     except Exception as e:
                         logger.error(
                             f"Error validating/creating content model '{content_type_name}': {e}. Falling back."
                         )
+                        logger.debug(
+                            f">> Content validation failed for: {json.dumps(content_for_validation, indent=2) if isinstance(content_for_validation, dict) else str(content_for_validation)}"
+                        )
                         # Fallback to raw content in Message
                         input_obj = message_class(
                             kind=inner_kind,
@@ -769,8 +836,12 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                             adapter=adapter,
                             api_key=api_key,
                         )
+                        logger.debug(
+                            f">> Created Message object with raw content fallback"
+                        )
                 else:
                     # No content type name or class found, use raw content
+                    logger.debug(f">> No content model class found, using raw content")
                     input_obj = message_class(
                         kind=inner_kind,
                         id=inner_msg_id,
@@ -783,6 +854,9 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
                         adapter=adapter,
                         api_key=api_key,
                     )
+                    logger.debug(
+                        f">> Created Message object with raw content (no content model class)"
+                    )
             else:  # Not a stream message, use the function's parameter type
                 param_type_name = (
                     param_type_str  # Assume param_type_str holds the class name
@@ -844,7 +918,15 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
         # Safe logging that avoids __repr__ issues with BaseModel objects
         try:
             if hasattr(input_obj, "model_dump"):
-                logger.debug(f"Input object (BaseModel): {input_obj.model_dump()}")
+                try:
+                    # Use model_dump_json for safer serialization that handles nested objects
+                    logger.debug(
+                        f"Input object (BaseModel): {input_obj.model_dump_json()}"
+                    )
+                except Exception as dump_e:
+                    logger.debug(
+                        f"Input object: <BaseModel object of type {type(input_obj).__name__}> (model_dump failed: {dump_e})"
+                    )
             else:
                 logger.debug(f"Input object: {input_obj}")
         except Exception as log_e:
@@ -854,15 +936,13 @@ def process_message(message_id: str, message_data: Dict[str, str]) -> None:
 
         # Execute the function
         logger.info("Executing function...")
+
+        # Add warning about potential print statement issues in user code
+        logger.debug(
+            ">> About to execute user function - note: print statements in user code may fail if the Message object has validation issues"
+        )
+
         result = target_function(input_obj)
-        # Safe logging that avoids __repr__ issues with BaseModel objects if this debug line is uncommented
-        # try:
-        #     if hasattr(result, "model_dump"):
-        #         logger.debug(f"Raw Result (BaseModel): {result.model_dump()}")
-        #     else:
-        #         logger.debug(f"Raw Result: {result}")
-        # except Exception as log_e:
-        #     logger.debug(f"Raw Result: <object of type {type(result).__name__}> (repr failed: {log_e})")
 
         result_content = None  # Default to None
         if result is not None:  # Only process if there's a result

nebu/processors/processor.py

@@ -2,7 +2,7 @@ import json
 import threading
 import time
 import uuid
-from typing import Any, Dict, Generic, List, Optional, TypeVar
+from typing import Any, Dict, Generic, List, Optional, TypeVar, cast, get_args
 
 import requests
 from pydantic import BaseModel
@@ -132,6 +132,30 @@ class Processor(Generic[InputType, OutputType]):
         self.processors_url = f"{self.orign_host}/v1/processors"
         self._log_thread: Optional[threading.Thread] = None
 
+        # Attempt to infer OutputType if schema_ is not provided
+        if self.schema_ is None and hasattr(self, "__orig_class__"):
+            type_args = get_args(self.__orig_class__)  # type: ignore
+            if len(type_args) == 2:
+                output_type_candidate = type_args[1]
+                # Check if it looks like a Pydantic model class
+                if isinstance(output_type_candidate, type) and issubclass(
+                    output_type_candidate, BaseModel
+                ):
+                    logger.debug(
+                        f"Inferred OutputType {output_type_candidate.__name__} from generic arguments."
+                    )
+                    self.schema_ = output_type_candidate
+                else:
+                    logger.debug(
+                        f"Second generic argument {output_type_candidate} is not a Pydantic BaseModel. "
+                        "Cannot infer OutputType."
+                    )
+            else:
+                logger.debug(
+                    "Could not infer OutputType from generic arguments: wrong number of type args found "
+                    f"(expected 2, got {len(type_args) if type_args else 0})."
+                )
+
         # Fetch existing Processors
         response = requests.get(
             self.processors_url, headers={"Authorization": f"Bearer {self.api_key}"}
@@ -222,12 +246,18 @@ class Processor(Generic[InputType, OutputType]):
         logs: bool = False,
         api_key: Optional[str] = None,
         user_key: Optional[str] = None,
+        timeout: Optional[float] = 600.0,
     ) -> OutputType | Dict[str, Any] | None:
         """
         Allows the Processor instance to be called like a function, sending data.
         """
         return self.send(
-            data=data, wait=wait, logs=logs, api_key=api_key, user_key=user_key
+            data=data,
+            wait=wait,
+            logs=logs,
+            api_key=api_key,
+            user_key=user_key,
+            timeout=timeout,
         )
 
     def send(
@@ -271,7 +301,9 @@ class Processor(Generic[InputType, OutputType]):
             timeout=timeout,
         )
         response.raise_for_status()
-        send_response_json = response.json()
+        raw_response_json = response.json()
+        raw_content = raw_response_json.get("content")
+        logger.debug(f">>> Raw content: {raw_content}")
 
         # --- Fetch Logs (if requested and not already running) ---
         if logs:
@@ -297,7 +329,33 @@ class Processor(Generic[InputType, OutputType]):
             else:
                 logger.info(f"Log fetching is already running for {processor_name}.")
 
-        return send_response_json
+        # Attempt to parse into OutputType if conditions are met
+        if (
+            wait
+            and self.schema_
+            and isinstance(self.schema_, type)
+            and issubclass(self.schema_, BaseModel)  # type: ignore
+            and isinstance(raw_content, dict)
+        ):  # Check if raw_content is a dict
+            try:
+                # self.schema_ is assumed to be the Pydantic model class for OutputType
+                # Parse raw_content instead of the full response
+                parsed_model = self.schema_.model_validate(raw_content)
+                # Cast to OutputType to satisfy the linter with generics
+                parsed_output: OutputType = cast(OutputType, parsed_model)
+                return parsed_output
+            except (
+                Exception
+            ) as e:  # Consider pydantic.ValidationError for more specific handling
+                schema_name = getattr(self.schema_, "__name__", str(self.schema_))
+                logger.error(
+                    f"Processor {processor_name}: Failed to parse 'content' field into output type {schema_name}. "
+                    f"Error: {e}. Returning raw JSON response."
+                )
+                # Fallback to returning the raw JSON response
+                return raw_content
+
+        return raw_content
 
     def scale(self, replicas: int) -> Dict[str, Any]:
         """

{nebu-0.1.105.dist-info → nebu-0.1.108.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nebu
-Version: 0.1.105
+Version: 0.1.108
 Summary: A globally distributed container runtime
 Requires-Python: >=3.10.14
 Description-Content-Type: text/markdown

{nebu-0.1.105.dist-info → nebu-0.1.108.dist-info}/RECORD

@@ -2,7 +2,7 @@ nebu/__init__.py,sha256=xNtWiN29MJZK_WBEUP-9hDmlkfLxoASVI-f4tNTXO58,454
 nebu/auth.py,sha256=N_v6SPFD9HU_UoRDTaouH03g2Hmo9C-xxqInE1FweXE,1471
 nebu/cache.py,sha256=JqRb4FdZrRrO4ePlwvsKdxRC8dNEFMxfTWag0aJz8Gw,4893
 nebu/config.py,sha256=C5Jt9Bd0i0HrgzBSVNJ-Ml3KwX_gaYbYYZEtNL2gvJg,7031
-nebu/data.py,sha256=4fEvO_xNCeTNW4RFr1u6iZ7j92S-5zWwxmFdXopxae8,63502
+nebu/data.py,sha256=v29F1Dwfa3cw6HzG2C6bxAfAJIu16k5PwZCC_niLwCU,63508
 nebu/errors.py,sha256=bBnK5YQ6qZg4OMY81AN2k03ppefg89FUwF_SHEMlqCA,170
 nebu/logging.py,sha256=VzpjCEoXm3c4i0sKJL5GTsPIhTQ6Y4BPUTzPmwhve7o,950
 nebu/meta.py,sha256=CzFHMND9seuewzq9zNNx9WTr6JvrCBExe7BLqDSr7lM,745
@@ -13,16 +13,16 @@ nebu/containers/container.py,sha256=Mrh_gvMsTvDkj3CwpqIPzJ72IMw0gQIg64y548vq0yg,
 nebu/containers/models.py,sha256=0j6NGy4yto-enRDh_4JH_ZTbHrLdSpuMOqNQPnIrwC4,6815
 nebu/namespaces/models.py,sha256=EqUOpzhVBhvJw2P92ONDUbIgC31M9jMmcaG5vyOrsWg,497
 nebu/namespaces/namespace.py,sha256=oeZyGqsIGIrppyjif1ZONsdTmqRgd9oSLFE1BChXTTE,5247
-nebu/processors/consumer.py,sha256=v5JsuvEEUUBbe8sKGB2QjHca8FPHMIjq9Ntk32Uf3G4,58674
+nebu/processors/consumer.py,sha256=9WapzBTPuXRunH-vjPerTlGZy__hn_d4m13l1ajebY8,62732
 nebu/processors/consumer_process_worker.py,sha256=h--eNFKaLbUayxn88mB8oGGdrU2liE1dnwm_TPlewX8,36960
 nebu/processors/decorate.py,sha256=AfHVCoNbW7RymccF5ewleEL-GlMiqVH1-t9bCmD60rk,58654
 nebu/processors/default.py,sha256=cy4ETMdbdRGkrvbYec1o60h7mGDlGN5JsuUph0ENtDU,364
 nebu/processors/models.py,sha256=g4B1t6Rgoy-NUEHBLeQc0EENzHXLDlWSio8Muv7cTDU,4093
-nebu/processors/processor.py,sha256=vbafqawdZhh5VRoBE5C5wRR_32iSyzoDTqK7qsNQvaM,19242
+nebu/processors/processor.py,sha256=PeJvxBPfjQ8uFqLcrGNk28yqBmqZC2VW4ssCsanZxYY,21985
 nebu/redis/models.py,sha256=coPovAcVXnOU1Xh_fpJL4PO3QctgK9nBe5QYoqEcnxg,1230
 nebu/services/service.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nebu-0.1.105.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
-nebu-0.1.105.dist-info/METADATA,sha256=B9wVIWPowGZHwobFzrRBymh14aUiwL_GAFTNRR80roo,1798
-nebu-0.1.105.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
-nebu-0.1.105.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
-nebu-0.1.105.dist-info/RECORD,,
+nebu-0.1.108.dist-info/licenses/LICENSE,sha256=xx0jnfkXJvxRnG63LTGOxlggYnIysveWIZ6H3PNdCrQ,11357
+nebu-0.1.108.dist-info/METADATA,sha256=drUdR1bXazQ1Yqlns8I1KPOt1qnoO6OhNQMAlDwaK9o,1798
+nebu-0.1.108.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+nebu-0.1.108.dist-info/top_level.txt,sha256=uLIbEKJeGSHWOAJN5S0i5XBGwybALlF9bYoB1UhdEgQ,5
+nebu-0.1.108.dist-info/RECORD,,