webscout 8.3.1__py3-none-any.whl → 8.3.3__py3-none-any.whl

webscout/AIutel.py

@@ -1,5 +1,6 @@
 import codecs
 import json
+import re
 from typing import (
     Any,
     AsyncGenerator,
@@ -48,17 +49,6 @@ def _process_chunk(
         error_handler (Optional[Callable[[Exception, str], Optional[Any]]]): An optional callback function that is called when JSON parsing fails.
             It receives the exception and the sanitized chunk as arguments.  It should return a value to yield instead of the raw chunk, or None to ignore.
 
-
-    Args:
-        chunk: Chunk of text to process.
-        intro_value: Prefix to remove from the chunk.
-        to_json: Parse the chunk as JSON if True.
-        skip_markers: List of markers to skip.
-        strip_chars: Characters to strip from the chunk.
-        yield_raw_on_error: Whether to return the raw chunk on parse errors.
-        error_handler: Optional callback ``Callable[[Exception, str], Optional[Any]]``
-            invoked when JSON parsing fails. The callback should return a value to
-            yield instead of the raw chunk, or ``None`` to ignore.
     """
     if not isinstance(chunk, str):
         return None
@@ -128,11 +118,6 @@ def _decode_byte_stream(
             Defaults to 'replace'.
         buffer_size (int): The size of the internal buffer used for decoding.
 
-    Args:
-        byte_iterator: Iterator yielding bytes
-        encoding: Character encoding to use
-        errors: How to handle encoding errors ('strict', 'ignore', 'replace')
-        buffer_size: Size of internal buffer for performance tuning
     """
     # Initialize decoder with the specified encoding
     try:
@@ -252,7 +237,7 @@ def _sanitize_stream_sync(
     Args:
         data: String, iterable of strings, or iterable of bytes to process.
         intro_value: Prefix indicating the start of meaningful data.
-        to_json: Parse JSON content if ``True``.
+        to_json: Parse the chunk as JSON if True.
         skip_markers: Lines containing any of these markers are skipped.
         strip_chars: Characters to strip from each line.
         start_marker: Begin processing only after this marker is found.
@@ -277,69 +262,50 @@ def _sanitize_stream_sync(
     processing_active = start_marker is None
     buffer = ""
     found_start = False if start_marker else True
+    line_iterator: Iterable[str]
 
-    # Fast path for single string processing
     if isinstance(data, str):
-        processed_item = None
-        if processing_active:
-            if to_json:
-                try:
-                    data = data.strip()
-                    if data:
-                        processed_item = json.loads(data)
-                except Exception as e:
-                    if error_handler:
-                        try:
-                            handled = error_handler(e, data)
-                            if handled is not None:
-                                processed_item = handled
-
-                        except Exception:
-                            pass
-                    if processed_item is None:
-                        processed_item = data if yield_raw_on_error else None
+        # If data is a string, decide whether to split it into lines
+        # or treat it as an iterable containing a single chunk.
+        temp_lines: List[str]
+        if line_delimiter is None:  # Default: split by newlines if present
+            if '\n' in data or '\r' in data:
+                temp_lines = data.splitlines()
             else:
-                processed_item = _process_chunk(
-                    data, intro_value, False, effective_skip_markers,
-                    strip_chars, yield_raw_on_error, error_handler
-                )
-            if processed_item is not None:
-                if content_extractor:
-                    try:
-                        final_content = content_extractor(processed_item)
-                        if final_content is not None:
-                            yield final_content
-                    except Exception:
-                        pass
-                else:
-                    yield processed_item
-        return
-
-    # Stream processing path
-    if not hasattr(data, '__iter__'):
-        raise TypeError(f"Input must be a string or an iterable, not {type(data).__name__}")
-
-    try:
-        iterator = iter(data)
-        first_item = next(iterator, None)
-        if first_item is None:
+                temp_lines = [data]  # Treat as a single line/chunk
+        elif line_delimiter in data:  # Custom delimiter found in string
+            temp_lines = data.split(line_delimiter)
+        else:  # Custom delimiter not found, or string is effectively a single segment
+            temp_lines = [data]
+        line_iterator = iter(temp_lines)
+    elif hasattr(data, '__iter__'):  # data is an iterable (but not a string)
+        _iter = iter(data)
+        first_item = next(_iter, None)
+
+        if first_item is None:  # Iterable was empty
             return
+
         from itertools import chain
-        stream = chain([first_item], iterator)
+        # Reconstruct the full iterable including the first_item
+        stream_input_iterable = chain([first_item], _iter)
 
-        # Determine if we're dealing with bytes or strings
         if isinstance(first_item, bytes):
+            # Ensure stream_input_iterable is typed as Iterable[bytes] for _decode_byte_stream
             line_iterator = _decode_byte_stream(
-                stream,
+                stream_input_iterable, # type: ignore
                 encoding=encoding,
                 errors=encoding_errors,
                 buffer_size=buffer_size
             )
         elif isinstance(first_item, str):
-            line_iterator = stream
+            # Ensure stream_input_iterable is typed as Iterable[str]
+            line_iterator = stream_input_iterable # type: ignore
         else:
-            raise TypeError(f"Stream must yield strings or bytes, not {type(first_item).__name__}")
+            raise TypeError(f"Iterable must yield strings or bytes, not {type(first_item).__name__}")
+    else:  # Not a string and not an iterable
+        raise TypeError(f"Input must be a string or an iterable, not {type(data).__name__}")
 
+    try:
         for line in line_iterator:
             if not line:
                 continue
@@ -630,10 +596,17 @@ async def _sanitize_stream_async(
 def sanitize_stream(
     data: Union[
         str,
+        bytes,
         Iterable[str],
         Iterable[bytes],
         AsyncIterable[str],
         AsyncIterable[bytes],
+        dict,
+        list,
+        int,
+        float,
+        bool,
+        None,
     ],
     intro_value: str = "data:",
     to_json: bool = True,
@@ -648,19 +621,16 @@ def sanitize_stream(
     buffer_size: int = 8192,
     line_delimiter: Optional[str] = None,
     error_handler: Optional[Callable[[Exception, str], Optional[Any]]] = None,
+    object_mode: Literal["as_is", "json", "str"] = "json",
+    raw: bool = False,
 ) -> Union[Generator[Any, None, None], AsyncGenerator[Any, None]]:
     """
     Processes streaming data (strings or bytes) in either synchronous or asynchronous mode.
-
-    This function acts as a unified interface for handling both synchronous and
-    asynchronous data streams. It automatically detects the type of input data and
-    dispatches it to the appropriate processing function (`_sanitize_stream_sync` or
-    `_sanitize_stream_async`).
+    Now supports non-iterable and miscellaneous input types (dict, list, int, float, bool, None).
 
     Args:
-        data (Union[str, Iterable[str], Iterable[bytes], AsyncIterable[str], AsyncIterable[bytes]]):
-            The data to be processed. Can be a string, a synchronous iterable of strings or bytes,
-            or an asynchronous iterable of strings or bytes.
+        data: The data to be processed. Can be a string, bytes, a synchronous iterable of strings or bytes,
+            an asynchronous iterable of strings or bytes, or a single object (dict, list, int, float, bool, None).
         intro_value (str): Prefix indicating the start of meaningful data. Defaults to "data:".
         to_json (bool): Parse JSON content if ``True``. Defaults to True.
         skip_markers (Optional[List[str]]): Lines containing any of these markers are skipped. Defaults to None.
@@ -678,25 +648,157 @@ def sanitize_stream(
         error_handler (Optional[Callable[[Exception, str], Optional[Any]]]):
             Callback invoked with ``(Exception, str)`` when JSON parsing fails.
             If the callback returns a value, it is yielded in place of the raw line. Defaults to None.
+        object_mode (Literal["as_is", "json", "str"]): How to handle non-string, non-iterable objects.
+            "json" (default) yields as JSON string, "str" yields as str(obj), "as_is" yields the object as-is.
+        raw (bool): If True, yields the raw response as returned by the API, chunk by chunk (no splitting or joining).
 
     Returns:
         Union[Generator[Any, None, None], AsyncGenerator[Any, None]]:
-            A generator or an asynchronous generator yielding the processed data.
-    """
+            A generator or an asynchronous generator yielding the processed data, or raw data if raw=True.
+    """    # --- RAW MODE: yield each chunk exactly as returned by the API ---
+    if raw:
+        def _raw_passthrough_sync(source_iter):
+            for chunk in source_iter:
+                if isinstance(chunk, (bytes, bytearray)):
+                    # Decode bytes preserving all whitespace and newlines
+                    yield chunk.decode(encoding, encoding_errors)
+                elif chunk is not None:
+                    # Yield string chunks as-is, preserving all formatting
+                    yield chunk
+                # Skip None chunks entirely
+        async def _raw_passthrough_async(source_aiter):
+            async for chunk in source_aiter:
+                if isinstance(chunk, (bytes, bytearray)):
+                    # Decode bytes preserving all whitespace and newlines
+                    yield chunk.decode(encoding, encoding_errors)
+                elif chunk is not None:
+                    # Yield string chunks as-is, preserving all formatting
+                    yield chunk
+                # Skip None chunks entirely
+        # Sync iterable (but not str/bytes)
+        if hasattr(data, "__iter__") and not isinstance(data, (str, bytes)):
+            return _raw_passthrough_sync(data)
+        # Async iterable
+        if hasattr(data, "__aiter__"):
+            return _raw_passthrough_async(data)
+        # Single string or bytes
+        if isinstance(data, (bytes, bytearray)):
+            def _yield_single():
+                yield data.decode(encoding, encoding_errors)
+            return _yield_single()
+        else:
+            def _yield_single():
+                if data is not None:
+                    yield data
+            return _yield_single()
+    # --- END RAW MODE ---
+
+    text_attr = getattr(data, "text", None)
+    content_attr = getattr(data, "content", None)
+
+    # Handle None
+    if data is None:
+        def _empty_gen():
+            if False:
+                yield None
+        return _empty_gen()
+
+    # Handle bytes directly
+    if isinstance(data, bytes):
+        try:
+            payload = data.decode(encoding, encoding_errors)
+        except Exception:
+            payload = str(data)
+        return _sanitize_stream_sync(
+            payload, intro_value, to_json, skip_markers, strip_chars,
+            start_marker, end_marker, content_extractor, yield_raw_on_error,
+            encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+        )
+
+    # Handle string directly
+    if isinstance(data, str):
+        return _sanitize_stream_sync(
+            data, intro_value, to_json, skip_markers, strip_chars,
+            start_marker, end_marker, content_extractor, yield_raw_on_error,
+            encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+        )
+
+    # Handle dict, list, int, float, bool (non-iterable, non-string/bytes)
+    if isinstance(data, (dict, list, int, float, bool)):
+        if object_mode == "as_is":
+            def _as_is_gen():
+                yield data
+            return _as_is_gen()
+        elif object_mode == "str":
+            return _sanitize_stream_sync(
+                str(data), intro_value, to_json, skip_markers, strip_chars,
+                start_marker, end_marker, content_extractor, yield_raw_on_error,
+                encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+            )
+        else:  # "json"
+            try:
+                json_str = json.dumps(data)
+            except Exception:
+                json_str = str(data)
+            return _sanitize_stream_sync(
+                json_str, intro_value, to_json, skip_markers, strip_chars,
+                start_marker, end_marker, content_extractor, yield_raw_on_error,
+                encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+            )
+
+    # Handle file-like objects (optional, treat as string if .read exists)
+    if hasattr(data, "read") and callable(data.read):
+        try:
+            file_content = data.read()
+            if isinstance(file_content, bytes):
+                file_content = file_content.decode(encoding, encoding_errors)
+            return _sanitize_stream_sync(
+                file_content, intro_value, to_json, skip_markers, strip_chars,
+                start_marker, end_marker, content_extractor, yield_raw_on_error,
+                encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+            )
+        except Exception:
+            pass  # fallback to next
+
+    # Handle .text or .content attributes
+    if isinstance(text_attr, str):
+        payload = text_attr
+        return _sanitize_stream_sync(
+            payload, intro_value, to_json, skip_markers, strip_chars,
+            start_marker, end_marker, content_extractor, yield_raw_on_error,
+            encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+        )
+    elif isinstance(content_attr, bytes):
+        try:
+            payload = content_attr.decode(encoding, encoding_errors)
+        except Exception:
+            payload = str(content_attr)
+        return _sanitize_stream_sync(
+            payload, intro_value, to_json, skip_markers, strip_chars,
+            start_marker, end_marker, content_extractor, yield_raw_on_error,
+            encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+        )
 
+    # Handle async iterables
     if hasattr(data, "__aiter__"):
         return _sanitize_stream_async(
             data, intro_value, to_json, skip_markers, strip_chars,
             start_marker, end_marker, content_extractor, yield_raw_on_error,
             encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
         )
+    # Handle sync iterables (but not strings/bytes)
+    if hasattr(data, "__iter__"):
+        return _sanitize_stream_sync(
+            data, intro_value, to_json, skip_markers, strip_chars,
+            start_marker, end_marker, content_extractor, yield_raw_on_error,
+            encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
+        )
+    # Fallback: treat as string
     return _sanitize_stream_sync(
-        data, intro_value, to_json, skip_markers, strip_chars,
+        str(data), intro_value, to_json, skip_markers, strip_chars,
         start_marker, end_marker, content_extractor, yield_raw_on_error,
         encoding, encoding_errors, buffer_size, line_delimiter, error_handler,
     )
-
-
 from .conversation import Conversation  # noqa: E402,F401
 from .Extra.autocoder import AutoCoder  # noqa: E402,F401
 from .optimizers import Optimizers  # noqa: E402,F401