sunholo 0.95.1__tar.gz → 0.95.2__tar.gz

{sunholo-0.95.1 → sunholo-0.95.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.95.1
+Version: 0.95.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.95.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.95.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0

{sunholo-0.95.1 → sunholo-0.95.2}/setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.95.1'
+version = '0.95.2'
 
 setup(
     name='sunholo',

{sunholo-0.95.1 → sunholo-0.95.2}/sunholo/agents/flask/vac_routes.py

@@ -185,11 +185,9 @@ if __name__ == "__main__":
 
         log.info(f"OpenAI response: {openai_response}")
         return jsonify(openai_response)
-    
+        
     def handle_stream_vac(self, vector_name):
         observed_stream_interpreter = self.stream_interpreter
-
-        # Determine if the stream interpreter is async or sync
         is_async = inspect.iscoroutinefunction(self.stream_interpreter)
 
         if is_async:
@@ -219,52 +217,53 @@ if __name__ == "__main__":
         def generate_response_content():
             try:
                 if is_async:
-                    # Run the async version
-                    async def process_async():
-                        async_gen = start_streaming_chat_async(
-                            question=all_input["user_input"],
-                            vector_name=vector_name,
-                            qna_func_async=observed_stream_interpreter,
-                            chat_history=all_input["chat_history"],
-                            wait_time=all_input["stream_wait_time"],
-                            timeout=all_input["stream_timeout"],
-                            trace_id=trace.id if trace else None,
-                            **all_input["kwargs"]
-                        )
-                        log.info(f"{async_gen=}")
-                        # Yield intermediate results as they become available
-                        async for chunk in async_gen:
-                            if isinstance(chunk, dict) and 'answer' in chunk:
-                                if trace:
-                                    chunk["trace_id"] = trace.id
-                                    chunk["trace_url"] = trace.get_trace_url()
-                                    generation.end(output=json.dumps(chunk))
-                                    span.end(output=json.dumps(chunk))
-                                    trace.update(output=json.dumps(chunk))
-                                
-                                archive_qa(chunk, vector_name)
-                                
-                                yield json.dumps(chunk)
-                            else:
-                                yield chunk
-
-                    try:
-                        loop = asyncio.get_running_loop()
-                        # Use the event loop to consume the async generator and yield results
-                        return [chunk for chunk in loop.run_until_complete(self._async_generator_to_list(process_async()))]
-                    except RuntimeError:
-                        # No running event loop, create a new one
-                        log.info("Creating new event loop")
-                        loop = asyncio.new_event_loop()
-                        asyncio.set_event_loop(loop)
-                        try:
-                            log.info(f"Run until complete loop for {process_async}")
-                            chunks = loop.run_until_complete(self._async_generator_to_list(process_async()))
-                            for chunk in chunks:
-                                yield chunk
-                        finally:
-                            loop.close()
-
+                    from queue import Queue, Empty
+                    result_queue = Queue()
+                    import threading
+
+                    def run_async():
+                        async def process_async():
+                            try:
+                                async_gen = start_streaming_chat_async(
+                                    question=all_input["user_input"],
+                                    vector_name=vector_name,
+                                    qna_func_async=observed_stream_interpreter,
+                                    chat_history=all_input["chat_history"],
+                                    wait_time=all_input["stream_wait_time"],
+                                    timeout=all_input["stream_timeout"],
+                                    trace_id=trace.id if trace else None,
+                                    **all_input["kwargs"]
+                                )
+                                log.info(f"{async_gen=}")
+                                async for chunk in async_gen:
+                                    if isinstance(chunk, dict) and 'answer' in chunk:
+                                        if trace:
+                                            chunk["trace_id"] = trace.id
+                                            chunk["trace_url"] = trace.get_trace_url()
+                                            generation.end(output=json.dumps(chunk))
+                                            span.end(output=json.dumps(chunk))
+                                            trace.update(output=json.dumps(chunk))
+                                        archive_qa(chunk, vector_name)
+                                        result_queue.put(json.dumps(chunk))
+                                    else:
+                                        result_queue.put(chunk)
+                            except Exception as e:
+                                result_queue.put(f"Streaming Error: {str(e)} {traceback.format_exc()}")
+                            finally:
+                                result_queue.put(None)  # Sentinel
+                        asyncio.run(process_async())
+
+                    thread = threading.Thread(target=run_async)
+                    thread.start()
+
+                    # Read from the queue and yield results
+                    while True:
+                        chunk = result_queue.get()
+                        if chunk is None:
+                            break
+                        yield chunk
+
+                    thread.join()
                 else:
                     log.info("sync streaming response")
                     for chunk in start_streaming_chat(
@@ -286,16 +285,16 @@ if __name__ == "__main__":
                                 generation.end(output=json.dumps(chunk))
                                 span.end(output=json.dumps(chunk))
                                 trace.update(output=json.dumps(chunk))
-                            return json.dumps(chunk)
+                            yield json.dumps(chunk)
                         else:
                             yield chunk
 
             except Exception as e:
                 yield f"Streaming Error: {str(e)} {traceback.format_exc()}"
-            
+
         # Here, the generator function will handle streaming the content to the client.
         response = Response(generate_response_content(), content_type='text/plain; charset=utf-8')
-        response.headers['Transfer-Encoding'] = 'chunked'  
+        response.headers['Transfer-Encoding'] = 'chunked'
 
         log.debug(f"streaming response: {response}")
         if trace:
@@ -307,12 +306,10 @@ if __name__ == "__main__":
         return response
 
     @staticmethod
-    async def _async_generator_to_list(async_gen):
-        """Helper function to collect an async generator's values into a list."""
-        result = []
-        async for item in async_gen:
-            result.append(item)
-        return result
+    async def _async_generator_to_stream(async_gen_func):
+        """Helper function to stream the async generator's values to the client."""
+        async for item in async_gen_func():
+            yield item
     
     def langfuse_eval_response(self, trace_id, eval_percent=0.01):
         """

{sunholo-0.95.1 → sunholo-0.95.2}/sunholo/invoke/async_class.py

@@ -1,11 +1,9 @@
 import asyncio
-import logging
+from ..custom_logging import log
 import traceback
 from typing import Callable, List, Any, AsyncGenerator, Dict
 from tenacity import AsyncRetrying, retry_if_exception_type, wait_random_exponential, stop_after_attempt
 
-logger = logging.getLogger(__name__)
-
 class AsyncTaskRunner:
     def __init__(self, retry_enabled=False, retry_kwargs=None):
         self.tasks = []
@@ -14,18 +12,19 @@ class AsyncTaskRunner:
     
     def add_task(self, func: Callable[..., Any], *args: Any):
         """Adds a task to the list of tasks to be executed."""
-        logger.info(f"Adding task: {func.__name__} with args: {args}")
+        log.info(f"Adding task: {func.__name__} with args: {args}")
         self.tasks.append((func.__name__, func, args))
     
     async def run_async_as_completed(self) -> AsyncGenerator[Dict[str, Any], None]:
         """Runs all tasks concurrently and yields results as they complete."""
-        logger.info("Running tasks asynchronously and yielding results as they complete")
+        log.info("Running tasks asynchronously and yielding results as they complete")
         tasks = {}
         for name, func, args in self.tasks:
             coro = self._task_wrapper(name, func, args)
             task = asyncio.create_task(coro)
             tasks[task] = name
         
+        log.info(f"Start async run with {len(self.tasks)} runners")
         while tasks:
             done, _ = await asyncio.wait(tasks.keys(), return_when=asyncio.FIRST_COMPLETED)
             for task in done:
@@ -34,7 +33,7 @@ class AsyncTaskRunner:
                     result = await task
                     yield {name: result}
                 except Exception as e:
-                    logger.error(f"Task {name} resulted in an error: {e}\n{traceback.format_exc()}")
+                    log.error(f"Task {name} resulted in an error: {e}\n{traceback.format_exc()}")
                     yield {name: e}
     
     async def _task_wrapper(self, name: str, func: Callable[..., Any], args: Any) -> Any:
@@ -65,13 +64,13 @@ class AsyncTaskRunner:
             try:
                 return await run_func()
             except Exception as e:
-                logger.error(f"Error in task {name}: {e}\n{traceback.format_exc()}")
+                log.error(f"Error in task {name}: {e}\n{traceback.format_exc()}")
                 raise
 
     def _log_results(self, results: List[Any]):
         """Logs the results of the task executions."""
         for result in results:
             if isinstance(result, Exception):
-                logger.error(f"Task resulted in an error: {result}")
+                log.error(f"Task resulted in an error: {result}")
             else:
-                logger.info(f"Task completed successfully: {result}")
+                log.info(f"Task completed successfully: {result}")

{sunholo-0.95.1 → sunholo-0.95.2}/sunholo/streaming/content_buffer.py

@@ -51,6 +51,17 @@ class ContentBuffer:
         Adds the given text to the existing content of the buffer.
         """
         self.content += text
+
+    async def async_write(self, text: str):
+        """
+        Asynchronously writes text to the content buffer.
+
+        Args:
+            text (str): The text to be added to the buffer.
+
+        Adds the given text to the existing content of the buffer.
+        """
+        self.content += text
     
     def read(self) -> str:
         """
@@ -63,6 +74,15 @@ class ContentBuffer:
         """   
         return self.content
 
+    async def async_read(self) -> str:
+        """
+        Asynchronously reads the entire content from the buffer.
+
+        Returns:
+            str: The content of the buffer.
+        """
+        return self.content
+
     def clear(self):
         """
         Clears the content buffer.
@@ -71,7 +91,15 @@ class ContentBuffer:
         """
         self.content = ""
 
-    
+    async def async_clear(self):
+        """
+        Asynchronously clears the content buffer.
+
+        Empties the buffer content, resetting it to an empty string.
+        """
+        self.content = ""
+
+
 class BufferStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler):
     """
     A callback handler for streaming LLM output to a content buffer.
@@ -136,6 +164,24 @@ class BufferStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler):
         if not self.in_code_block:
             self._process_buffer()
 
+    async def async_on_llm_new_token(self, token: str, **kwargs: Any) -> None:
+        """
+        Asynchronously processes a new token from the LLM output.
+
+        Args:
+            token (str): The new token generated by the LLM.
+            **kwargs: Additional keyword arguments.
+        """
+        log.debug(f"async_on_llm_new_token: {token}")
+        
+        self.buffer += token
+
+        if '```' in token:
+            self.in_code_block = not self.in_code_block
+
+        if not self.in_code_block:
+            await self._async_process_buffer()
+
     def _process_buffer(self):
         """
         Processes the buffer content and writes to the content buffer.
@@ -154,6 +200,24 @@ class BufferStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler):
                 self.content_buffer.write(self.buffer)
                 self.buffer = ""
 
+    async def _async_process_buffer(self):
+        """
+        Asynchronously processes the buffer content and writes to the content buffer.
+
+        If the buffer ends with a numbered list pattern or specified tokens, the buffer is flushed
+        to the content buffer. Otherwise, the buffer is left intact for further accumulation.
+        """
+        matches = list(re.finditer(r'\n(\d+\.\s)', self.buffer))
+        if matches:
+            last_match = matches[-1]
+            start_of_last_match = last_match.start() + 1
+            await self.content_buffer.async_write(self.buffer[:start_of_last_match])
+            self.buffer = self.buffer[start_of_last_match:]
+        else:
+            if any(self.buffer.endswith(t) for t in self.tokens):
+                await self.content_buffer.async_write(self.buffer)
+                self.buffer = ""
+
     def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
         """
         Handles the end of LLM streaming.
@@ -172,3 +236,19 @@ class BufferStreamingStdOutCallbackHandler(StreamingStdOutCallbackHandler):
 
         self.stream_finished.set()
         log.info("Streaming LLM response ended successfully")
+
+    async def async_on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
+        """
+        Asynchronously handles the end of LLM streaming.
+
+        Args:
+            response (LLMResult): The result returned by the LLM.
+            **kwargs: Additional keyword arguments.
+        """
+        if self.buffer:
+            await self.content_buffer.async_write(self.buffer)
+            self.buffer = ""
+            log.info("Flushing remaining LLM response buffer")
+
+        self.stream_finished.set()
+        log.info("Streaming LLM response ended successfully")

{sunholo-0.95.1 → sunholo-0.95.2}/sunholo.egg-info/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.1
 Name: sunholo
-Version: 0.95.1
+Version: 0.95.2
 Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
 Home-page: https://github.com/sunholo-data/sunholo-py
-Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.95.1.tar.gz
+Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.95.2.tar.gz
 Author: Holosun ApS
 Author-email: multivac@sunholo.com
 License: Apache License, Version 2.0