sunholo 0.141.1__py3-none-any.whl → 0.142.0__py3-none-any.whl

sunholo/invoke/async_class.py

@@ -8,12 +8,39 @@ from tenacity import AsyncRetrying, retry_if_exception_type, wait_random_exponen
 log = setup_logging("sunholo_AsyncTaskRunner")
 
 class AsyncTaskRunner:
-    def __init__(self, retry_enabled:bool=False, retry_kwargs:dict=None, timeout:int=120, max_concurrency:int=20):
+    def __init__(self, 
+                 retry_enabled: bool = False, 
+                 retry_kwargs: dict = None, 
+                 timeout: int = 120, 
+                 max_concurrency: int = 20,
+                 heartbeat_extends_timeout: bool = False,
+                 hard_timeout: int = None):
+        """
+        Initialize AsyncTaskRunner with configurable timeout behavior.
+        
+        Args:
+            retry_enabled: Whether to enable retries
+            retry_kwargs: Retry configuration
+            timeout: Base timeout for tasks (seconds)
+            max_concurrency: Maximum concurrent tasks
+            heartbeat_extends_timeout: If True, heartbeats reset the timeout timer
+            hard_timeout: Maximum absolute timeout regardless of heartbeats (seconds).
+                         If None, defaults to timeout * 5 when heartbeat_extends_timeout=True
+        """
         self.tasks = []
         self.retry_enabled = retry_enabled
         self.retry_kwargs = retry_kwargs or {}
         self.timeout = timeout
         self.semaphore = asyncio.Semaphore(max_concurrency)
+        self.heartbeat_extends_timeout = heartbeat_extends_timeout
+        
+        # Set hard timeout
+        if hard_timeout is not None:
+            self.hard_timeout = hard_timeout
+        elif heartbeat_extends_timeout:
+            self.hard_timeout = timeout * 5  # Default to 5x base timeout
+        else:
+            self.hard_timeout = timeout  # Same as regular timeout
 
     def add_task(self, func: Callable[..., Any], *args: Any, **kwargs: Any):
         """
@@ -39,9 +66,11 @@ class AsyncTaskRunner:
         for name, func, args, kwargs in self.tasks:
             log.info(f"Executing task: {name=}, {func=} with args: {args}, kwargs: {kwargs}")
             completion_event = asyncio.Event()
-            task_coro = self._run_with_retries_and_timeout(name, func, args, kwargs, queue, completion_event)            
+            last_heartbeat = {'time': time.time()}  # Shared mutable object for heartbeat tracking
+            
+            task_coro = self._run_with_retries_and_timeout(name, func, args, kwargs, queue, completion_event, last_heartbeat)            
             task = asyncio.create_task(task_coro)
-            heartbeat_coro = self._send_heartbeat(name, completion_event, queue)
+            heartbeat_coro = self._send_heartbeat(name, completion_event, queue, last_heartbeat)
             heartbeat_task = asyncio.create_task(heartbeat_coro)
             task_infos.append({
                 'name': name,
@@ -93,9 +122,12 @@ class AsyncTaskRunner:
                                             args: tuple,
                                             kwargs: dict,
                                             queue: asyncio.Queue, 
-                                            completion_event: asyncio.Event) -> None:
+                                            completion_event: asyncio.Event,
+                                            last_heartbeat: dict) -> None:
         try:
             log.info(f"run_with_retries_and_timeout: {name=}, {func=} with args: {args}, kwargs: {kwargs}")
+            log.info(f"Timeout mode: heartbeat_extends_timeout={self.heartbeat_extends_timeout}, timeout={self.timeout}s, hard_timeout={self.hard_timeout}s")
+            
             if self.retry_enabled:
                 retry_kwargs = {
                     'wait': wait_random_exponential(multiplier=1, max=60),
@@ -106,13 +138,13 @@ class AsyncTaskRunner:
                 async for attempt in AsyncRetrying(**retry_kwargs):
                     with attempt:
                         log.info(f"Starting task '{name}' with retry")
-                        result = await asyncio.wait_for(self._execute_task(func, *args, **kwargs), timeout=self.timeout)
+                        result = await self._execute_task_with_timeout(func, name, last_heartbeat, *args, **kwargs)
                         await queue.put({'type': 'task_complete', 'func_name': name, 'result': result})
                         log.info(f"Sent 'task_complete' message for task '{name}'")
                         return
             else:
                 log.info(f"Starting task '{name}' with no retry")
-                result = await asyncio.wait_for(self._execute_task(func, *args, **kwargs), timeout=self.timeout)
+                result = await self._execute_task_with_timeout(func, name, last_heartbeat, *args, **kwargs)
                 await queue.put({'type': 'task_complete', 'func_name': name, 'result': result})
                 log.info(f"Sent 'task_complete' message for task '{name}'")
         except asyncio.TimeoutError:
@@ -125,6 +157,55 @@ class AsyncTaskRunner:
             log.info(f"Task '{name}' completed.")
             completion_event.set()
 
+    async def _execute_task_with_timeout(self, func: Callable[..., Any], name: str, last_heartbeat: dict, *args: Any, **kwargs: Any) -> Any:
+        """
+        Execute task with either fixed timeout or heartbeat-extendable timeout.
+        """
+        if not self.heartbeat_extends_timeout:
+            # Original behavior - fixed timeout
+            return await asyncio.wait_for(self._execute_task(func, *args, **kwargs), timeout=self.timeout)
+        else:
+            # New behavior - heartbeat extends timeout
+            return await self._execute_task_with_heartbeat_timeout(func, name, last_heartbeat, *args, **kwargs)
+
+    async def _execute_task_with_heartbeat_timeout(self, func: Callable[..., Any], name: str, last_heartbeat: dict, *args: Any, **kwargs: Any) -> Any:
+        """
+        Execute task with heartbeat-extendable timeout and hard timeout limit.
+        """
+        start_time = time.time()
+        task = asyncio.create_task(self._execute_task(func, *args, **kwargs))
+        
+        while not task.done():
+            current_time = time.time()
+            
+            # Check hard timeout first (absolute limit)
+            if current_time - start_time > self.hard_timeout:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+                raise asyncio.TimeoutError(f"Hard timeout exceeded ({self.hard_timeout}s)")
+            
+            # Check soft timeout (extends with heartbeats)
+            time_since_heartbeat = current_time - last_heartbeat['time']
+            if time_since_heartbeat > self.timeout:
+                task.cancel()
+                try:
+                    await task
+                except asyncio.CancelledError:
+                    pass
+                raise asyncio.TimeoutError(f"Timeout exceeded - no heartbeat for {self.timeout}s")
+            
+            # Wait a bit before checking again
+            try:
+                await asyncio.wait_for(asyncio.shield(task), timeout=1.0)
+                break  # Task completed
+            except asyncio.TimeoutError:
+                continue  # Check timeouts again
+        
+        return await task
+
     async def _execute_task(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
         """
         Executes the given task function and returns its result.
@@ -143,14 +224,16 @@ class AsyncTaskRunner:
             else:
                 return await asyncio.to_thread(func, *args, **kwargs)
 
-    async def _send_heartbeat(self, func_name: str, completion_event: asyncio.Event, queue: asyncio.Queue, interval: int = 2):
+    async def _send_heartbeat(self, func_name: str, completion_event: asyncio.Event, queue: asyncio.Queue, last_heartbeat: dict, interval: int = 2):
         """
         Sends periodic heartbeat updates to indicate the task is still in progress.
+        Updates last_heartbeat time if heartbeat_extends_timeout is enabled.
 
         Args:
             func_name (str): The name of the task function.
             completion_event (asyncio.Event): Event to signal when the task is completed.
             queue (asyncio.Queue): The queue to send heartbeat messages to.
+            last_heartbeat (dict): Mutable dict containing the last heartbeat time.
             interval (int): How frequently to send heartbeat messages (in seconds).
         """
         start_time = time.time()
@@ -158,7 +241,14 @@ class AsyncTaskRunner:
         try:
             while not completion_event.is_set():
                 await asyncio.sleep(interval)
-                elapsed_time = int(time.time() - start_time)
+                current_time = time.time()
+                elapsed_time = int(current_time - start_time)
+                
+                # Update last heartbeat time if heartbeat extends timeout
+                if self.heartbeat_extends_timeout:
+                    last_heartbeat['time'] = current_time
+                    log.debug(f"Updated heartbeat time for task '{func_name}' at {current_time}")
+                
                 heartbeat_message = {
                     'type': 'heartbeat',
                     'name': func_name,

{sunholo-0.141.1.dist-info → sunholo-0.142.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sunholo
-Version: 0.141.1
+Version: 0.142.0
 Summary: AI DevOps - a package to help deploy GenAI to the Cloud.
 Author-email: Holosun ApS <multivac@sunholo.com>
 License: Apache License, Version 2.0

{sunholo-0.141.1.dist-info → sunholo-0.142.0.dist-info}/RECORD

@@ -96,7 +96,7 @@ sunholo/genai/init.py,sha256=yG8E67TduFCTQPELo83OJuWfjwTnGZsyACospahyEaY,687
 sunholo/genai/process_funcs_cls.py,sha256=D6eNrc3vtTZzwdkacZNOSfit499N_o0C5AHspyUJiYE,33690
 sunholo/genai/safety.py,sha256=mkFDO_BeEgiKjQd9o2I4UxB6XI7a9U-oOFjZ8LGRUC4,1238
 sunholo/invoke/__init__.py,sha256=o1RhwBGOtVK0MIdD55fAIMCkJsxTksi8GD5uoqVKI-8,184
-sunholo/invoke/async_class.py,sha256=OmRHaRf_yXNDWftWPpziytXh1TAhMumxnNhN_PGpQFs,8230
+sunholo/invoke/async_class.py,sha256=ZMzxKQtelbYibu9Fac7P9OU3GorH8KxawZxSMv5EO9A,12514
 sunholo/invoke/direct_vac_func.py,sha256=dACx3Zh7uZnuWLIFYiyLoyXUhh5-eUpd2RatDUd9ov8,9753
 sunholo/invoke/invoke_vac_utils.py,sha256=sJc1edHTHMzMGXjji1N67c3iUaP7BmAL5nj82Qof63M,2053
 sunholo/langfuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -169,9 +169,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
 sunholo/vertex/memory_tools.py,sha256=tBZxqVZ4InTmdBvLlOYwoSEWu4-kGquc-gxDwZCC4FA,7667
 sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
 sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
-sunholo-0.141.1.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
-sunholo-0.141.1.dist-info/METADATA,sha256=Ett7hxu4MPWMTgGQyVNADbvuuPOWvXIVUl2aRLXgNn8,18292
-sunholo-0.141.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-sunholo-0.141.1.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
-sunholo-0.141.1.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
-sunholo-0.141.1.dist-info/RECORD,,
+sunholo-0.142.0.dist-info/licenses/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
+sunholo-0.142.0.dist-info/METADATA,sha256=9v1AM0UCFJv_qx8vYOFO8gprTDulHCAxnMck7Jcmszs,18292
+sunholo-0.142.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+sunholo-0.142.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
+sunholo-0.142.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
+sunholo-0.142.0.dist-info/RECORD,,