qalita 2.9.2__py3-none-any.whl → 2.10.1__py3-none-any.whl

qalita/grpc/client.py

@@ -26,6 +26,7 @@ class GrpcClient:
     - Keep-alive management
     - Bidirectional streaming support
     - Thread-safe connection state
+    - Stability detection before resetting reconnection counter
     """
     
     def __init__(
@@ -36,6 +37,7 @@ class GrpcClient:
         max_reconnect_attempts: int = 10,
         initial_reconnect_delay: float = 1.0,
         max_reconnect_delay: float = 60.0,
+        stability_threshold_seconds: float = 30.0,
     ):
         """
         Initialize the gRPC client.
@@ -47,6 +49,7 @@ class GrpcClient:
             max_reconnect_attempts: Maximum reconnection attempts (0 = unlimited)
             initial_reconnect_delay: Initial delay between reconnection attempts
             max_reconnect_delay: Maximum delay between reconnection attempts
+            stability_threshold_seconds: Time the connection must be stable before resetting attempts counter
         """
         self._url = url
         self._token = token
@@ -54,6 +57,7 @@ class GrpcClient:
         self._max_reconnect_attempts = max_reconnect_attempts
         self._initial_reconnect_delay = initial_reconnect_delay
         self._max_reconnect_delay = max_reconnect_delay
+        self._stability_threshold_seconds = stability_threshold_seconds
         
         # Connection state - set before parsing URL
         self._use_secure_channel = False
@@ -66,12 +70,21 @@ class GrpcClient:
         self._stub: Optional[qalita_pb2_grpc.WorkerServiceStub] = None
         self._connected = False
         self._reconnect_attempts = 0
+        self._current_reconnect_delay = initial_reconnect_delay
+        self._last_successful_stream_start: Optional[datetime] = None
+        self._stream_healthy = False
         
         # Stream state
         self._stream_call = None
         self._outgoing_queue: asyncio.Queue = asyncio.Queue()
         self._stream_active = False
         
+        # Stream health monitoring
+        self._last_message_received: Optional[datetime] = None
+        self._last_message_sent: Optional[datetime] = None
+        self._stream_health_timeout = 45.0  # Consider stream dead if no response in 45s
+        self._force_reconnect = False
+        
         # Callbacks
         self._on_job_received: Optional[Callable] = None
         self._on_routine_received: Optional[Callable] = None
@@ -129,10 +142,22 @@ class GrpcClient:
         """
         Establish connection to the gRPC server.
         
+        Note: This method does NOT reset _reconnect_attempts. The counter is only
+        reset after the stream has been stable for _stability_threshold_seconds.
+        
         Returns:
             True if connection successful, False otherwise
         """
         try:
+            # Close any existing channel first
+            if self._channel:
+                try:
+                    await self._channel.close()
+                except Exception:
+                    pass
+                self._channel = None
+                self._stub = None
+            
             # Channel options for long-running streams
             channel_options = [
                 ('grpc.keepalive_time_ms', 30000),
@@ -142,6 +167,10 @@ class GrpcClient:
                 ('grpc.http2.max_pings_without_data', 0),
                 ('grpc.max_receive_message_length', 50 * 1024 * 1024),
                 ('grpc.max_send_message_length', 50 * 1024 * 1024),
+                # Additional options for better connection resilience
+                ('grpc.initial_reconnect_backoff_ms', 1000),
+                ('grpc.max_reconnect_backoff_ms', 60000),
+                ('grpc.enable_retries', 1),
             ]
             
             # Create channel - secure for production, insecure for local dev
@@ -160,7 +189,7 @@ class GrpcClient:
             
             self._stub = qalita_pb2_grpc.WorkerServiceStub(self._channel)
             self._connected = True
-            self._reconnect_attempts = 0
+            # Note: Do NOT reset _reconnect_attempts here - only reset after stable stream
             
             logger.info(f"Connected to gRPC server at {self._grpc_target}")
             return True
@@ -197,31 +226,102 @@ class GrpcClient:
         """
         Attempt to reconnect with exponential backoff.
         
+        The reconnection counter persists across reconnection cycles. It only resets
+        when the connection has been stable (stream healthy for _stability_threshold_seconds).
+        
         Returns:
             True if reconnection successful, False if max attempts exceeded
         """
-        delay = self._initial_reconnect_delay
+        self._reconnect_attempts += 1
+        self._stream_healthy = False
+        
+        # Check if max attempts exceeded
+        if self._max_reconnect_attempts > 0 and self._reconnect_attempts > self._max_reconnect_attempts:
+            logger.error(
+                f"Max reconnection attempts exceeded ({self._reconnect_attempts}/{self._max_reconnect_attempts}). "
+                f"Will continue trying with max backoff delay."
+            )
+            # Don't return False - keep trying but with max delay
+            # In production, we want the worker to eventually recover
         
-        while (self._max_reconnect_attempts == 0 or 
-               self._reconnect_attempts < self._max_reconnect_attempts):
-            
-            self._reconnect_attempts += 1
-            logger.warning(
-                f"Reconnection attempt {self._reconnect_attempts}"
-                f"{f'/{self._max_reconnect_attempts}' if self._max_reconnect_attempts > 0 else ''}"
+        logger.warning(
+            f"Reconnection attempt {self._reconnect_attempts}"
+            f"{f'/{self._max_reconnect_attempts}' if self._max_reconnect_attempts > 0 else ''} "
+            f"(delay: {self._current_reconnect_delay:.1f}s)"
+        )
+        
+        # Wait before attempting reconnection (exponential backoff)
+        await asyncio.sleep(self._current_reconnect_delay)
+        
+        # Attempt to connect
+        if await self.connect():
+            # Increase delay for next attempt (in case this stream also fails quickly)
+            self._current_reconnect_delay = min(
+                self._current_reconnect_delay * 2,
+                self._max_reconnect_delay
             )
-            
-            await asyncio.sleep(delay)
-            
-            if await self.connect():
-                return True
-            
-            # Exponential backoff
-            delay = min(delay * 2, self._max_reconnect_delay)
+            return True
         
-        logger.error("Max reconnection attempts exceeded")
+        # Connection failed, increase delay for next attempt
+        self._current_reconnect_delay = min(
+            self._current_reconnect_delay * 2,
+            self._max_reconnect_delay
+        )
         return False
     
+    def _mark_stream_stable(self) -> None:
+        """
+        Mark the stream as stable and reset reconnection counters.
+        
+        Called when the stream has been healthy for _stability_threshold_seconds.
+        """
+        if not self._stream_healthy:
+            logger.info("Stream connection is now stable - resetting reconnection counters")
+            self._stream_healthy = True
+            self._reconnect_attempts = 0
+            self._current_reconnect_delay = self._initial_reconnect_delay
+    
+    async def _check_stream_health(self) -> None:
+        """
+        Check if the stream is actually working by comparing sent vs received timestamps.
+        
+        If we've been sending messages but haven't received any response (ack or other)
+        for _stream_health_timeout seconds, the stream is probably dead and we should reconnect.
+        """
+        now = datetime.now(timezone.utc)
+        
+        # Need both timestamps to make a comparison
+        if not self._last_message_sent:
+            return
+        
+        # Calculate time since last message sent and received
+        time_since_sent = (now - self._last_message_sent).total_seconds()
+        
+        if self._last_message_received:
+            time_since_received = (now - self._last_message_received).total_seconds()
+        else:
+            # Never received anything - use time since stream started
+            if self._last_successful_stream_start:
+                time_since_received = (now - self._last_successful_stream_start).total_seconds()
+            else:
+                return
+        
+        # If we've been sending but not receiving for too long, stream is dead
+        if time_since_received > self._stream_health_timeout:
+            logger.warning(
+                f"Stream appears dead: last sent {time_since_sent:.1f}s ago, "
+                f"last received {time_since_received:.1f}s ago (timeout: {self._stream_health_timeout}s)"
+            )
+            logger.warning("Forcing reconnection due to unresponsive stream...")
+            self._force_reconnect = True
+            
+            # Cancel the stream call to force the error path
+            if self._stream_call:
+                try:
+                    self._stream_call.cancel()
+                except Exception as e:
+                    logger.debug(f"Error cancelling stream for forced reconnect: {e}")
+    
     # =========================================================================
     # Unary RPCs
     # =========================================================================
@@ -646,12 +746,14 @@ class GrpcClient:
     async def _outgoing_messages(self) -> AsyncIterator[qalita_pb2.WorkerMessage]:
         """Generator for outgoing stream messages."""
         logger.info("Outgoing messages generator started")
-        while self._stream_active:
+        while self._stream_active and not self._force_reconnect:
             try:
                 # Use get_nowait in a loop with sleep to avoid blocking gRPC
                 try:
                     msg = self._outgoing_queue.get_nowait()
-                    logger.debug(f"Yielding message type: {msg.WhichOneof('payload')}")
+                    msg_type = msg.WhichOneof('payload')
+                    logger.debug(f"Yielding message type: {msg_type}")
+                    self._last_message_sent = datetime.now(timezone.utc)
                     yield msg
                 except asyncio.QueueEmpty:
                     # No message available, yield control briefly
@@ -672,7 +774,9 @@ class GrpcClient:
         - Keep-alive signals (sent every 10 seconds)
         - Incoming job assignments
         - Incoming routine triggers
-        - Automatic reconnection on failure
+        - Automatic reconnection on failure with exponential backoff
+        - Stability detection to reset reconnection counters
+        - Dead stream detection (sending but not receiving)
         """
         if not self._connected:
             if not await self.connect():
@@ -681,14 +785,30 @@ class GrpcClient:
         # Recreate queue in async context to ensure proper event loop binding
         self._outgoing_queue = asyncio.Queue()
         self._stream_active = True
+        self._stream_healthy = False
+        self._last_successful_stream_start = None
+        self._last_message_received = None
+        self._last_message_sent = None
+        self._force_reconnect = False
         
         async def keep_alive_loop():
-            """Send keep-alive every 10 seconds."""
+            """Send keep-alive every 10 seconds and monitor stream health."""
             logger.info(f"Keep-alive loop started, worker_id={self._worker_id}")
-            while self._stream_active:
+            while self._stream_active and not self._force_reconnect:
                 try:
                     logger.debug(f"Sending keep-alive for worker {self._worker_id}")
                     await self.send_keep_alive()
+                    
+                    # Check if stream has been healthy long enough to reset counters
+                    if (self._last_successful_stream_start and 
+                        not self._stream_healthy):
+                        elapsed = (datetime.now(timezone.utc) - self._last_successful_stream_start).total_seconds()
+                        if elapsed >= self._stability_threshold_seconds:
+                            self._mark_stream_stable()
+                    
+                    # Health check: detect dead stream (sending but not receiving)
+                    await self._check_stream_health()
+                    
                     await asyncio.sleep(10)
                 except asyncio.CancelledError:
                     logger.info("Keep-alive loop cancelled")
@@ -696,15 +816,28 @@ class GrpcClient:
                 except Exception as e:
                     logger.error(f"Keep-alive error: {e}")
         
-        async def process_stream():
-            """Process incoming stream messages."""
+        async def process_single_stream() -> bool:
+            """
+            Process incoming stream messages for one connection attempt.
+            
+            Returns:
+                True if stream ended gracefully (should not reconnect)
+                False if stream had an error (should attempt reconnection)
+            """
             try:
                 self._stream_call = self._stub.Connect(
                     self._outgoing_messages(),
                     metadata=self.metadata,
                 )
                 
+                # Mark the time when stream successfully started
+                self._last_successful_stream_start = datetime.now(timezone.utc)
+                logger.info("Stream established successfully")
+                
                 async for msg in self._stream_call:
+                    # Each message received confirms the stream is working
+                    self._last_message_received = datetime.now(timezone.utc)
+                    
                     if msg.HasField('job_assignment'):
                         job = msg.job_assignment.job
                         logger.info(f"Received job assignment: {job.id}")
@@ -737,26 +870,88 @@ class GrpcClient:
                     
                     elif msg.HasField('ack'):
                         logger.debug(f"Received ack: {msg.ack.message_type}")
+                        # Ack received means stream is working, check stability
+                        if (self._last_successful_stream_start and 
+                            not self._stream_healthy):
+                            elapsed = (datetime.now(timezone.utc) - self._last_successful_stream_start).total_seconds()
+                            if elapsed >= self._stability_threshold_seconds:
+                                self._mark_stream_stable()
                     
                     elif msg.HasField('error'):
                         logger.error(f"Server error: {msg.error.code} - {msg.error.message}")
+                
+                # Stream ended normally (server closed it gracefully)
+                logger.info("Stream ended normally")
+                return False  # Still try to reconnect for continuous operation
+            
+            except asyncio.CancelledError:
+                # CancelledError is a BaseException in Python 3.8+, must catch explicitly
+                if self._force_reconnect:
+                    logger.info("Stream cancelled due to forced reconnect (dead stream detection)")
+                    return False  # Reconnect
+                else:
+                    logger.info("Stream cancelled by client")
+                    return True  # Don't reconnect if we intentionally cancelled it
             
             except grpc.aio.AioRpcError as e:
                 if e.code() == grpc.StatusCode.CANCELLED:
-                    logger.info("Stream cancelled")
+                    if self._force_reconnect:
+                        logger.info("Stream cancelled due to forced reconnect (dead stream detection)")
+                        return False  # Reconnect
+                    else:
+                        logger.info("Stream cancelled by client")
+                        return True  # Don't reconnect if we intentionally cancelled it
                 else:
-                    logger.error(f"Stream error: {e.code()} - {e.details()}")
-                    # Attempt reconnection
-                    if self._stream_active and await self._reconnect():
-                        await process_stream()
+                    # Calculate how long the stream was alive
+                    stream_duration = 0
+                    if self._last_successful_stream_start:
+                        stream_duration = (datetime.now(timezone.utc) - self._last_successful_stream_start).total_seconds()
+                    
+                    logger.error(
+                        f"Stream error after {stream_duration:.1f}s: {e.code()} - {e.details()}"
+                    )
+                    return False  # Should attempt reconnection
+            
+            except Exception as e:
+                logger.error(f"Unexpected stream error: {e}")
+                return False  # Should attempt reconnection
         
-        # Run keep-alive and stream processing concurrently
+        # Main stream loop with reconnection handling
         keep_alive_task = asyncio.create_task(keep_alive_loop())
         
         try:
-            await process_stream()
+            while self._stream_active:
+                # Reset state before starting/restarting stream
+                self._force_reconnect = False
+                self._last_message_received = None
+                self._last_message_sent = None
+                
+                # Process the stream
+                should_stop = await process_single_stream()
+                
+                if should_stop or not self._stream_active:
+                    break
+                
+                # Stream failed, attempt reconnection
+                self._last_successful_stream_start = None
+                
+                # Recreate the outgoing queue to clear any stale messages
+                self._outgoing_queue = asyncio.Queue()
+                
+                # Attempt reconnection (this handles backoff)
+                if not await self._reconnect():
+                    # _reconnect now always returns True after sleeping and connecting
+                    # It only returns False if connect() itself fails
+                    # In that case, keep trying
+                    logger.warning("Reconnection failed, will retry...")
+                    continue
+                
+                # Reconnected successfully, loop will start a new stream
+                logger.info("Reconnected, restarting stream...")
+                
         finally:
             self._stream_active = False
+            self._force_reconnect = True  # Stop the outgoing generator
             keep_alive_task.cancel()
             try:
                 await keep_alive_task

qalita/internal/action_executor.py

@@ -36,7 +36,7 @@ class ActionResult:
 
 # Supported action types
 ACTION_TYPES = {
-    "query": "Execute a SQL query on a database source",
+    "query": "Execute a SQL query on any data source (database or file via pandasql)",
     "read_data": "Read data from a file or database source",
     "filter": "Filter data based on conditions",
     "aggregate": "Perform aggregation on data",
@@ -48,6 +48,14 @@ ACTION_TYPES = {
     "tail": "Get last N rows from a source",
 }
 
+# Check if pandasql is available for file SQL queries
+_PANDASQL_AVAILABLE = False
+try:
+    import pandasql
+    _PANDASQL_AVAILABLE = True
+except ImportError:
+    pass
+
 
 class ActionExecutor:
     """
@@ -171,14 +179,7 @@ class ActionExecutor:
         return source_type in ("file", "csv", "excel", "parquet", "json", "folder")
     
     def _execute_query(self, source_config: dict, params: dict) -> ActionResult:
-        """Execute a SQL query on a database source."""
-        if not self._is_database_source(source_config):
-            return ActionResult(
-                ok=False,
-                action_type="query",
-                error=f"Query action only supported for database sources, not {source_config.get('type')}",
-            )
-        
+        """Execute a SQL query on any data source (database or file via pandasql)."""
         sql = params.get("sql")
         if not sql:
             return ActionResult(
@@ -189,11 +190,32 @@ class ActionExecutor:
         
         limit = params.get("limit", DEFAULT_ROW_LIMIT)
         
-        # Add LIMIT if not present (for safety)
+        # Security: reject modification queries
         sql_lower = sql.strip().lower()
-        if "limit" not in sql_lower and not sql_lower.startswith(("insert", "update", "delete", "create", "drop", "alter")):
+        if sql_lower.startswith(("insert", "update", "delete", "create", "drop", "alter", "truncate")):
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error="Modification queries (INSERT, UPDATE, DELETE, etc.) are not allowed",
+            )
+        
+        # Add LIMIT if not present (for safety)
+        if "limit" not in sql_lower:
             sql = f"{sql.rstrip(';')} LIMIT {limit}"
         
+        if self._is_database_source(source_config):
+            return self._execute_database_query(source_config, sql, limit)
+        elif self._is_file_source(source_config):
+            return self._execute_file_query(source_config, sql, limit)
+        else:
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error=f"Query action not supported for source type: {source_config.get('type')}",
+            )
+    
+    def _execute_database_query(self, source_config: dict, sql: str, limit: int) -> ActionResult:
+        """Execute a SQL query on a database source."""
         try:
             engine = self._get_database_engine(source_config)
             with engine.connect() as conn:
@@ -213,6 +235,97 @@ class ActionExecutor:
                 error=f"Query execution failed: {str(e)}",
             )
     
+    def _execute_file_query(self, source_config: dict, sql: str, limit: int) -> ActionResult:
+        """Execute a SQL query on a file source using pandasql."""
+        import os
+        
+        if not _PANDASQL_AVAILABLE:
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error="SQL queries on file sources require 'pandasql'. Install with: pip install pandasql",
+            )
+        
+        config = source_config.get("config", {})
+        source_type = source_config.get("type", "").lower()
+        path = config.get("path")
+        
+        if not path:
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error="File path not configured",
+            )
+        
+        if not os.path.exists(path):
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error=f"File not found: {path}",
+            )
+        
+        try:
+            # Load the file into a DataFrame
+            source_data = self._load_file_to_dataframe(path, source_type)
+            
+            if source_data is None:
+                return ActionResult(
+                    ok=False,
+                    action_type="query",
+                    error=f"Unsupported file type: {source_type}",
+                )
+            
+            # Execute the SQL query using pandasql
+            # The user must use 'source_data' as the table name in their query
+            from pandasql import sqldf
+            
+            # Create a local namespace for pandasql
+            local_env = {"source_data": source_data}
+            result_df = sqldf(sql, local_env)
+            
+            # Apply limit if result is larger
+            if len(result_df) > limit:
+                result_df = result_df.head(limit)
+            
+            preview = _dataframe_to_preview(result_df, limit)
+            return ActionResult(
+                ok=True,
+                action_type="query",
+                data=preview,
+                result_json=json.dumps({
+                    "rows_returned": len(result_df),
+                    "columns": list(result_df.columns),
+                    "source_rows": len(source_data),
+                }),
+            )
+        except Exception as e:
+            error_msg = str(e)
+            # Provide helpful hint if table name is wrong
+            if "no such table" in error_msg.lower():
+                error_msg += ". Hint: Use 'source_data' as the table name, e.g., SELECT * FROM source_data"
+            return ActionResult(
+                ok=False,
+                action_type="query",
+                error=f"Query execution failed: {error_msg}",
+            )
+    
+    def _load_file_to_dataframe(self, path: str, source_type: str) -> Optional[pd.DataFrame]:
+        """Load a file into a pandas DataFrame for SQL querying."""
+        try:
+            if source_type in ("csv", "file") or path.endswith(".csv"):
+                return pd.read_csv(path, low_memory=False)
+            elif source_type == "excel" or path.endswith((".xlsx", ".xls")):
+                return pd.read_excel(path, engine="openpyxl")
+            elif source_type == "parquet" or path.endswith(".parquet"):
+                return pd.read_parquet(path)
+            elif source_type == "json" or path.endswith(".json"):
+                return pd.read_json(path)
+            else:
+                return None
+        except Exception as e:
+            logger.error(f"Failed to load file {path}: {e}")
+            return None
+    
     def _read_data(self, source_config: dict, params: dict) -> ActionResult:
         """Read data from a source."""
         limit = params.get("limit", DEFAULT_ROW_LIMIT)

qalita/internal/config.py

@@ -51,6 +51,13 @@ class Config(object):
                 logger.info(f"Loading source configuration from [{abs_path}]")
             with open(abs_path, "r") as file:
                 self.config = yaml.safe_load(file)
+                # Handle empty file or invalid YAML that returns None
+                if self.config is None:
+                    logger.warning(
+                        f"Configuration file [{abs_path}] is empty, initializing with default structure."
+                    )
+                    self.config = {"version": 1, "sources": []}
+                    self.save_source_config()
                 return self.config
         except FileNotFoundError:
             logger.warning(

qalita/internal/utils.py

@@ -224,7 +224,7 @@ def safe_path_check(path: str) -> str:  # lgtm[py/path-injection]
 
 
 def get_version():
-    return "2.9.2"
+    return "2.10.1"
 
 
 def make_tarfile(output_filename, source_dir):

{qalita-2.9.2.dist-info → qalita-2.10.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: qalita
-Version: 2.9.2
+Version: 2.10.1
 Summary: QALITA Platform Command Line Interface
 Author-email: QALITA SAS <contact@qalita.io>
 License-File: LICENSE
@@ -34,6 +34,7 @@ Requires-Dist: loguru>=0.7.0
 Requires-Dist: openpyxl>=3.1.5
 Requires-Dist: oracledb>=2.5.0
 Requires-Dist: pandas>=2.0.0
+Requires-Dist: pandasql>=0.7.3
 Requires-Dist: paramiko>=3.4.0
 Requires-Dist: psycopg2-binary>=2.9.9
 Requires-Dist: pyarrow>=14.0.0