hexdag 0.5.0.dev1__py3-none-any.whl

hexdag/core/utils/async_warnings.py

@@ -0,0 +1,206 @@
+"""Runtime warnings for synchronous I/O operations in async contexts.
+
+This module provides utilities to detect and warn about blocking I/O operations
+at runtime when executing within async functions or coroutines.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import inspect
+import warnings
+from typing import TYPE_CHECKING, Any, TypeVar
+
+from hexdag.core.logging import get_logger
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+logger = get_logger(__name__)
+
+T = TypeVar("T")
+
+
+class AsyncIOWarning(UserWarning):
+    """Warning emitted when sync I/O is detected in async context."""
+
+    pass
+
+
+def _is_in_async_context() -> bool:
+    """Check if code is running in an async context.
+
+    Returns
+    -------
+        True if running in async context (event loop is running)
+    """
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False
+
+
+def warn_sync_io(operation: str, suggestion: str | None = None) -> None:
+    """Emit a warning about synchronous I/O in async context.
+
+    Args
+    ----
+        operation: Description of the blocking operation
+        suggestion: Optional suggestion for async alternative
+    """
+    if not _is_in_async_context():
+        return
+
+    frame = inspect.currentframe()
+    if frame and frame.f_back:
+        caller_frame = frame.f_back
+        filename = caller_frame.f_code.co_filename
+        line_number = caller_frame.f_lineno
+        function_name = caller_frame.f_code.co_name
+
+        message = (
+            f"Blocking I/O operation '{operation}' detected in async context "
+            f"(function: {function_name}, {filename}:{line_number})"
+        )
+
+        if suggestion:
+            message += f". {suggestion}"
+
+        warnings.warn(message, AsyncIOWarning, stacklevel=3)
+        logger.warning(message)
+
+
+def warn_if_async[T](func: Callable[..., T]) -> Callable[..., T]:
+    """Warn if a synchronous function is called in async context.
+
+    Args
+    ----
+        func: Function to wrap
+
+    Returns
+    -------
+        Wrapped function that warns if called in async context
+
+    Example
+    -------
+        >>> @warn_if_async
+        ... def sync_database_query(sql: str) -> list:
+        ...     return connection.execute(sql).fetchall()
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args: Any, **kwargs: Any) -> T:
+        if _is_in_async_context():
+            warn_sync_io(
+                f"{func.__name__}()",
+                f"Consider using an async version of {func.__name__}",
+            )
+        return func(*args, **kwargs)
+
+    return wrapper
+
+
+class SyncIOMonitor:
+    """Context manager to monitor and warn about sync I/O operations.
+
+    This can be used to wrap code sections that should use async I/O but might
+    accidentally use blocking operations.
+
+    Example
+    -------
+        >>> async def process_data():
+        ...     with SyncIOMonitor("data processing"):
+        ...         # Any blocking I/O here will trigger warnings
+        ...         data = process_file()  # Would warn if this blocks
+    """
+
+    def __init__(self, context_name: str = "code block") -> None:
+        """Initialize the monitor.
+
+        Args
+        ----
+            context_name: Name of the context being monitored
+        """
+        self.context_name = context_name
+        self.is_async = False
+
+    def __enter__(self) -> SyncIOMonitor:
+        """Enter the monitoring context."""
+        self.is_async = _is_in_async_context()
+        return self
+
+    def __exit__(
+        self,
+        _exc_type: Any,  # noqa: ARG002
+        _exc_val: Any,  # noqa: ARG002
+        _exc_tb: Any,  # noqa: ARG002
+    ) -> None:
+        """Exit the monitoring context."""
+        pass
+
+    def check_operation(self, operation: str, suggestion: str | None = None) -> None:
+        """Check and warn about an operation if in async context.
+
+        Args
+        ----
+            operation: Description of the operation
+            suggestion: Optional suggestion for async alternative
+        """
+        if self.is_async:
+            warn_sync_io(operation, suggestion)
+
+
+# Commonly monitored operations
+def warn_file_open(path: str) -> None:
+    """Warn about sync file open in async context.
+
+    Args
+    ----
+        path: File path being opened
+    """
+    warn_sync_io(
+        f"open('{path}')",
+        "Use aiofiles.open() for async file I/O",
+    )
+
+
+def warn_sqlite_connect(db_path: str) -> None:
+    """Warn about sync SQLite connection in async context.
+
+    Args
+    ----
+        db_path: Database path
+    """
+    warn_sync_io(
+        f"sqlite3.connect('{db_path}')",
+        "Use aiosqlite.connect() for async database operations",
+    )
+
+
+def warn_requests_call(method: str, url: str) -> None:
+    """Warn about sync HTTP request in async context.
+
+    Args
+    ----
+        method: HTTP method (GET, POST, etc.)
+        url: Request URL
+    """
+    warn_sync_io(
+        f"requests.{method.lower()}('{url}')",
+        "Use aiohttp.ClientSession for async HTTP requests",
+    )
+
+
+def warn_time_sleep(seconds: float) -> None:
+    """Warn about sync sleep in async context.
+
+    Args
+    ----
+        seconds: Sleep duration
+    """
+    warn_sync_io(
+        f"time.sleep({seconds})",
+        "Use await asyncio.sleep() in async functions",
+    )

hexdag/core/utils/schema_conversion.py

@@ -0,0 +1,78 @@
+"""Shared utilities for converting YAML schema strings to Python types.
+
+This module provides utilities used by both YAML pipeline builders and node factories
+to convert YAML-friendly type representations to actual Python types.
+"""
+
+from functools import singledispatch
+from typing import Any
+
+# Supported type name mappings
+VALID_TYPE_NAMES: dict[str, Any] = {
+    "str": str,
+    "int": int,
+    "float": float,
+    "bool": bool,
+    "list": list,
+    "dict": dict,
+    "Any": Any,
+}
+
+
+@singledispatch
+def normalize_schema(schema: Any) -> Any:
+    """Normalize schema to use Python types (accepts both string names and type objects).
+
+    Uses singledispatch to handle different input types elegantly.
+
+    Args:
+        schema: Schema in various formats (dict, type, Pydantic model, etc.)
+
+    Returns:
+        Normalized schema with actual Python type objects
+
+    Examples:
+        >>> normalize_schema({"name": "str"})  # YAML format
+        {'name': <class 'str'>}
+
+        >>> normalize_schema({"name": str})  # Already normalized
+        {'name': <class 'str'>}
+
+        >>> normalize_schema(str)  # Pass-through for types
+        <class 'str'>
+    """
+    # Default: pass through for types, Pydantic models, etc.
+    return schema
+
+
+@normalize_schema.register(dict)
+def _(schema: dict) -> dict[str, type]:
+    """Convert dict schema with string type names to Python types."""
+    converted: dict[str, Any] = {}
+
+    for key, value in schema.items():
+        if isinstance(value, str):
+            # String type name - convert to actual type
+            if value not in VALID_TYPE_NAMES:
+                valid_names = ", ".join(sorted(VALID_TYPE_NAMES.keys()))
+                raise ValueError(
+                    f"Invalid type '{value}' for field '{key}'. Supported types: {valid_names}"
+                )
+            converted[key] = VALID_TYPE_NAMES[value]
+        elif isinstance(value, dict):
+            # Nested schema - recurse
+            converted[key] = normalize_schema(value)
+        elif isinstance(value, type):
+            # Already a type - pass through
+            converted[key] = value
+        else:
+            raise ValueError(
+                f"Field '{key}' has invalid value: {value!r}. "
+                f"Expected type name string, type object, or nested schema dict."
+            )
+
+    return converted
+
+
+# Backward compatibility alias
+convert_yaml_schema = normalize_schema

hexdag/core/utils/sql_validation.py

@@ -0,0 +1,86 @@
+"""SQL validation utilities for preventing injection attacks."""
+
+import re
+
+from hexdag.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def validate_sql_identifier(
+    identifier: str,
+    identifier_type: str = "identifier",
+    raise_on_invalid: bool = False,
+) -> bool:
+    """Validate SQL identifier to prevent injection attacks.
+
+    SQL identifiers (table names, column names, etc.) must follow specific rules
+    to prevent SQL injection attacks. This function validates that an identifier
+    contains only safe characters.
+
+    Valid identifiers:
+    - Start with a letter (a-z, A-Z) or underscore (_)
+    - Contain only letters, numbers, and underscores
+    - Examples: "users", "user_data", "Table123", "_private"
+
+    Invalid identifiers:
+    - Start with numbers: "123table"
+    - Contain special characters: "user-data", "user.table", "user name"
+    - SQL keywords without quoting: "SELECT", "DROP"
+
+    Parameters
+    ----------
+    identifier : str
+        The SQL identifier to validate (e.g., table name, column name)
+    identifier_type : str, optional
+        Human-readable type name for error messages (default: "identifier")
+        Examples: "table", "column", "database"
+    raise_on_invalid : bool, optional
+        If True, raises ValueError on invalid identifier
+        If False, logs warning and returns False (default: False)
+
+    Returns
+    -------
+    bool
+        True if identifier is valid, False otherwise
+
+    Raises
+    ------
+    ValueError
+        If raise_on_invalid=True and identifier is invalid
+
+    Examples
+    --------
+    >>> validate_sql_identifier("users")
+    True
+
+    >>> validate_sql_identifier("user_data")
+    True
+
+    >>> validate_sql_identifier("123invalid")
+    False
+
+    >>> validate_sql_identifier("user-data")
+    False
+
+    >>> validate_sql_identifier("user.table", "table", raise_on_invalid=True)  # doctest: +SKIP
+    Traceback (most recent call last):
+        ...
+    ValueError: Invalid table 'user.table'. Must start with letter/underscore...
+    """
+    # Validate against safe pattern: starts with letter/underscore, contains only alphanumerics/_
+    is_valid = bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", identifier))
+
+    if not is_valid:
+        msg = (
+            f"Invalid {identifier_type} '{identifier}'. "
+            "Must start with letter/underscore and contain only "
+            "letters, numbers, and underscores."
+        )
+
+        if raise_on_invalid:
+            raise ValueError(msg)
+
+        logger.warning(msg)
+
+    return is_valid

hexdag/core/validation/secure_json.py

@@ -0,0 +1,148 @@
+import json
+import logging
+import re
+from dataclasses import dataclass
+from typing import Any, Literal
+
+import orjson
+
+ErrorCode = Literal[
+    "too_large",
+    "too_deep",
+    "invalid_syntax",
+    "unrecoverable",
+    "no_json_found",
+]
+
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SafeJSONResult:
+    data: Any | None = None
+    error: ErrorCode | None = None
+    message: str | None = None
+    line: int | None = None
+    col: int | None = None
+    preview: str | None = None
+
+    @property
+    def ok(self) -> bool:
+        return self.error is None
+
+
+class SafeJSON:
+    def __init__(self, max_size_bytes: int = 1_000_000, max_depth: int = 20):
+        self.max_size_bytes = max_size_bytes
+        self.max_depth = max_depth
+
+    def loads(self, data: str | bytes | bytearray) -> SafeJSONResult:
+        text = (
+            data.decode("utf-8", errors="strict")
+            if isinstance(data, (bytes, bytearray))
+            else str(data)
+        )
+
+        if len(text.encode("utf-8")) > self.max_size_bytes:
+            return SafeJSONResult(error="too_large", message="JSON exceeds size limit")
+
+        if self._estimate_depth(text) > self.max_depth:
+            return SafeJSONResult(error="too_deep", message="JSON exceeds depth limit")
+
+        # Step 1: try orjson
+        try:
+            return SafeJSONResult(data=orjson.loads(text))
+        except Exception as exc:
+            logger.debug("orjson initial parse failed; attempting cleanup fallback: %s", exc)
+
+        # Step 2: cleanup + retry
+        cleaned = self._cleanup(text)
+        if (
+            len(cleaned.encode("utf-8")) <= self.max_size_bytes
+            and self._estimate_depth(cleaned) <= self.max_depth
+        ):
+            try:
+                return SafeJSONResult(data=orjson.loads(cleaned))
+            except Exception as exc:  # noqa: BLE001 - we deliberately log and fall back
+                logger.debug(
+                    "orjson cleaned parse failed; falling back to stdlib for diagnostics: %s", exc
+                )
+
+        # Step 3: stdlib json for diagnostics
+        try:
+            return SafeJSONResult(data=json.loads(cleaned, parse_constant=lambda _: None))
+        except json.JSONDecodeError as e:
+            preview = self._format_error_line(cleaned, e.lineno, e.colno)
+            return SafeJSONResult(
+                error="invalid_syntax",
+                message=e.msg,
+                line=e.lineno,
+                col=e.colno,
+                preview=preview,
+            )
+        except Exception:
+            return SafeJSONResult(error="unrecoverable", message="Unrecoverable JSON")
+
+    def loads_from_text(self, text: str) -> SafeJSONResult:
+        candidate = self._extract_json(text)
+        if not candidate:
+            return SafeJSONResult(error="no_json_found", message="No JSON found in text")
+        return self.loads(candidate)
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _cleanup(text: str) -> str:
+        text = re.sub(r"(?m)\s*(//|#).*?$", "", text)
+        text = re.sub(r",\s*([}\]])", r"\1", text)
+        return re.sub(r"(?<!\\)'([^']*?)'(?!\\)", r'"\1"', text)
+
+    @staticmethod
+    def _extract_json(text: str) -> str | None:
+        if not text:
+            return None
+        match = re.search(r"```json\s*([\s\S]*?)```", text, re.IGNORECASE)
+        if match:
+            return match.group(1).strip()
+        match = re.search(r"```[\w-]*\s*([\s\S]*?)```", text)
+        if match and match.group(1).lstrip().startswith(("{", "[")):
+            return match.group(1).strip()
+        match = re.search(r"[\[{][\s\S]*[\]}]", text)
+        return match.group(0).strip() if match else None
+
+    @staticmethod
+    def _estimate_depth(text: str) -> int:
+        depth = 0
+        max_depth = 0
+        in_str: str | None = None
+        esc = False
+        for ch in text:
+            if in_str:
+                if esc:
+                    esc = False
+                elif ch == "\\":
+                    esc = True
+                elif ch == in_str:
+                    in_str = None
+                continue
+            if ch in ('"', "'"):
+                in_str = ch
+            elif ch in "{[":
+                depth += 1
+                max_depth = max(max_depth, depth)
+            elif ch in "]}":
+                depth = max(depth - 1, 0)
+        return max_depth
+
+    @staticmethod
+    def _format_error_line(text: str, line_no: int, col_no: int, context: int = 1) -> str | None:
+        """Return a snippet of the line with a caret pointing at the error col."""
+        lines = text.splitlines()
+        if 1 <= line_no <= len(lines):
+            line = lines[line_no - 1]
+            caret_line = " " * (col_no - 1) + "^"
+            return f"{line}\n{caret_line}"
+        return None