provide-foundation 0.0.0.dev0__py3-none-any.whl

provide/foundation/observability/openobserve/streaming.py

@@ -0,0 +1,235 @@
+"""
+Streaming search operations for OpenObserve.
+"""
+
+from collections.abc import Generator
+import json
+import time
+from typing import Any
+
+import requests
+
+from provide.foundation.logger import get_logger
+from provide.foundation.observability.openobserve.auth import get_auth_headers
+from provide.foundation.observability.openobserve.client import OpenObserveClient
+from provide.foundation.observability.openobserve.exceptions import (
+    OpenObserveStreamingError,
+)
+from provide.foundation.observability.openobserve.models import parse_relative_time
+
+log = get_logger(__name__)
+
+
+def stream_logs(
+    sql: str,
+    start_time: str | int | None = None,
+    poll_interval: int = 5,
+    client: OpenObserveClient | None = None,
+) -> Generator[dict[str, Any], None, None]:
+    """Stream logs from OpenObserve with polling.
+
+    Continuously polls for new logs and yields them as they arrive.
+
+    Args:
+        sql: SQL query to execute
+        start_time: Initial start time
+        poll_interval: Seconds between polls
+        client: OpenObserve client
+
+    Yields:
+        Log entries as they arrive
+    """
+    if client is None:
+        client = OpenObserveClient.from_config()
+
+    # Track the last seen timestamp to avoid duplicates
+    last_timestamp = (
+        parse_relative_time(start_time) if start_time else parse_relative_time("-1m")
+    )
+    seen_ids = set()
+
+    log.info(f"Starting log stream with query: {sql}")
+
+    while True:
+        try:
+            # Search for new logs since last timestamp
+            response = client.search(
+                sql=sql,
+                start_time=last_timestamp,
+                end_time="now",
+                size=1000,
+            )
+
+            # Process new logs
+            new_count = 0
+            for hit in response.hits:
+                # Create a unique ID for deduplication
+                # Use combination of timestamp and a hash of content
+                timestamp = hit.get("_timestamp", 0)
+                log_id = f"{timestamp}:{hash(json.dumps(hit, sort_keys=True))}"
+
+                if log_id not in seen_ids:
+                    seen_ids.add(log_id)
+                    new_count += 1
+                    yield hit
+
+                    # Update last timestamp
+                    if timestamp > last_timestamp:
+                        last_timestamp = (
+                            timestamp + 1
+                        )  # Add 1 microsecond to avoid duplicates
+
+            if new_count > 0:
+                log.debug(f"Streamed {new_count} new log entries")
+
+            # Clean up old seen IDs to prevent memory growth
+            # Keep only IDs from the last minute
+            cutoff_time = parse_relative_time("-1m")
+            seen_ids = {lid for lid in seen_ids if int(lid.split(":")[0]) > cutoff_time}
+
+            # Wait before next poll
+            time.sleep(poll_interval)
+
+        except KeyboardInterrupt:
+            log.info("Stream interrupted by user")
+            break
+        except Exception as e:
+            log.error(f"Error during streaming: {e}")
+            raise OpenObserveStreamingError(f"Streaming failed: {e}")
+
+
+def stream_search_http2(
+    sql: str,
+    start_time: str | int | None = None,
+    end_time: str | int | None = None,
+    client: OpenObserveClient | None = None,
+) -> Generator[dict[str, Any], None, None]:
+    """Stream search results using HTTP/2 streaming endpoint.
+
+    This uses the native HTTP/2 streaming capability of OpenObserve
+    for real-time log streaming.
+
+    Args:
+        sql: SQL query to execute
+        start_time: Start time
+        end_time: End time
+        client: OpenObserve client
+
+    Yields:
+        Log entries as they stream
+    """
+    if client is None:
+        client = OpenObserveClient.from_config()
+
+    # Parse times
+    start_ts = (
+        parse_relative_time(start_time) if start_time else parse_relative_time("-1h")
+    )
+    end_ts = parse_relative_time(end_time) if end_time else parse_relative_time("now")
+
+    # Prepare request
+    url = f"{client.url}/api/{client.organization}/_search_stream"
+    params = {
+        "is_ui_histogram": "false",
+        "is_multi_stream_search": "false",
+    }
+    data = {
+        "sql": sql,
+        "start_time": start_ts,
+        "end_time": end_ts,
+    }
+
+    headers = get_auth_headers(client.username, client.password)
+
+    log.info(f"Starting HTTP/2 stream with query: {sql}")
+
+    try:
+        # Make streaming request
+        with requests.post(
+            url,
+            params=params,
+            json=data,
+            headers=headers,
+            stream=True,
+            timeout=client.timeout,
+        ) as response:
+            response.raise_for_status()
+
+            # Process streaming response
+            for line in response.iter_lines():
+                if line:
+                    try:
+                        # Decode and parse JSON line
+                        data = json.loads(line.decode("utf-8"))
+
+                        # Handle different response formats
+                        if isinstance(data, dict):
+                            if "hits" in data:
+                                # Batch of results
+                                for hit in data["hits"]:
+                                    yield hit
+                            else:
+                                # Single result
+                                yield data
+                    except json.JSONDecodeError as e:
+                        log.warning(f"Failed to parse stream line: {e}")
+                        continue
+
+    except requests.exceptions.RequestException as e:
+        raise OpenObserveStreamingError(f"HTTP/2 streaming failed: {e}")
+
+
+def tail_logs(
+    stream: str = "default",
+    filter_sql: str | None = None,
+    follow: bool = True,
+    lines: int = 10,
+    client: OpenObserveClient | None = None,
+) -> Generator[dict[str, Any], None, None]:
+    """Tail logs similar to 'tail -f' command.
+
+    Args:
+        stream: Stream name to tail
+        filter_sql: Optional SQL WHERE clause for filtering
+        follow: If True, continue streaming new logs
+        lines: Number of initial lines to show
+        client: OpenObserve client
+
+    Yields:
+        Log entries
+    """
+    # Build SQL query
+    where_clause = f"WHERE {filter_sql}" if filter_sql else ""
+    sql = (
+        f"SELECT * FROM {stream} {where_clause} ORDER BY _timestamp DESC LIMIT {lines}"
+    )
+
+    if client is None:
+        client = OpenObserveClient.from_config()
+
+    # Get initial logs
+    log.info(f"Fetching last {lines} logs from {stream}")
+    response = client.search(sql=sql, start_time="-1h")
+
+    # Yield initial logs in reverse order (oldest first)
+    for hit in reversed(response.hits):
+        yield hit
+
+    # If follow mode, continue streaming
+    if follow:
+        # Get the latest timestamp from initial results
+        if response.hits:
+            last_timestamp = max(hit.get("_timestamp", 0) for hit in response.hits)
+        else:
+            last_timestamp = parse_relative_time("-1s")
+
+        # Build streaming query
+        stream_sql = f"SELECT * FROM {stream} {where_clause} ORDER BY _timestamp ASC"
+
+        # Stream new logs
+        for log_entry in stream_logs(
+            sql=stream_sql,
+            start_time=last_timestamp + 1,
+            client=client,
+        ):
+            yield log_entry

provide/foundation/platform/__init__.py

@@ -0,0 +1,44 @@
+"""Platform detection and information utilities.
+
+Provides cross-platform detection and system information gathering.
+"""
+
+from provide.foundation.platform.detection import (
+    PlatformError,
+    get_arch_name,
+    get_cpu_type,
+    get_os_name,
+    get_os_version,
+    get_platform_string,
+    normalize_platform_components,
+)
+from provide.foundation.platform.info import (
+    SystemInfo,
+    get_system_info,
+    is_64bit,
+    is_arm,
+    is_linux,
+    is_macos,
+    is_windows,
+)
+
+__all__ = [
+    # Classes
+    "PlatformError",
+    "SystemInfo",
+    # Detection functions
+    "get_arch_name",
+    "get_cpu_type",
+    "get_os_name",
+    "get_os_version",
+    "get_platform_string",
+    "get_system_info",
+    # Platform checks
+    "is_64bit",
+    "is_arm",
+    "is_linux",
+    "is_macos",
+    "is_windows",
+    # Utilities
+    "normalize_platform_components",
+]

provide/foundation/platform/detection.py

@@ -0,0 +1,193 @@
+"""Core platform detection functions."""
+
+import platform
+import re
+
+from provide.foundation.errors.platform import PlatformError
+from provide.foundation.logger import get_logger
+
+plog = get_logger(__name__)
+
+
+def get_os_name() -> str:
+    """
+    Get normalized OS name.
+
+    Returns:
+        Normalized OS name (darwin, linux, windows)
+    """
+    try:
+        os_name = platform.system().lower()
+        if os_name in ("darwin", "macos"):
+            return "darwin"
+        return os_name
+    except Exception as e:
+        plog.error("Failed to detect OS", error=str(e))
+        raise PlatformError(
+            "Failed to detect operating system",
+            code="PLATFORM_OS_DETECTION_FAILED",
+            error=str(e),
+        ) from e
+
+
+def get_arch_name() -> str:
+    """
+    Get normalized architecture name.
+
+    Returns:
+        Normalized architecture (amd64, arm64, x86, i386)
+    """
+    try:
+        arch = platform.machine().lower()
+        # Normalize common architectures
+        if arch in ["x86_64", "amd64"]:
+            return "amd64"
+        elif arch in ["aarch64", "arm64"]:
+            return "arm64"
+        elif arch in ["i686", "i586", "i486"]:
+            return "x86"
+        return arch
+    except Exception as e:
+        plog.error("Failed to detect architecture", error=str(e))
+        raise PlatformError(
+            "Failed to detect architecture",
+            code="PLATFORM_ARCH_DETECTION_FAILED",
+            error=str(e),
+        ) from e
+
+
+def get_platform_string() -> str:
+    """
+    Get normalized platform string in format 'os_arch'.
+
+    Returns:
+        Platform string like 'darwin_arm64' or 'linux_amd64'
+    """
+    os_name = get_os_name()
+    arch = get_arch_name()
+    platform_str = f"{os_name}_{arch}"
+    plog.debug("Detected platform", platform=platform_str, os=os_name, arch=arch)
+    return platform_str
+
+
+def get_os_version() -> str | None:
+    """
+    Get OS version information.
+
+    Returns:
+        OS version string or None if unavailable
+    """
+    try:
+        system = platform.system()
+
+        if system == "Darwin":
+            # macOS version
+            mac_ver = platform.mac_ver()
+            if mac_ver[0]:
+                return mac_ver[0]
+        elif system == "Linux":
+            # Linux kernel version
+            release = platform.release()
+            if release:
+                # Extract major.minor version
+                parts = release.split(".")
+                if len(parts) >= 2:
+                    return f"{parts[0]}.{parts[1]}"
+                return release
+        elif system == "Windows":
+            # Windows version
+            version = platform.version()
+            if version:
+                return version
+
+        # Fallback to platform.release()
+        release = platform.release()
+        if release:
+            return release
+    except Exception as e:
+        plog.warning("Failed to detect OS version", error=str(e))
+
+    return None
+
+
+def get_cpu_type() -> str | None:
+    """
+    Get CPU type/family information.
+
+    Returns:
+        CPU type string or None if unavailable
+    """
+    try:
+        processor = platform.processor()
+        if processor:
+            # Clean up common processor strings
+            if "Intel" in processor:
+                # Extract Intel CPU model
+                if "Core" in processor:
+                    match = re.search(r"Core\(TM\)\s+(\w+)", processor)
+                    if match:
+                        return f"Intel Core {match.group(1)}"
+                return "Intel"
+            elif "AMD" in processor:
+                # Extract AMD CPU model
+                if "Ryzen" in processor:
+                    match = re.search(r"Ryzen\s+(\d+)", processor)
+                    if match:
+                        return f"AMD Ryzen {match.group(1)}"
+                return "AMD"
+            elif (
+                "Apple" in processor
+                or "M1" in processor
+                or "M2" in processor
+                or "M3" in processor
+            ):
+                # Apple Silicon
+                match = re.search(r"(M\d+\w*)", processor)
+                if match:
+                    return f"Apple {match.group(1)}"
+                return "Apple Silicon"
+            elif processor:
+                # Return cleaned processor string
+                return processor.strip()
+    except Exception as e:
+        plog.warning("Failed to detect CPU type", error=str(e))
+
+    return None
+
+
+def normalize_platform_components(os_name: str, arch_name: str) -> tuple[str, str]:
+    """
+    Normalize OS and architecture names to standard format.
+
+    Args:
+        os_name: Operating system name
+        arch_name: Architecture name
+
+    Returns:
+        Tuple of (normalized_os, normalized_arch)
+    """
+    # Normalize OS names
+    os_map = {
+        "linux": "linux",
+        "darwin": "darwin",
+        "macos": "darwin",
+        "windows": "windows",
+        "win32": "windows",
+    }
+
+    # Normalize architecture names
+    arch_map = {
+        "x86_64": "amd64",
+        "amd64": "amd64",
+        "aarch64": "arm64",
+        "arm64": "arm64",
+        "i686": "x86",
+        "i586": "x86",
+        "i486": "x86",
+        "i386": "i386",
+    }
+
+    normalized_os = os_map.get(os_name.lower(), os_name.lower())
+    normalized_arch = arch_map.get(arch_name.lower(), arch_name.lower())
+
+    return normalized_os, normalized_arch

provide/foundation/platform/info.py

@@ -0,0 +1,157 @@
+"""System information gathering utilities."""
+
+import contextlib
+import os
+import platform
+import shutil
+import sys
+
+from attrs import define
+
+from provide.foundation.logger import get_logger
+from provide.foundation.platform.detection import (
+    get_arch_name,
+    get_cpu_type,
+    get_os_name,
+    get_os_version,
+    get_platform_string,
+)
+
+plog = get_logger(__name__)
+
+
+@define
+class SystemInfo:
+    """System information container."""
+
+    os_name: str
+    arch: str
+    platform: str
+    os_version: str | None
+    cpu_type: str | None
+    python_version: str
+    hostname: str | None
+    username: str | None
+    home_dir: str | None
+    temp_dir: str | None
+    num_cpus: int | None
+    total_memory: int | None
+    available_memory: int | None
+    disk_usage: dict[str, dict[str, int]] | None
+
+
+def get_system_info() -> SystemInfo:
+    """
+    Gather comprehensive system information.
+
+    Returns:
+        SystemInfo object with all available system details
+    """
+    # Basic platform info
+    os_name = get_os_name()
+    arch = get_arch_name()
+    platform_str = get_platform_string()
+    os_version = get_os_version()
+    cpu_type = get_cpu_type()
+
+    # Python info
+    python_version = platform.python_version()
+
+    # System info
+    hostname = None
+    with contextlib.suppress(Exception):
+        hostname = platform.node()
+
+    # User info
+    username = os.environ.get("USER") or os.environ.get("USERNAME")
+    home_dir = os.path.expanduser("~")
+    temp_dir = os.environ.get("TMPDIR") or os.environ.get("TEMP") or "/tmp"
+
+    # CPU info
+    num_cpus = None
+    with contextlib.suppress(Exception):
+        num_cpus = os.cpu_count()
+
+    # Memory info (requires psutil for accurate values)
+    total_memory = None
+    available_memory = None
+    try:
+        import psutil
+
+        mem = psutil.virtual_memory()
+        total_memory = mem.total
+        available_memory = mem.available
+    except ImportError:
+        plog.debug("psutil not available, memory info limited")
+    except Exception as e:
+        plog.debug("Failed to get memory info", error=str(e))
+
+    # Disk usage
+    disk_usage = None
+    try:
+        disk_usage = {}
+        for path in ["/", home_dir, temp_dir]:
+            if os.path.exists(path):
+                usage = shutil.disk_usage(path)
+                disk_usage[path] = {
+                    "total": usage.total,
+                    "used": usage.used,
+                    "free": usage.free,
+                }
+    except Exception as e:
+        plog.debug("Failed to get disk usage", error=str(e))
+
+    info = SystemInfo(
+        os_name=os_name,
+        arch=arch,
+        platform=platform_str,
+        os_version=os_version,
+        cpu_type=cpu_type,
+        python_version=python_version,
+        hostname=hostname,
+        username=username,
+        home_dir=home_dir,
+        temp_dir=temp_dir,
+        num_cpus=num_cpus,
+        total_memory=total_memory,
+        available_memory=available_memory,
+        disk_usage=disk_usage,
+    )
+
+    plog.debug(
+        "System information gathered",
+        platform=platform_str,
+        os=os_name,
+        arch=arch,
+        python=python_version,
+        cpus=num_cpus,
+    )
+
+    return info
+
+
+# Platform detection functions
+def is_windows() -> bool:
+    """Check if running on Windows."""
+    return sys.platform.startswith("win")
+
+
+def is_macos() -> bool:
+    """Check if running on macOS."""
+    return sys.platform == "darwin"
+
+
+def is_linux() -> bool:
+    """Check if running on Linux."""
+    return sys.platform.startswith("linux")
+
+
+def is_arm() -> bool:
+    """Check if running on ARM architecture."""
+    machine = platform.machine().lower()
+    return "arm" in machine or "aarch" in machine
+
+
+def is_64bit() -> bool:
+    """Check if running on 64-bit architecture."""
+    return platform.machine().endswith("64") or sys.maxsize > 2**32

provide/foundation/process/__init__.py

@@ -0,0 +1,39 @@
+"""Process execution utilities.
+
+Provides sync and async subprocess execution with consistent error handling,
+and advanced process lifecycle management.
+"""
+
+from provide.foundation.errors.runtime import ProcessError
+from provide.foundation.process.async_runner import (
+    async_run_command,
+    async_run_shell,
+    async_stream_command,
+)
+from provide.foundation.process.lifecycle import (
+    ManagedProcess,
+    wait_for_process_output,
+)
+from provide.foundation.process.runner import (
+    CompletedProcess,
+    run_command,
+    run_shell,
+    stream_command,
+)
+
+__all__ = [
+    # Core types
+    "CompletedProcess",
+    "ProcessError",
+    # Sync execution
+    "run_command",
+    "run_shell",
+    "stream_command",
+    # Async execution
+    "async_run_command",
+    "async_run_shell",
+    "async_stream_command",
+    # Process lifecycle management
+    "ManagedProcess",
+    "wait_for_process_output",
+]