provide-foundation 0.0.0.dev0__py3-none-any.whl

provide/foundation/observability/openobserve/commands.py

@@ -0,0 +1,357 @@
+"""
+CLI commands for OpenObserve integration.
+
+These commands are auto-registered by Foundation's command discovery system.
+"""
+
+try:
+    import click
+
+    _HAS_CLICK = True
+except ImportError:
+    click = None
+    _HAS_CLICK = False
+
+from provide.foundation.logger import get_logger
+
+log = get_logger(__name__)
+
+
+if _HAS_CLICK:
+    from provide.foundation.observability.openobserve import (
+        OpenObserveClient,
+        format_output,
+        search_logs,
+        tail_logs,
+    )
+
+    @click.group("openobserve", help="Query and manage OpenObserve logs")
+    @click.pass_context
+    def openobserve_group(ctx):
+        """OpenObserve log querying and streaming commands."""
+        # Initialize client and store in context
+        try:
+            client = OpenObserveClient.from_config()
+            ctx.obj = client
+        except Exception as e:
+            log.warning(f"Failed to initialize OpenObserve client: {e}")
+            ctx.obj = None
+
+    @openobserve_group.command("query")
+    @click.option(
+        "--sql",
+        required=True,
+        help="SQL query to execute",
+    )
+    @click.option(
+        "--start",
+        "-s",
+        default="-1h",
+        help="Start time (e.g., -1h, -30m, 2024-01-01)",
+    )
+    @click.option(
+        "--end",
+        "-e",
+        default="now",
+        help="End time (e.g., now, -5m, 2024-01-02)",
+    )
+    @click.option(
+        "--size",
+        "-n",
+        default=100,
+        type=int,
+        help="Number of results to return",
+    )
+    @click.option(
+        "--format",
+        "-f",
+        type=click.Choice(["json", "log", "table", "csv", "summary"]),
+        default="log",
+        help="Output format",
+    )
+    @click.option(
+        "--pretty",
+        is_flag=True,
+        help="Pretty print JSON output",
+    )
+    @click.pass_obj
+    def query_command(client, sql, start, end, size, format, pretty):
+        """Execute SQL query against OpenObserve logs."""
+        if client is None:
+            click.echo(
+                "OpenObserve not configured. Set OPENOBSERVE_URL, OPENOBSERVE_USER, and OPENOBSERVE_PASSWORD.",
+                err=True,
+            )
+            return 1
+
+        try:
+            response = search_logs(
+                sql=sql,
+                start_time=start,
+                end_time=end,
+                size=size,
+                client=client,
+            )
+
+            output = format_output(response, format_type=format, pretty=pretty)
+            click.echo(output)
+
+        except Exception as e:
+            click.echo(f"Query failed: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("tail")
+    @click.option(
+        "--stream",
+        "-s",
+        default="default",
+        help="Stream name to tail",
+    )
+    @click.option(
+        "--filter",
+        "-f",
+        "filter_sql",
+        help="SQL WHERE clause for filtering (e.g., \"level='ERROR'\")",
+    )
+    @click.option(
+        "--lines",
+        "-n",
+        default=10,
+        type=int,
+        help="Number of initial lines to show",
+    )
+    @click.option(
+        "--follow",
+        "-F",
+        is_flag=True,
+        default=True,
+        help="Follow mode (like tail -f)",
+    )
+    @click.option(
+        "--format",
+        type=click.Choice(["log", "json"]),
+        default="log",
+        help="Output format",
+    )
+    @click.pass_obj
+    def tail_command(client, stream, filter_sql, lines, follow, format):
+        """Tail logs from OpenObserve (like 'tail -f')."""
+        if client is None:
+            click.echo(
+                "OpenObserve not configured. Set OPENOBSERVE_URL, OPENOBSERVE_USER, and OPENOBSERVE_PASSWORD.",
+                err=True,
+            )
+            return 1
+
+        try:
+            click.echo(f"Tailing logs from stream '{stream}'...")
+            if filter_sql:
+                click.echo(f"Filter: {filter_sql}")
+
+            for log_entry in tail_logs(
+                stream=stream,
+                filter_sql=filter_sql,
+                follow=follow,
+                lines=lines,
+                client=client,
+            ):
+                output = format_output(log_entry, format_type=format)
+                click.echo(output)
+
+        except KeyboardInterrupt:
+            click.echo("\nStopped tailing logs.")
+        except Exception as e:
+            click.echo(f"Tail failed: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("errors")
+    @click.option(
+        "--stream",
+        "-s",
+        default="default",
+        help="Stream name to search",
+    )
+    @click.option(
+        "--start",
+        default="-1h",
+        help="Start time",
+    )
+    @click.option(
+        "--size",
+        "-n",
+        default=100,
+        type=int,
+        help="Number of results",
+    )
+    @click.option(
+        "--format",
+        "-f",
+        type=click.Choice(["json", "log", "table", "csv", "summary"]),
+        default="log",
+        help="Output format",
+    )
+    @click.pass_obj
+    def errors_command(client, stream, start, size, format):
+        """Search for error logs."""
+        if client is None:
+            click.echo("OpenObserve not configured.", err=True)
+            return 1
+
+        try:
+            from provide.foundation.observability.openobserve import search_errors
+
+            response = search_errors(
+                stream=stream,
+                start_time=start,
+                size=size,
+                client=client,
+            )
+
+            if response.total == 0:
+                click.echo("No errors found in the specified time range.")
+            else:
+                output = format_output(response, format_type=format)
+                click.echo(output)
+
+        except Exception as e:
+            click.echo(f"Search failed: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("trace")
+    @click.argument("trace_id")
+    @click.option(
+        "--stream",
+        "-s",
+        default="default",
+        help="Stream name to search",
+    )
+    @click.option(
+        "--format",
+        "-f",
+        type=click.Choice(["json", "log", "table"]),
+        default="log",
+        help="Output format",
+    )
+    @click.pass_obj
+    def trace_command(client, trace_id, stream, format):
+        """Search for logs by trace ID."""
+        if client is None:
+            click.echo("OpenObserve not configured.", err=True)
+            return 1
+
+        try:
+            from provide.foundation.observability.openobserve import search_by_trace_id
+
+            response = search_by_trace_id(
+                trace_id=trace_id,
+                stream=stream,
+                client=client,
+            )
+
+            if response.total == 0:
+                click.echo(f"No logs found for trace ID: {trace_id}")
+            else:
+                output = format_output(response, format_type=format)
+                click.echo(output)
+
+        except Exception as e:
+            click.echo(f"Search failed: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("streams")
+    @click.pass_obj
+    def streams_command(client):
+        """List available streams."""
+        if client is None:
+            click.echo("OpenObserve not configured.", err=True)
+            return 1
+
+        try:
+            streams = client.list_streams()
+
+            if not streams:
+                click.echo("No streams found.")
+            else:
+                click.echo("Available streams:")
+                for stream in streams:
+                    click.echo(f"  - {stream.name} ({stream.stream_type})")
+                    if stream.doc_count > 0:
+                        click.echo(f"    Documents: {stream.doc_count:,}")
+                        click.echo(f"    Size: {stream.original_size:,} bytes")
+
+        except Exception as e:
+            click.echo(f"Failed to list streams: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("history")
+    @click.option(
+        "--size",
+        "-n",
+        default=20,
+        type=int,
+        help="Number of history entries",
+    )
+    @click.option(
+        "--stream",
+        "-s",
+        help="Filter by stream name",
+    )
+    @click.pass_obj
+    def history_command(client, size, stream):
+        """View search history."""
+        if client is None:
+            click.echo("OpenObserve not configured.", err=True)
+            return 1
+
+        try:
+            response = client.get_search_history(
+                stream_name=stream,
+                size=size,
+            )
+
+            if response.total == 0:
+                click.echo("No search history found.")
+            else:
+                click.echo(f"Search history ({response.total} entries):")
+                for hit in response.hits:
+                    sql = hit.get("sql", "N/A")
+                    took = hit.get("took", 0)
+                    records = hit.get("scan_records", 0)
+                    click.echo(f"\n  Query: {sql}")
+                    click.echo(f"  Time: {took:.2f}ms, Records: {records:,}")
+
+        except Exception as e:
+            click.echo(f"Failed to get history: {e}", err=True)
+            return 1
+
+    @openobserve_group.command("test")
+    @click.pass_obj
+    def test_command(client):
+        """Test connection to OpenObserve."""
+        if client is None:
+            click.echo("OpenObserve not configured.", err=True)
+            return 1
+
+        click.echo(f"Testing connection to {client.url}...")
+
+        if client.test_connection():
+            click.echo("✅ Connection successful!")
+            click.echo(f"Organization: {client.organization}")
+            click.echo(f"User: {client.username}")
+        else:
+            click.echo("❌ Connection failed!")
+            return 1
+
+    # Export the command group for auto-discovery
+    __all__ = ["openobserve_group"]
+
+else:
+    # Stub when click is not available
+    def openobserve_group(*args, **kwargs):
+        """OpenObserve command stub when click is not available."""
+        raise ImportError(
+            "CLI commands require optional dependencies. "
+            "Install with: pip install 'provide-foundation[cli]'"
+        )
+
+    __all__ = []

provide/foundation/observability/openobserve/exceptions.py

@@ -0,0 +1,41 @@
+"""
+Custom exceptions for OpenObserve integration.
+"""
+
+from provide.foundation.errors import FoundationError
+
+
+class OpenObserveError(FoundationError):
+    """Base exception for OpenObserve-related errors."""
+
+    pass
+
+
+class OpenObserveConnectionError(OpenObserveError):
+    """Error connecting to OpenObserve API."""
+
+    pass
+
+
+class OpenObserveAuthenticationError(OpenObserveError):
+    """Authentication failed with OpenObserve."""
+
+    pass
+
+
+class OpenObserveQueryError(OpenObserveError):
+    """Error executing query in OpenObserve."""
+
+    pass
+
+
+class OpenObserveStreamingError(OpenObserveError):
+    """Error during streaming operations."""
+
+    pass
+
+
+class OpenObserveConfigError(OpenObserveError):
+    """Configuration error for OpenObserve."""
+
+    pass

provide/foundation/observability/openobserve/formatters.py

@@ -0,0 +1,298 @@
+"""
+Output formatting utilities for OpenObserve results.
+"""
+
+import csv
+from datetime import datetime
+import io
+import json
+from typing import Any
+
+from provide.foundation.observability.openobserve.models import SearchResponse
+
+
+def format_json(response: SearchResponse | dict[str, Any], pretty: bool = True) -> str:
+    """Format response as JSON.
+
+    Args:
+        response: Search response or log entry
+        pretty: If True, use pretty printing
+
+    Returns:
+        JSON string
+    """
+    if isinstance(response, SearchResponse):
+        data = {
+            "hits": response.hits,
+            "total": response.total,
+            "took": response.took,
+            "scan_size": response.scan_size,
+        }
+    else:
+        data = response
+
+    if pretty:
+        return json.dumps(data, indent=2, sort_keys=False)
+    else:
+        return json.dumps(data)
+
+
+def format_log_line(entry: dict[str, Any]) -> str:
+    """Format a log entry as a traditional log line.
+
+    Args:
+        entry: Log entry dictionary
+
+    Returns:
+        Formatted log line
+    """
+    # Extract common fields
+    timestamp = entry.get("_timestamp", 0)
+    level = entry.get("level", "INFO")
+    message = entry.get("message", "")
+    service = entry.get("service", "")
+
+    # Convert timestamp to readable format
+    if timestamp:
+        # Assuming microseconds
+        dt = datetime.fromtimestamp(timestamp / 1_000_000)
+        time_str = dt.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
+    else:
+        time_str = "unknown"
+
+    # Build log line
+    parts = [time_str, f"[{level:5s}]"]
+
+    if service:
+        parts.append(f"[{service}]")
+
+    parts.append(message)
+
+    # Add additional fields as key=value
+    exclude_fields = {"_timestamp", "level", "message", "service", "_p"}
+    extra_fields = []
+    for key, value in entry.items():
+        if key not in exclude_fields:
+            extra_fields.append(f"{key}={value}")
+
+    if extra_fields:
+        parts.append(f"({', '.join(extra_fields)})")
+
+    return " ".join(parts)
+
+
+def format_table(response: SearchResponse, columns: list[str] | None = None) -> str:
+    """Format response as a table.
+
+    Args:
+        response: Search response
+        columns: Specific columns to include (None for all)
+
+    Returns:
+        Table string
+    """
+    if not response.hits:
+        return "No results found"
+
+    # Determine columns
+    if columns is None:
+        # Get all unique keys from hits
+        all_keys = set()
+        for hit in response.hits:
+            all_keys.update(hit.keys())
+        # Sort columns, putting common ones first
+        priority_cols = ["_timestamp", "level", "service", "message"]
+        columns = []
+        for col in priority_cols:
+            if col in all_keys:
+                columns.append(col)
+                all_keys.remove(col)
+        columns.extend(sorted(all_keys))
+
+    # Filter out internal columns if not explicitly requested
+    if "_p" in columns and "_p" not in (columns or []):
+        columns = [c for c in columns if not c.startswith("_") or c == "_timestamp"]
+
+    # Try to use tabulate if available
+    try:
+        from tabulate import tabulate
+
+        # Prepare data
+        headers = columns
+        rows = []
+        for hit in response.hits:
+            row = []
+            for col in columns:
+                value = hit.get(col, "")
+                # Format timestamp
+                if col == "_timestamp" and value:
+                    dt = datetime.fromtimestamp(value / 1_000_000)
+                    value = dt.strftime("%Y-%m-%d %H:%M:%S")
+                # Truncate long values
+                value_str = str(value)
+                if len(value_str) > 50:
+                    value_str = value_str[:47] + "..."
+                row.append(value_str)
+            rows.append(row)
+
+        return tabulate(rows, headers=headers, tablefmt="grid")
+
+    except ImportError:
+        # Fallback to simple formatting
+        lines = []
+
+        # Header
+        lines.append(" | ".join(columns))
+        lines.append("-" * (len(columns) * 15))
+
+        # Rows
+        for hit in response.hits:
+            row_values = []
+            for col in columns:
+                value = hit.get(col, "")
+                if col == "_timestamp" and value:
+                    dt = datetime.fromtimestamp(value / 1_000_000)
+                    value = dt.strftime("%H:%M:%S")
+                value_str = str(value)[:12]
+                row_values.append(value_str)
+            lines.append(" | ".join(row_values))
+
+        return "\n".join(lines)
+
+
+def format_csv(response: SearchResponse, columns: list[str] | None = None) -> str:
+    """Format response as CSV.
+
+    Args:
+        response: Search response
+        columns: Specific columns to include (None for all)
+
+    Returns:
+        CSV string
+    """
+    if not response.hits:
+        return ""
+
+    # Determine columns
+    if columns is None:
+        all_keys = set()
+        for hit in response.hits:
+            all_keys.update(hit.keys())
+        columns = sorted(all_keys)
+
+    # Create CSV
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=columns, extrasaction="ignore")
+
+    writer.writeheader()
+    for hit in response.hits:
+        # Format timestamp for readability
+        if "_timestamp" in hit:
+            hit = hit.copy()
+            timestamp = hit["_timestamp"]
+            if timestamp:
+                dt = datetime.fromtimestamp(timestamp / 1_000_000)
+                hit["_timestamp"] = dt.isoformat()
+        writer.writerow(hit)
+
+    return output.getvalue()
+
+
+def format_summary(response: SearchResponse) -> str:
+    """Format a summary of the search response.
+
+    Args:
+        response: Search response
+
+    Returns:
+        Summary string
+    """
+    lines = [
+        f"Total hits: {response.total}",
+        f"Returned: {len(response.hits)}",
+        f"Query time: {response.took}ms",
+        f"Scan size: {response.scan_size:,} bytes",
+    ]
+
+    if response.trace_id:
+        lines.append(f"Trace ID: {response.trace_id}")
+
+    if response.is_partial:
+        lines.append("⚠️  Results are partial")
+
+    if response.function_error:
+        lines.append("Errors:")
+        for error in response.function_error:
+            lines.append(f"  - {error}")
+
+    # Add level distribution if available
+    level_counts = {}
+    for hit in response.hits:
+        level = hit.get("level", "UNKNOWN")
+        level_counts[level] = level_counts.get(level, 0) + 1
+
+    if level_counts:
+        lines.append("\nLevel distribution:")
+        for level, count in sorted(level_counts.items()):
+            lines.append(f"  {level}: {count}")
+
+    return "\n".join(lines)
+
+
+def format_output(
+    response: SearchResponse | dict[str, Any],
+    format_type: str = "log",
+    **kwargs,
+) -> str:
+    """Format output based on specified type.
+
+    Args:
+        response: Search response or log entry
+        format_type: Output format (json, log, table, csv, summary)
+        **kwargs: Additional format-specific options
+
+    Returns:
+        Formatted string
+    """
+    match format_type.lower():
+        case "json":
+            return format_json(response, **kwargs)
+        case "log":
+            if isinstance(response, dict):
+                return format_log_line(response)
+            else:
+                return "\n".join(format_log_line(hit) for hit in response.hits)
+        case "table":
+            if isinstance(response, SearchResponse):
+                return format_table(response, **kwargs)
+            else:
+                # Single entry as table
+                single_response = SearchResponse(
+                    hits=[response],
+                    total=1,
+                    took=0,
+                    scan_size=0,
+                )
+                return format_table(single_response, **kwargs)
+        case "csv":
+            if isinstance(response, SearchResponse):
+                return format_csv(response, **kwargs)
+            else:
+                single_response = SearchResponse(
+                    hits=[response],
+                    total=1,
+                    took=0,
+                    scan_size=0,
+                )
+                return format_csv(single_response, **kwargs)
+        case "summary":
+            if isinstance(response, SearchResponse):
+                return format_summary(response)
+            else:
+                return "Single log entry (use 'log' or 'json' format for details)"
+        case _:
+            # Default to log format
+            if isinstance(response, dict):
+                return format_log_line(response)
+            else:
+                return "\n".join(format_log_line(hit) for hit in response.hits)