provide-foundation 0.0.0.dev0__py3-none-any.whl

provide/foundation/observability/openobserve/models.py

@@ -0,0 +1,134 @@
+"""
+Data models for OpenObserve API requests and responses.
+"""
+
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Any
+
+
+@dataclass
+class SearchQuery:
+    """Search query parameters for OpenObserve."""
+
+    sql: str
+    start_time: int  # Microseconds since epoch
+    end_time: int  # Microseconds since epoch
+    from_offset: int = 0
+    size: int = 100
+
+    def to_dict(self) -> dict[str, Any]:
+        """Convert to API request format."""
+        return {
+            "query": {
+                "sql": self.sql,
+                "start_time": self.start_time,
+                "end_time": self.end_time,
+                "from": self.from_offset,
+                "size": self.size,
+            }
+        }
+
+
+@dataclass
+class SearchResponse:
+    """Response from OpenObserve search API."""
+
+    hits: list[dict[str, Any]]
+    total: int
+    took: int  # Milliseconds
+    scan_size: int
+    trace_id: str | None = None
+    from_offset: int = 0
+    size: int = 0
+    is_partial: bool = False
+    function_error: list[str] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SearchResponse":
+        """Create from API response."""
+        return cls(
+            hits=data.get("hits", []),
+            total=data.get("total", 0),
+            took=data.get("took", 0),
+            scan_size=data.get("scan_size", 0),
+            trace_id=data.get("trace_id"),
+            from_offset=data.get("from", 0),
+            size=data.get("size", 0),
+            is_partial=data.get("is_partial", False),
+            function_error=data.get("function_error", []),
+        )
+
+
+@dataclass
+class StreamInfo:
+    """Information about an OpenObserve stream."""
+
+    name: str
+    storage_type: str
+    stream_type: str
+    doc_count: int = 0
+    compressed_size: int = 0
+    original_size: int = 0
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "StreamInfo":
+        """Create from API response."""
+        return cls(
+            name=data.get("name", ""),
+            storage_type=data.get("storage_type", ""),
+            stream_type=data.get("stream_type", ""),
+            doc_count=data.get("stats", {}).get("doc_count", 0),
+            compressed_size=data.get("stats", {}).get("compressed_size", 0),
+            original_size=data.get("stats", {}).get("original_size", 0),
+        )
+
+
+def parse_relative_time(time_str: str, now: datetime | None = None) -> int:
+    """Parse relative time strings like '-1h', '-30m' to microseconds since epoch.
+
+    Args:
+        time_str: Time string (e.g., '-1h', '-30m', 'now')
+        now: Current time (for testing), defaults to datetime.now()
+
+    Returns:
+        Microseconds since epoch
+    """
+    from datetime import timedelta
+
+    if now is None:
+        now = datetime.now()
+
+    if time_str == "now":
+        return int(now.timestamp() * 1_000_000)
+
+    if time_str.startswith("-"):
+        # Parse relative time
+        value = time_str[1:]
+        if value.endswith("h"):
+            delta = timedelta(hours=int(value[:-1]))
+        elif value.endswith("m"):
+            delta = timedelta(minutes=int(value[:-1]))
+        elif value.endswith("s"):
+            delta = timedelta(seconds=int(value[:-1]))
+        elif value.endswith("d"):
+            delta = timedelta(days=int(value[:-1]))
+        else:
+            # Assume seconds if no unit
+            delta = timedelta(seconds=int(value))
+
+        target_time = now - delta
+        return int(target_time.timestamp() * 1_000_000)
+
+    # Try to parse as timestamp
+    try:
+        timestamp = int(time_str)
+        # If it's already in microseconds (large number), return as-is
+        if timestamp > 1_000_000_000_000:
+            return timestamp
+        # Otherwise assume seconds and convert
+        return timestamp * 1_000_000
+    except ValueError:
+        # Try to parse as ISO datetime
+        dt = datetime.fromisoformat(time_str)
+        return int(dt.timestamp() * 1_000_000)

provide/foundation/observability/openobserve/otlp.py

@@ -0,0 +1,320 @@
+"""
+OTLP integration for sending logs to OpenObserve.
+"""
+
+from datetime import datetime
+import json
+from typing import Any
+
+from provide.foundation.logger import get_logger
+from provide.foundation.observability.openobserve.client import OpenObserveClient
+
+log = get_logger(__name__)
+
+# OpenTelemetry feature detection
+try:
+    from opentelemetry import trace
+    from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+    from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler
+    from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+    from opentelemetry.sdk.resources import Resource
+    from opentelemetry.semconv.resource import ResourceAttributes
+
+    _HAS_OTEL_LOGS = True
+except ImportError:
+    _HAS_OTEL_LOGS = False
+
+
+def send_log_otlp(
+    message: str,
+    level: str = "INFO",
+    service: str | None = None,
+    attributes: dict[str, Any] | None = None,
+) -> bool:
+    """Send a log via OTLP if available.
+
+    Args:
+        message: Log message
+        level: Log level
+        service: Service name (uses config if not provided)
+        attributes: Additional attributes
+
+    Returns:
+        True if sent successfully via OTLP, False otherwise
+    """
+    if not _HAS_OTEL_LOGS:
+        return False
+
+    try:
+        from provide.foundation.logger.config import TelemetryConfig
+
+        config = TelemetryConfig.from_env()
+
+        if not config.otlp_endpoint:
+            return False
+
+        # Create resource with service info
+        resource_attrs = {
+            ResourceAttributes.SERVICE_NAME: service
+            or config.service_name
+            or "foundation",
+        }
+        if config.service_version:
+            resource_attrs[ResourceAttributes.SERVICE_VERSION] = config.service_version
+
+        resource = Resource.create(resource_attrs)
+
+        # Configure OTLP exporter
+        headers = config.get_otlp_headers_dict()
+        if config.openobserve_org:
+            # Add organization header for OpenObserve
+            headers["organization"] = config.openobserve_org
+            headers["stream-name"] = config.openobserve_stream
+
+        # Determine endpoint for logs
+        if config.otlp_traces_endpoint:
+            # Replace /traces with /logs
+            logs_endpoint = config.otlp_traces_endpoint.replace(
+                "/v1/traces", "/v1/logs"
+            )
+        else:
+            logs_endpoint = f"{config.otlp_endpoint}/v1/logs"
+
+        exporter = OTLPLogExporter(
+            endpoint=logs_endpoint,
+            headers=headers,
+        )
+
+        # Create logger provider
+        logger_provider = LoggerProvider(resource=resource)
+        logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+        # Get logger and emit log
+        otel_logger = logger_provider.get_logger(__name__)
+
+        # Add trace context if available
+        log_attrs = attributes or {}
+        current_span = trace.get_current_span()
+        if current_span and current_span.is_recording():
+            span_context = current_span.get_span_context()
+            log_attrs["trace_id"] = f"{span_context.trace_id:032x}"
+            log_attrs["span_id"] = f"{span_context.span_id:016x}"
+
+        # Map level to severity number
+        severity_map = {
+            "TRACE": 1,
+            "DEBUG": 5,
+            "INFO": 9,
+            "WARN": 13,
+            "WARNING": 13,
+            "ERROR": 17,
+            "FATAL": 21,
+            "CRITICAL": 21,
+        }
+        severity = severity_map.get(level.upper(), 9)
+
+        # Emit log record
+        otel_logger.emit(
+            severity_number=severity,
+            severity_text=level.upper(),
+            body=message,
+            attributes=log_attrs,
+        )
+
+        # Force flush to ensure delivery
+        logger_provider.force_flush()
+
+        log.debug(f"Sent log via OTLP: {message[:50]}...")
+        return True
+
+    except Exception as e:
+        log.debug(f"Failed to send via OTLP: {e}")
+        return False
+
+
+def send_log_bulk(
+    message: str,
+    level: str = "INFO",
+    service: str | None = None,
+    attributes: dict[str, Any] | None = None,
+    client: OpenObserveClient | None = None,
+) -> bool:
+    """Send a log via OpenObserve bulk API.
+
+    Args:
+        message: Log message
+        level: Log level
+        service: Service name
+        attributes: Additional attributes
+        client: OpenObserve client (creates new if not provided)
+
+    Returns:
+        True if sent successfully
+    """
+    try:
+        if client is None:
+            client = OpenObserveClient.from_config()
+
+        from provide.foundation.logger.config import TelemetryConfig
+
+        config = TelemetryConfig.from_env()
+
+        # Build log entry
+        log_entry = {
+            "_timestamp": int(datetime.now().timestamp() * 1_000_000),
+            "level": level.upper(),
+            "message": message,
+            "service": service or config.service_name or "foundation",
+        }
+
+        # Add attributes
+        if attributes:
+            log_entry.update(attributes)
+
+        # Add trace context if available
+        try:
+            from opentelemetry import trace
+
+            current_span = trace.get_current_span()
+            if current_span and current_span.is_recording():
+                span_context = current_span.get_span_context()
+                log_entry["trace_id"] = f"{span_context.trace_id:032x}"
+                log_entry["span_id"] = f"{span_context.span_id:016x}"
+        except ImportError:
+            pass
+
+        # Try Foundation's tracer context
+        try:
+            from provide.foundation.tracer.context import (
+                get_current_span,
+                get_current_trace_id,
+            )
+
+            span = get_current_span()
+            if span:
+                log_entry["trace_id"] = span.trace_id
+                log_entry["span_id"] = span.span_id
+            elif trace_id := get_current_trace_id():
+                log_entry["trace_id"] = trace_id
+        except ImportError:
+            pass
+
+        # Format as bulk request
+        stream = config.openobserve_stream
+        bulk_data = (
+            json.dumps({"index": {"_index": stream}})
+            + "\n"
+            + json.dumps(log_entry)
+            + "\n"
+        )
+
+        # Send via bulk API
+        import requests
+
+        # Build URL - check if client.url already includes /api/{org}
+        if f"/api/{client.organization}" in client.url:
+            url = f"{client.url}/_bulk"
+        else:
+            url = f"{client.url}/api/{client.organization}/_bulk"
+
+        response = requests.post(
+            url,
+            headers=client.session.headers,
+            data=bulk_data,
+            timeout=client.timeout,
+        )
+
+        if response.status_code == 200:
+            log.debug(f"Sent log via bulk API: {message[:50]}...")
+            return True
+        else:
+            log.debug(f"Failed to send via bulk API: {response.status_code}")
+            return False
+
+    except Exception as e:
+        log.debug(f"Failed to send via bulk API: {e}")
+        return False
+
+
+def send_log(
+    message: str,
+    level: str = "INFO",
+    service: str | None = None,
+    attributes: dict[str, Any] | None = None,
+    prefer_otlp: bool = True,
+    client: OpenObserveClient | None = None,
+) -> bool:
+    """Send a log using OTLP if available, otherwise bulk API.
+
+    Args:
+        message: Log message
+        level: Log level
+        service: Service name
+        attributes: Additional attributes
+        prefer_otlp: Try OTLP first if True
+        client: OpenObserve client for bulk API
+
+    Returns:
+        True if sent successfully
+    """
+    # Try OTLP first if preferred and available
+    if prefer_otlp and _HAS_OTEL_LOGS:
+        if send_log_otlp(message, level, service, attributes):
+            return True
+
+    # Fall back to bulk API
+    return send_log_bulk(message, level, service, attributes, client)
+
+
+def create_otlp_logger_provider() -> Any | None:
+    """Create an OTLP logger provider for continuous logging.
+
+    Returns:
+        LoggerProvider if OTLP is available and configured, None otherwise
+    """
+    if not _HAS_OTEL_LOGS:
+        return None
+
+    try:
+        from provide.foundation.logger.config import TelemetryConfig
+
+        config = TelemetryConfig.from_env()
+
+        if not config.otlp_endpoint:
+            return None
+
+        # Create resource
+        resource_attrs = {
+            ResourceAttributes.SERVICE_NAME: config.service_name or "foundation",
+        }
+        if config.service_version:
+            resource_attrs[ResourceAttributes.SERVICE_VERSION] = config.service_version
+
+        resource = Resource.create(resource_attrs)
+
+        # Configure exporter
+        headers = config.get_otlp_headers_dict()
+        if config.openobserve_org:
+            headers["organization"] = config.openobserve_org
+            headers["stream-name"] = config.openobserve_stream
+
+        logs_endpoint = f"{config.otlp_endpoint}/v1/logs"
+        if config.otlp_traces_endpoint:
+            logs_endpoint = config.otlp_traces_endpoint.replace(
+                "/v1/traces", "/v1/logs"
+            )
+
+        exporter = OTLPLogExporter(
+            endpoint=logs_endpoint,
+            headers=headers,
+        )
+
+        # Create provider
+        logger_provider = LoggerProvider(resource=resource)
+        logger_provider.add_log_record_processor(BatchLogRecordProcessor(exporter))
+
+        return logger_provider
+
+    except Exception as e:
+        log.debug(f"Failed to create OTLP logger provider: {e}")
+        return None

provide/foundation/observability/openobserve/search.py

@@ -0,0 +1,222 @@
+"""
+Search operations for OpenObserve.
+"""
+
+
+from provide.foundation.logger import get_logger
+from provide.foundation.observability.openobserve.client import OpenObserveClient
+from provide.foundation.observability.openobserve.models import SearchResponse
+
+log = get_logger(__name__)
+
+
+def search_logs(
+    sql: str,
+    start_time: str | int | None = None,
+    end_time: str | int | None = None,
+    size: int = 100,
+    client: OpenObserveClient | None = None,
+) -> SearchResponse:
+    """Search logs in OpenObserve.
+
+    Args:
+        sql: SQL query to execute
+        start_time: Start time (relative like "-1h" or microseconds)
+        end_time: End time (relative like "now" or microseconds)
+        size: Number of results to return
+        client: OpenObserve client (creates new if not provided)
+
+    Returns:
+        SearchResponse with results
+    """
+    if client is None:
+        client = OpenObserveClient.from_config()
+
+    return client.search(
+        sql=sql,
+        start_time=start_time,
+        end_time=end_time,
+        size=size,
+    )
+
+
+def search_by_trace_id(
+    trace_id: str,
+    stream: str = "default",
+    client: OpenObserveClient | None = None,
+) -> SearchResponse:
+    """Search for logs by trace ID.
+
+    Args:
+        trace_id: Trace ID to search for
+        stream: Stream name to search in
+        client: OpenObserve client (creates new if not provided)
+
+    Returns:
+        SearchResponse with matching logs
+    """
+    sql = (
+        f"SELECT * FROM {stream} WHERE trace_id = '{trace_id}' ORDER BY _timestamp ASC"
+    )
+    return search_logs(sql=sql, start_time="-24h", client=client)
+
+
+def search_by_level(
+    level: str,
+    stream: str = "default",
+    start_time: str | int | None = None,
+    end_time: str | int | None = None,
+    size: int = 100,
+    client: OpenObserveClient | None = None,
+) -> SearchResponse:
+    """Search for logs by level.
+
+    Args:
+        level: Log level to filter (ERROR, WARN, INFO, DEBUG, etc.)
+        stream: Stream name to search in
+        start_time: Start time
+        end_time: End time
+        size: Number of results
+        client: OpenObserve client
+
+    Returns:
+        SearchResponse with matching logs
+    """
+    sql = f"SELECT * FROM {stream} WHERE level = '{level}' ORDER BY _timestamp DESC"
+    return search_logs(
+        sql=sql,
+        start_time=start_time,
+        end_time=end_time,
+        size=size,
+        client=client,
+    )
+
+
+def search_errors(
+    stream: str = "default",
+    start_time: str | int | None = None,
+    size: int = 100,
+    client: OpenObserveClient | None = None,
+) -> SearchResponse:
+    """Search for error logs.
+
+    Args:
+        stream: Stream name to search in
+        start_time: Start time
+        size: Number of results
+        client: OpenObserve client
+
+    Returns:
+        SearchResponse with error logs
+    """
+    return search_by_level(
+        level="ERROR",
+        stream=stream,
+        start_time=start_time,
+        size=size,
+        client=client,
+    )
+
+
+def search_by_service(
+    service: str,
+    stream: str = "default",
+    start_time: str | int | None = None,
+    end_time: str | int | None = None,
+    size: int = 100,
+    client: OpenObserveClient | None = None,
+) -> SearchResponse:
+    """Search for logs by service name.
+
+    Args:
+        service: Service name to filter
+        stream: Stream name to search in
+        start_time: Start time
+        end_time: End time
+        size: Number of results
+        client: OpenObserve client
+
+    Returns:
+        SearchResponse with matching logs
+    """
+    sql = f"SELECT * FROM {stream} WHERE service = '{service}' ORDER BY _timestamp DESC"
+    return search_logs(
+        sql=sql,
+        start_time=start_time,
+        end_time=end_time,
+        size=size,
+        client=client,
+    )
+
+
+def aggregate_by_level(
+    stream: str = "default",
+    start_time: str | int | None = None,
+    end_time: str | int | None = None,
+    client: OpenObserveClient | None = None,
+) -> dict[str, int]:
+    """Get count of logs by level.
+
+    Args:
+        stream: Stream name to search in
+        start_time: Start time
+        end_time: End time
+        client: OpenObserve client
+
+    Returns:
+        Dictionary mapping level to count
+    """
+    sql = f"SELECT level, COUNT(*) as count FROM {stream} GROUP BY level"
+    response = search_logs(
+        sql=sql,
+        start_time=start_time,
+        end_time=end_time,
+        size=1000,
+        client=client,
+    )
+
+    result = {}
+    for hit in response.hits:
+        level = hit.get("level", "UNKNOWN")
+        count = hit.get("count", 0)
+        result[level] = count
+
+    return result
+
+
+def get_current_trace_logs(
+    stream: str = "default",
+    client: OpenObserveClient | None = None,
+) -> SearchResponse | None:
+    """Get logs for the current active trace.
+
+    Args:
+        stream: Stream name to search in
+        client: OpenObserve client
+
+    Returns:
+        SearchResponse with logs for current trace, or None if no active trace
+    """
+    # Try to get current trace ID from OpenTelemetry
+    try:
+        from opentelemetry import trace
+
+        current_span = trace.get_current_span()
+        if current_span and current_span.is_recording():
+            span_context = current_span.get_span_context()
+            trace_id = f"{span_context.trace_id:032x}"
+            return search_by_trace_id(trace_id, stream=stream, client=client)
+    except ImportError:
+        pass
+
+    # Try to get from Foundation tracer
+    try:
+        from provide.foundation.tracer.context import get_current_trace_id
+
+        trace_id = get_current_trace_id()
+        if trace_id:
+            return search_by_trace_id(trace_id, stream=stream, client=client)
+    except ImportError:
+        pass
+
+    return None