logler 1.1.0__cp311-cp311-macosx_10_12_x86_64.whl → 1.1.2__cp311-cp311-macosx_10_12_x86_64.whl

logler/__init__.py

@@ -2,7 +2,7 @@
 Logler - Beautiful local log viewer with thread tracking and real-time updates.
 """
 
-__version__ = "1.0.7"
+__version__ = "1.1.2"
 __author__ = "Logler Contributors"
 
 from .parser import LogParser, LogEntry
@@ -10,7 +10,62 @@ from .tracker import ThreadTracker
 from .log_reader import LogReader
 from .tree_formatter import format_tree, format_waterfall, print_tree, print_waterfall
 
+# Pydantic models for type-safe log analysis
+from .models import (
+    # Core entry models
+    LogEntry as TypedLogEntry,
+    LogLevel,
+    LogFormat,
+    # Search models
+    SearchResult,
+    SearchResults,
+    SearchSummary,
+    SearchCount,
+    # Timeline models
+    ThreadTimeline,
+    # Hierarchy models
+    SpanNode,
+    ThreadHierarchy,
+    BottleneckInfo,
+    NodeType,
+    DetectionMethod,
+    # Pattern models
+    PatternMatch,
+    PatternResults,
+    # Sampling
+    SamplingResult,
+    # Error analysis
+    ErrorAnalysis,
+    RootCause,
+    PropagationChain,
+    ImpactSummary,
+    # File/context
+    FileMetadata,
+    ContextResult,
+    # Cross-service
+    TimelineEntry,
+    CrossServiceTimeline,
+    # Correlation
+    CorrelationLink,
+    CorrelationChains,
+    # Export
+    TraceSpan,
+    TraceExport,
+    # Insights
+    Insight,
+    InsightsResult,
+    # Schema
+    SchemaField,
+    LogSchema,
+    # Helper functions
+    parse_log_entry,
+    parse_search_results,
+    parse_thread_hierarchy,
+    parse_error_analysis,
+)
+
 __all__ = [
+    # Original exports
     "LogParser",
     "LogEntry",
     "ThreadTracker",
@@ -19,4 +74,54 @@ __all__ = [
     "format_waterfall",
     "print_tree",
     "print_waterfall",
+    # Pydantic models - Core
+    "TypedLogEntry",
+    "LogLevel",
+    "LogFormat",
+    # Pydantic models - Search
+    "SearchResult",
+    "SearchResults",
+    "SearchSummary",
+    "SearchCount",
+    # Pydantic models - Timeline
+    "ThreadTimeline",
+    # Pydantic models - Hierarchy
+    "SpanNode",
+    "ThreadHierarchy",
+    "BottleneckInfo",
+    "NodeType",
+    "DetectionMethod",
+    # Pydantic models - Patterns
+    "PatternMatch",
+    "PatternResults",
+    # Pydantic models - Sampling
+    "SamplingResult",
+    # Pydantic models - Error analysis
+    "ErrorAnalysis",
+    "RootCause",
+    "PropagationChain",
+    "ImpactSummary",
+    # Pydantic models - File/context
+    "FileMetadata",
+    "ContextResult",
+    # Pydantic models - Cross-service
+    "TimelineEntry",
+    "CrossServiceTimeline",
+    # Pydantic models - Correlation
+    "CorrelationLink",
+    "CorrelationChains",
+    # Pydantic models - Export
+    "TraceSpan",
+    "TraceExport",
+    # Pydantic models - Insights
+    "Insight",
+    "InsightsResult",
+    # Pydantic models - Schema
+    "SchemaField",
+    "LogSchema",
+    # Helper functions
+    "parse_log_entry",
+    "parse_search_results",
+    "parse_thread_hierarchy",
+    "parse_error_analysis",
 ]

logler/investigate.py

@@ -2197,7 +2197,11 @@ def cross_service_timeline(
 
     for service_name, service_files in files.items():
         if correlation_id:
-            result = follow_thread(service_files, correlation_id=correlation_id, trace_id=trace_id)
+            # WORKAROUND: Only pass correlation_id OR trace_id, not both, to avoid
+            # Rust-side deduplication bug that causes duplicate entries when multiple
+            # IDs match the same log entry. Prefer correlation_id when both are provided.
+            # TODO: Remove this workaround when Rust deduplication is fixed (Phase 2)
+            result = follow_thread(service_files, correlation_id=correlation_id)
             entries = result.get("entries", [])
         elif trace_id:
             result = follow_thread(service_files, trace_id=trace_id)

logler/llm_cli.py

@@ -1424,3 +1424,494 @@ def session_conclude(
 
     except Exception as e:
         _error_json(f"Internal error: {str(e)}", EXIT_INTERNAL_ERROR)
+
+
+# =============================================================================
+# SQL Query Command - High value for LLM log analysis
+# =============================================================================
+
+
+@llm.command()
+@click.argument("query", required=False)
+@click.option("--files", "-f", multiple=True, help="Files to load (supports globs)")
+@click.option("--stdin", is_flag=True, help="Read SQL query from stdin")
+@click.option("--pretty", is_flag=True, help="Pretty-print JSON output")
+def sql(query: Optional[str], files: tuple, stdin: bool, pretty: bool):
+    """
+    Execute SQL queries on log files using DuckDB.
+
+    Loads log files into a 'logs' table with columns:
+    - line_number, timestamp, level, message, thread_id,
+    - correlation_id, trace_id, span_id, file, raw
+
+    Supports all DuckDB SQL including:
+    - Aggregations (COUNT, GROUP BY, HAVING)
+    - Window functions
+    - CTEs (WITH clauses)
+    - JOINs (if loading multiple file groups)
+
+    Examples:
+        # Count errors by level
+        logler llm sql "SELECT level, COUNT(*) FROM logs GROUP BY level" -f "*.log"
+
+        # Find top error messages
+        logler llm sql "SELECT message, COUNT(*) as cnt FROM logs WHERE level='ERROR'
+                        GROUP BY message ORDER BY cnt DESC LIMIT 10" -f app.log
+
+        # Query from stdin
+        echo "SELECT * FROM logs LIMIT 5" | logler llm sql --stdin -f "*.log"
+
+        # Complex analysis
+        logler llm sql "
+            WITH error_threads AS (
+                SELECT DISTINCT thread_id FROM logs WHERE level = 'ERROR'
+            )
+            SELECT l.* FROM logs l
+            JOIN error_threads e ON l.thread_id = e.thread_id
+            ORDER BY l.timestamp
+        " -f "*.log"
+    """
+    import duckdb
+
+    try:
+        # Get query from argument or stdin
+        if stdin:
+            import sys as _sys
+
+            query = _sys.stdin.read().strip()
+        elif not query:
+            _error_json("SQL query required. Provide as argument or use --stdin.")
+
+        file_list = _expand_globs(list(files)) if files else _expand_globs(["*.log"])
+        if not file_list:
+            _error_json(f"No files found matching: {files or ['*.log']}")
+
+        # Parse log files
+        from .parser import LogParser
+
+        parser = LogParser()
+        entries = []
+
+        for file_path in file_list:
+            try:
+                with open(file_path, "r", errors="replace") as f:
+                    for i, line in enumerate(f):
+                        line = line.rstrip()
+                        if not line:
+                            continue
+
+                        entry = parser.parse_line(i + 1, line)
+                        entries.append(
+                            {
+                                "line_number": i + 1,
+                                "timestamp": str(entry.timestamp) if entry.timestamp else None,
+                                "level": str(entry.level).upper() if entry.level else None,
+                                "message": entry.message,
+                                "thread_id": entry.thread_id,
+                                "correlation_id": entry.correlation_id,
+                                "trace_id": getattr(entry, "trace_id", None),
+                                "span_id": getattr(entry, "span_id", None),
+                                "file": file_path,
+                                "raw": line,
+                            }
+                        )
+            except (FileNotFoundError, PermissionError) as e:
+                _error_json(f"Cannot read file {file_path}: {e}")
+
+        if not entries:
+            _output_json(
+                {
+                    "query": query,
+                    "files": file_list,
+                    "total_entries": 0,
+                    "results": [],
+                    "error": "No log entries found",
+                },
+                pretty,
+            )
+            sys.exit(EXIT_NO_RESULTS)
+
+        # Create DuckDB connection and load data
+        conn = duckdb.connect(":memory:")
+
+        # Create table from entries
+        conn.execute(
+            """
+            CREATE TABLE logs (
+                line_number INTEGER,
+                timestamp VARCHAR,
+                level VARCHAR,
+                message VARCHAR,
+                thread_id VARCHAR,
+                correlation_id VARCHAR,
+                trace_id VARCHAR,
+                span_id VARCHAR,
+                file VARCHAR,
+                raw VARCHAR
+            )
+        """
+        )
+
+        # Insert entries
+        conn.executemany(
+            """
+            INSERT INTO logs VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """,
+            [
+                (
+                    e["line_number"],
+                    e["timestamp"],
+                    e["level"],
+                    e["message"],
+                    e["thread_id"],
+                    e["correlation_id"],
+                    e["trace_id"],
+                    e["span_id"],
+                    e["file"],
+                    e["raw"],
+                )
+                for e in entries
+            ],
+        )
+
+        # Execute the user's query
+        try:
+            result = conn.execute(query).fetchall()
+            columns = [desc[0] for desc in conn.description]
+        except duckdb.Error as e:
+            _error_json(f"SQL error: {e}", EXIT_USER_ERROR)
+
+        # Convert results to list of dicts
+        rows = [dict(zip(columns, row)) for row in result]
+
+        output = {
+            "query": query,
+            "files": file_list,
+            "total_entries": len(entries),
+            "columns": columns,
+            "row_count": len(rows),
+            "results": rows,
+        }
+
+        _output_json(output, pretty)
+        sys.exit(EXIT_SUCCESS if rows else EXIT_NO_RESULTS)
+
+    except Exception as e:
+        _error_json(f"Internal error: {str(e)}", EXIT_INTERNAL_ERROR)
+
+
+# =============================================================================
+# Bottleneck Analysis Command
+# =============================================================================
+
+
+@llm.command()
+@click.argument("identifier")
+@click.option("--files", "-f", multiple=True, help="Files to search (supports globs)")
+@click.option("--threshold-ms", type=int, default=100, help="Minimum duration to consider (ms)")
+@click.option("--top-n", type=int, default=10, help="Number of top bottlenecks to return")
+@click.option("--pretty", is_flag=True, help="Pretty-print JSON output")
+def bottleneck(identifier: str, files: tuple, threshold_ms: int, top_n: int, pretty: bool):
+    """
+    Analyze performance bottlenecks for a trace/correlation ID.
+
+    Identifies the slowest operations and shows where time is spent.
+
+    Example:
+        logler llm bottleneck trace-abc123 --files "*.log" --top-n 5
+    """
+    from . import investigate
+
+    try:
+        file_list = _expand_globs(list(files)) if files else _expand_globs(["*.log"])
+        if not file_list:
+            _error_json(f"No files found matching: {files or ['*.log']}")
+
+        # Get hierarchy to analyze
+        hierarchy = investigate.follow_thread_hierarchy(
+            files=file_list,
+            root_identifier=identifier,
+        )
+
+        if not hierarchy.get("roots"):
+            _output_json(
+                {
+                    "identifier": identifier,
+                    "error": "No hierarchy found for identifier",
+                },
+                pretty,
+            )
+            sys.exit(EXIT_NO_RESULTS)
+
+        # Collect all nodes with durations
+        nodes_with_duration = []
+
+        def collect_nodes(node: Dict[str, Any], path: List[str]):
+            node_id = node.get("id", "unknown")
+            duration = node.get("duration_ms", 0) or 0
+            current_path = path + [node_id]
+
+            if duration >= threshold_ms:
+                nodes_with_duration.append(
+                    {
+                        "node_id": node_id,
+                        "name": node.get("name") or node.get("operation_name"),
+                        "duration_ms": duration,
+                        "depth": node.get("depth", 0),
+                        "entry_count": node.get("entry_count", 0),
+                        "error_count": node.get("error_count", 0),
+                        "path": current_path,
+                        "children_count": len(node.get("children", [])),
+                    }
+                )
+
+            for child in node.get("children", []):
+                collect_nodes(child, current_path)
+
+        for root in hierarchy.get("roots", []):
+            collect_nodes(root, [])
+
+        # Sort by duration descending
+        nodes_with_duration.sort(key=lambda x: -x["duration_ms"])
+        top_bottlenecks = nodes_with_duration[:top_n]
+
+        # Calculate percentages
+        total_duration = hierarchy.get("total_duration_ms", 0) or 1
+        for node in top_bottlenecks:
+            node["percentage"] = round(node["duration_ms"] / total_duration * 100, 1)
+
+        output = {
+            "identifier": identifier,
+            "total_duration_ms": hierarchy.get("total_duration_ms"),
+            "total_nodes": hierarchy.get("total_nodes", 0),
+            "analysis": {
+                "threshold_ms": threshold_ms,
+                "nodes_above_threshold": len(nodes_with_duration),
+            },
+            "bottlenecks": top_bottlenecks,
+            "hierarchy_bottleneck": hierarchy.get("bottleneck"),
+        }
+
+        _output_json(output, pretty)
+        sys.exit(EXIT_SUCCESS)
+
+    except Exception as e:
+        _error_json(f"Internal error: {str(e)}", EXIT_INTERNAL_ERROR)
+
+
+# =============================================================================
+# Context Command
+# =============================================================================
+
+
+@llm.command()
+@click.argument("file")
+@click.argument("line", type=int)
+@click.option("--before", "-B", type=int, default=10, help="Lines before")
+@click.option("--after", "-A", type=int, default=10, help="Lines after")
+@click.option("--pretty", is_flag=True, help="Pretty-print JSON output")
+def context(file: str, line: int, before: int, after: int, pretty: bool):
+    """
+    Get context lines around a specific log entry.
+
+    Returns parsed entries with context, useful for understanding
+    what happened before and after a specific log line.
+
+    Example:
+        logler llm context app.log 1523 --before 20 --after 10
+    """
+    from . import investigate
+
+    try:
+        if not Path(file).exists():
+            _error_json(f"File not found: {file}")
+
+        result = investigate.get_context(
+            file=file,
+            line_number=line,
+            lines_before=before,
+            lines_after=after,
+        )
+
+        # Transform to cleaner output
+        output = {
+            "file": file,
+            "line_number": line,
+            "context_lines": {"before": before, "after": after},
+            "target": result.get("target"),
+            "context_before": result.get("context_before", []),
+            "context_after": result.get("context_after", []),
+        }
+
+        _output_json(output, pretty)
+        sys.exit(EXIT_SUCCESS)
+
+    except Exception as e:
+        _error_json(f"Internal error: {str(e)}", EXIT_INTERNAL_ERROR)
+
+
+# =============================================================================
+# Trace Export Command
+# =============================================================================
+
+
+@llm.command("export")
+@click.argument("identifier")
+@click.option("--files", "-f", multiple=True, help="Files to search (supports globs)")
+@click.option(
+    "--format",
+    "export_format",
+    type=click.Choice(["jaeger", "zipkin", "otlp"]),
+    default="jaeger",
+    help="Export format",
+)
+@click.option("--pretty", is_flag=True, help="Pretty-print JSON output")
+def export_trace(identifier: str, files: tuple, export_format: str, pretty: bool):
+    """
+    Export traces to Jaeger/Zipkin/OTLP format.
+
+    Converts log-based traces to standard distributed tracing formats
+    that can be imported into Jaeger, Zipkin, or other tracing systems.
+
+    Example:
+        logler llm export trace-abc123 --files "*.log" --format jaeger
+    """
+    from . import investigate
+
+    try:
+        file_list = _expand_globs(list(files)) if files else _expand_globs(["*.log"])
+        if not file_list:
+            _error_json(f"No files found matching: {files or ['*.log']}")
+
+        # Get hierarchy for the trace
+        hierarchy = investigate.follow_thread_hierarchy(
+            files=file_list,
+            root_identifier=identifier,
+        )
+
+        if not hierarchy.get("roots"):
+            _output_json(
+                {
+                    "identifier": identifier,
+                    "format": export_format,
+                    "error": "No trace data found for identifier",
+                },
+                pretty,
+            )
+            sys.exit(EXIT_NO_RESULTS)
+
+        # Convert hierarchy to spans
+        spans = []
+
+        def node_to_span(node: Dict[str, Any], parent_span_id: Optional[str] = None):
+            node_id = node.get("id", "unknown")
+
+            # Generate span ID if not present
+            span_id = node.get("span_id") or f"span-{hash(node_id) & 0xFFFFFFFF:08x}"
+
+            span = {
+                "traceId": identifier.replace("trace-", "").replace("-", "")[:32].ljust(32, "0"),
+                "spanId": span_id.replace("-", "")[:16].ljust(16, "0"),
+                "operationName": node.get("name") or node.get("operation_name") or node_id,
+                "serviceName": node.get("service_name", "unknown"),
+                "startTime": node.get("start_time"),
+                "duration": (node.get("duration_ms", 0) or 0) * 1000,  # Convert to microseconds
+                "tags": [],
+                "logs": [],
+            }
+
+            if parent_span_id:
+                span["parentSpanId"] = parent_span_id.replace("-", "")[:16].ljust(16, "0")
+
+            # Add tags
+            if node.get("error_count", 0) > 0:
+                span["tags"].append({"key": "error", "value": True})
+
+            if node.get("entry_count"):
+                span["tags"].append({"key": "log.entry_count", "value": node["entry_count"]})
+
+            spans.append(span)
+
+            for child in node.get("children", []):
+                node_to_span(child, span_id)
+
+        for root in hierarchy.get("roots", []):
+            node_to_span(root)
+
+        # Format output based on target format
+        if export_format == "jaeger":
+            trace_output = {
+                "data": [
+                    {
+                        "traceID": identifier.replace("trace-", "")
+                        .replace("-", "")[:32]
+                        .ljust(32, "0"),
+                        "spans": spans,
+                        "processes": {
+                            "p1": {
+                                "serviceName": "logler-export",
+                                "tags": [],
+                            }
+                        },
+                    }
+                ],
+                "total": 1,
+                "limit": 0,
+                "offset": 0,
+                "errors": None,
+            }
+        elif export_format == "zipkin":
+            trace_output = [
+                {
+                    "traceId": span["traceId"],
+                    "id": span["spanId"],
+                    "name": span["operationName"],
+                    "timestamp": span.get("startTime"),
+                    "duration": span["duration"],
+                    "localEndpoint": {"serviceName": span.get("serviceName", "unknown")},
+                    "parentId": span.get("parentSpanId"),
+                    "tags": {t["key"]: str(t["value"]) for t in span.get("tags", [])},
+                }
+                for span in spans
+            ]
+        else:  # otlp
+            trace_output = {
+                "resourceSpans": [
+                    {
+                        "resource": {
+                            "attributes": [
+                                {"key": "service.name", "value": {"stringValue": "logler-export"}}
+                            ]
+                        },
+                        "scopeSpans": [
+                            {
+                                "scope": {"name": "logler"},
+                                "spans": [
+                                    {
+                                        "traceId": span["traceId"],
+                                        "spanId": span["spanId"],
+                                        "name": span["operationName"],
+                                        "startTimeUnixNano": span.get("startTime"),
+                                        "endTimeUnixNano": None,
+                                        "parentSpanId": span.get("parentSpanId"),
+                                    }
+                                    for span in spans
+                                ],
+                            }
+                        ],
+                    }
+                ]
+            }
+
+        output = {
+            "identifier": identifier,
+            "format": export_format,
+            "span_count": len(spans),
+            "export": trace_output,
+        }
+
+        _output_json(output, pretty)
+        sys.exit(EXIT_SUCCESS)
+
+    except Exception as e:
+        _error_json(f"Internal error: {str(e)}", EXIT_INTERNAL_ERROR)