logler 1.0.7__cp311-cp311-win_amd64.whl

logler/log_reader.py

@@ -0,0 +1,267 @@
+"""Log file reading and streaming module."""
+
+import os
+import time
+from pathlib import Path
+from typing import Iterator, Optional
+
+
+class LogReader:
+    """Efficient log file reader with support for tailing and large files."""
+
+    def __init__(self, file_path: str, buffer_size: int = 8192):
+        """
+        Initialize the log reader.
+
+        Args:
+            file_path: Path to the log file
+            buffer_size: Buffer size for reading (default 8KB)
+        """
+        self.file_path = Path(file_path)
+        self.buffer_size = buffer_size
+
+        if not self.file_path.exists():
+            raise FileNotFoundError(f"Log file not found: {file_path}")
+
+        if not self.file_path.is_file():
+            raise ValueError(f"Not a file: {file_path}")
+
+    def read_lines(
+        self, start_line: int = 0, max_lines: Optional[int] = None, reverse: bool = False
+    ) -> Iterator[str]:
+        """
+        Read lines from the log file.
+
+        Args:
+            start_line: Line number to start from (0-indexed)
+            max_lines: Maximum number of lines to read (None for all)
+            reverse: Read lines in reverse order
+
+        Yields:
+            Log lines as strings
+        """
+        if reverse:
+            yield from self._read_lines_reverse(start_line, max_lines)
+        else:
+            yield from self._read_lines_forward(start_line, max_lines)
+
+    def _read_lines_forward(
+        self, start_line: int = 0, max_lines: Optional[int] = None
+    ) -> Iterator[str]:
+        """Read lines forward from the file."""
+        with open(self.file_path, "r", encoding="utf-8", errors="replace") as f:
+            line_num = 0
+            lines_read = 0
+
+            for line in f:
+                if line_num >= start_line:
+                    yield line.rstrip("\n\r")
+                    lines_read += 1
+
+                    if max_lines and lines_read >= max_lines:
+                        break
+
+                line_num += 1
+
+    def _read_lines_reverse(
+        self, start_line: int = 0, max_lines: Optional[int] = None
+    ) -> Iterator[str]:
+        """
+        Read lines in reverse order using an efficient algorithm.
+
+        This reads the file from the end in chunks to avoid loading
+        the entire file into memory.
+        """
+        with open(self.file_path, "rb") as f:
+            # Seek to end of file
+            f.seek(0, os.SEEK_END)
+            file_size = f.tell()
+
+            # Read in chunks from the end
+            buffer = b""
+            position = file_size
+            lines = []
+
+            while position > 0:
+                # Determine chunk size
+                chunk_size = min(self.buffer_size, position)
+                position -= chunk_size
+
+                # Read chunk
+                f.seek(position)
+                chunk = f.read(chunk_size)
+
+                # Combine with previous buffer
+                buffer = chunk + buffer
+
+                # Split into lines
+                chunk_lines = buffer.split(b"\n")
+
+                # Keep the first incomplete line in buffer
+                buffer = chunk_lines[0]
+                chunk_lines = chunk_lines[1:]
+
+                # Add lines in reverse
+                for line in reversed(chunk_lines):
+                    if line or lines:  # Skip empty lines at the end
+                        try:
+                            decoded = line.decode("utf-8", errors="replace")
+                            lines.append(decoded.rstrip("\r"))
+                        except UnicodeDecodeError:
+                            continue
+
+            # Add the first line if buffer has content
+            if buffer:
+                try:
+                    decoded = buffer.decode("utf-8", errors="replace")
+                    lines.append(decoded.rstrip("\r"))
+                except UnicodeDecodeError:
+                    pass
+
+            # Apply start_line and max_lines
+            lines_to_yield = lines[start_line:]
+            if max_lines:
+                lines_to_yield = lines_to_yield[:max_lines]
+
+            for line in lines_to_yield:
+                yield line
+
+    def tail(
+        self, num_lines: int = 10, follow: bool = False, sleep_interval: float = 0.1
+    ) -> Iterator[str]:
+        """
+        Tail the log file (like tail -f).
+
+        Args:
+            num_lines: Number of initial lines to show
+            follow: If True, continue watching for new lines
+            sleep_interval: How long to sleep between checks (seconds)
+
+        Yields:
+            Log lines as strings
+        """
+        # First, yield the last n lines
+        for line in self._read_lines_reverse(max_lines=num_lines):
+            pass  # Skip initial lines in non-follow mode if we just want to set position
+
+        lines = list(self._read_lines_reverse(max_lines=num_lines))
+        for line in reversed(lines):
+            yield line
+
+        if not follow:
+            return
+
+        # Follow mode: watch for new lines
+        with open(self.file_path, "r", encoding="utf-8", errors="replace") as f:
+            # Seek to end
+            f.seek(0, os.SEEK_END)
+
+            while True:
+                line = f.readline()
+                if line:
+                    yield line.rstrip("\n\r")
+                else:
+                    # Check if file was truncated (log rotation)
+                    current_pos = f.tell()
+                    f.seek(0, os.SEEK_END)
+                    end_pos = f.tell()
+
+                    if current_pos > end_pos:
+                        # File was truncated, start from beginning
+                        f.seek(0)
+                    else:
+                        # No new data, sleep
+                        time.sleep(sleep_interval)
+
+    def search(
+        self,
+        pattern: str,
+        case_sensitive: bool = False,
+        regex: bool = False,
+        max_lines: Optional[int] = None,
+    ) -> Iterator[tuple[int, str]]:
+        """
+        Search for lines matching a pattern.
+
+        Args:
+            pattern: Search pattern (string or regex)
+            case_sensitive: Whether search is case-sensitive
+            regex: Whether pattern is a regex
+            max_lines: Maximum number of matching lines to return
+
+        Yields:
+            Tuples of (line_number, line_content)
+        """
+        import re as regex_module
+
+        def make_regex_matcher(compiled):
+            return lambda line: compiled.search(line)
+
+        def make_case_insensitive_matcher(pat):
+            return lambda line: pat in line.lower()
+
+        def make_case_sensitive_matcher(pat):
+            return lambda line: pat in line
+
+        if regex:
+            flags = 0 if case_sensitive else regex_module.IGNORECASE
+            compiled_pattern = regex_module.compile(pattern, flags)
+            match_func = make_regex_matcher(compiled_pattern)
+        else:
+            if not case_sensitive:
+                pattern = pattern.lower()
+                match_func = make_case_insensitive_matcher(pattern)
+            else:
+                match_func = make_case_sensitive_matcher(pattern)
+
+        with open(self.file_path, "r", encoding="utf-8", errors="replace") as f:
+            matches_found = 0
+
+            for line_num, line in enumerate(f, 1):
+                line = line.rstrip("\n\r")
+
+                if match_func(line):
+                    yield (line_num, line)
+                    matches_found += 1
+
+                    if max_lines and matches_found >= max_lines:
+                        break
+
+    def get_file_info(self) -> dict:
+        """
+        Get information about the log file.
+
+        Returns:
+            Dictionary with file metadata
+        """
+        stat = self.file_path.stat()
+
+        return {
+            "path": str(self.file_path.absolute()),
+            "size": stat.st_size,
+            "size_human": self._format_bytes(stat.st_size),
+            "modified": stat.st_mtime,
+            "created": stat.st_ctime,
+        }
+
+    @staticmethod
+    def _format_bytes(size: int) -> str:
+        """Format bytes to human-readable string."""
+        for unit in ["B", "KB", "MB", "GB", "TB"]:
+            if size < 1024.0:
+                return f"{size:.2f} {unit}"
+            size /= 1024.0
+        return f"{size:.2f} PB"
+
+    def count_lines(self) -> int:
+        """
+        Count total number of lines in the file.
+
+        Returns:
+            Number of lines
+        """
+        count = 0
+        with open(self.file_path, "rb") as f:
+            for _ in f:
+                count += 1
+        return count

logler/parser.py

@@ -0,0 +1,207 @@
+"""
+Log parsing module with support for multiple formats.
+"""
+
+import re
+import json
+from datetime import datetime
+from typing import Optional, Dict, Any
+from dataclasses import dataclass, field
+from enum import Enum
+
+
+class LogLevel(str, Enum):
+    """Log levels."""
+
+    TRACE = "TRACE"
+    DEBUG = "DEBUG"
+    INFO = "INFO"
+    WARN = "WARN"
+    WARNING = "WARNING"
+    ERROR = "ERROR"
+    CRITICAL = "CRITICAL"
+    FATAL = "FATAL"
+    UNKNOWN = "UNKNOWN"
+
+
+@dataclass
+class LogEntry:
+    """Parsed log entry."""
+
+    line_number: int
+    raw: str
+    timestamp: Optional[datetime] = None
+    level: str = "UNKNOWN"
+    message: str = ""
+    thread_id: Optional[str] = None
+    correlation_id: Optional[str] = None
+    trace_id: Optional[str] = None
+    span_id: Optional[str] = None
+    service_name: Optional[str] = None
+    fields: Dict[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self):
+        """Ensure message is set."""
+        if not self.message and self.raw:
+            self.message = self.raw
+
+
+class LogParser:
+    """Parse log entries from various formats."""
+
+    # Regex patterns
+    PATTERNS = {
+        "timestamp": re.compile(
+            r"\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?"
+        ),
+        "log_level": re.compile(
+            r"\b(TRACE|DEBUG|INFO|INFORMATION|WARN|WARNING|ERROR|ERR|FATAL|CRITICAL|CRIT)\b",
+            re.IGNORECASE,
+        ),
+        "thread_id": re.compile(r"(?:thread[=:\s]+|tid[=:\s]+|\[)([a-zA-Z0-9_-]+)(?:\])?"),
+        "correlation_id": re.compile(
+            r"(?:correlation[_-]?id|request[_-]?id|req[_-]?id)[=:\s]+([a-zA-Z0-9_-]+)"
+        ),
+        "trace_id": re.compile(r"(?:trace[_-]?id|traceId)[=:\s]+([a-fA-F0-9]{16,32})"),
+        "span_id": re.compile(r"(?:span[_-]?id|spanId)[=:\s]+([a-fA-F0-9]{8,16})"),
+    }
+
+    def parse_line(self, line_number: int, raw: str) -> LogEntry:
+        """Parse a single log line."""
+        # Try JSON first
+        if raw.strip().startswith("{"):
+            try:
+                data = json.loads(raw.strip())
+                return self._parse_json(line_number, raw, data)
+            except (json.JSONDecodeError, ValueError):
+                pass
+
+        # Parse as plain text
+        return self._parse_plain(line_number, raw)
+
+    def _parse_json(self, line_number: int, raw: str, data: dict) -> LogEntry:
+        """Parse JSON log entry."""
+        entry = LogEntry(line_number=line_number, raw=raw)
+
+        # Extract timestamp
+        for ts_field in ["timestamp", "time", "ts", "@timestamp", "datetime"]:
+            if ts_field in data:
+                try:
+                    ts_str = str(data[ts_field])
+                    entry.timestamp = datetime.fromisoformat(ts_str.replace("Z", "+00:00"))
+                except (ValueError, AttributeError):
+                    pass
+                break
+
+        # Extract level
+        for level_field in ["level", "severity", "loglevel", "lvl"]:
+            if level_field in data:
+                entry.level = str(data[level_field]).upper()
+                break
+
+        # Extract message
+        for msg_field in ["message", "msg", "text", "content"]:
+            if msg_field in data:
+                entry.message = str(data[msg_field])
+                break
+
+        # Extract thread ID
+        for thread_field in ["thread", "thread_id", "threadId", "tid"]:
+            if thread_field in data:
+                entry.thread_id = str(data[thread_field])
+                break
+
+        # Extract correlation ID
+        for corr_field in ["correlation_id", "correlationId", "request_id", "requestId"]:
+            if corr_field in data:
+                entry.correlation_id = str(data[corr_field])
+                break
+
+        # Extract trace/span IDs
+        if "trace_id" in data or "traceId" in data:
+            entry.trace_id = str(data.get("trace_id") or data.get("traceId"))
+        if "span_id" in data or "spanId" in data:
+            entry.span_id = str(data.get("span_id") or data.get("spanId"))
+
+        # Extract service name
+        for service_field in ["service", "service_name", "serviceName"]:
+            if service_field in data:
+                entry.service_name = str(data[service_field])
+                break
+
+        # Store other fields
+        skip_fields = {
+            "timestamp",
+            "time",
+            "ts",
+            "@timestamp",
+            "datetime",
+            "level",
+            "severity",
+            "loglevel",
+            "lvl",
+            "message",
+            "msg",
+            "text",
+            "content",
+            "thread",
+            "thread_id",
+            "threadId",
+            "tid",
+            "correlation_id",
+            "correlationId",
+            "request_id",
+            "requestId",
+            "trace_id",
+            "traceId",
+            "span_id",
+            "spanId",
+            "service",
+            "service_name",
+            "serviceName",
+        }
+        entry.fields = {k: v for k, v in data.items() if k not in skip_fields}
+
+        return entry
+
+    def _parse_plain(self, line_number: int, raw: str) -> LogEntry:
+        """Parse plain text log entry."""
+        entry = LogEntry(line_number=line_number, raw=raw, message=raw)
+
+        # Extract timestamp
+        ts_match = self.PATTERNS["timestamp"].search(raw)
+        if ts_match:
+            try:
+                ts_str = ts_match.group(0)
+                entry.timestamp = datetime.fromisoformat(
+                    ts_str.replace(" ", "T").replace("Z", "+00:00")
+                )
+            except ValueError:
+                pass
+
+        # Extract log level
+        level_match = self.PATTERNS["log_level"].search(raw)
+        if level_match:
+            entry.level = level_match.group(1).upper()
+
+        # Extract thread ID
+        thread_match = self.PATTERNS["thread_id"].search(raw)
+        if thread_match:
+            entry.thread_id = thread_match.group(1)
+
+        # Extract correlation ID
+        corr_match = self.PATTERNS["correlation_id"].search(raw)
+        if corr_match:
+            entry.correlation_id = corr_match.group(1)
+
+        # Extract trace ID
+        trace_match = self.PATTERNS["trace_id"].search(raw)
+        if trace_match:
+            entry.trace_id = trace_match.group(1)
+
+        # Extract span ID
+        span_match = self.PATTERNS["span_id"].search(raw)
+        if span_match:
+            entry.span_id = span_match.group(1)
+
+        return entry

logler/safe_regex.py

@@ -0,0 +1,124 @@
+"""
+Safe regex compilation with timeout protection against ReDoS attacks.
+
+This module provides a safe_compile function that wraps re.compile with:
+- Pattern length validation
+- Compilation timeout (Unix only, graceful fallback on Windows)
+- Clear error messages
+"""
+
+import re
+import threading
+from typing import Optional
+
+
+class RegexTimeoutError(Exception):
+    """Raised when regex compilation times out."""
+
+    pass
+
+
+class RegexPatternTooLongError(Exception):
+    """Raised when regex pattern exceeds maximum allowed length."""
+
+    pass
+
+
+# Maximum pattern length to prevent ReDoS via complexity
+MAX_PATTERN_LENGTH = 1000
+
+# Timeout for regex compilation in seconds
+COMPILE_TIMEOUT = 2.0
+
+
+def _compile_with_timeout(
+    pattern: str,
+    flags: int,
+    timeout: float,
+    result_container: dict,
+) -> None:
+    """Worker function for threaded compilation."""
+    try:
+        result_container["result"] = re.compile(pattern, flags)
+    except re.error as e:
+        result_container["error"] = e
+    except Exception as e:
+        result_container["error"] = e
+
+
+def safe_compile(
+    pattern: str,
+    flags: int = 0,
+    timeout: float = COMPILE_TIMEOUT,
+    max_length: int = MAX_PATTERN_LENGTH,
+) -> re.Pattern:
+    """
+    Safely compile a regex pattern with timeout protection.
+
+    Args:
+        pattern: The regex pattern to compile
+        flags: Optional regex flags (e.g., re.IGNORECASE)
+        timeout: Maximum time in seconds to allow for compilation
+        max_length: Maximum allowed pattern length
+
+    Returns:
+        Compiled regex pattern
+
+    Raises:
+        RegexPatternTooLongError: If pattern exceeds max_length
+        RegexTimeoutError: If compilation takes longer than timeout
+        re.error: If the pattern is invalid
+    """
+    # Validate pattern length
+    if len(pattern) > max_length:
+        raise RegexPatternTooLongError(
+            f"Regex pattern length {len(pattern)} exceeds maximum {max_length}"
+        )
+
+    # Use threading for cross-platform timeout support
+    result_container: dict = {}
+    thread = threading.Thread(
+        target=_compile_with_timeout,
+        args=(pattern, flags, timeout, result_container),
+    )
+    thread.start()
+    thread.join(timeout=timeout)
+
+    if thread.is_alive():
+        # Thread is still running - compilation timed out
+        # Note: We can't actually kill the thread, but we return an error
+        raise RegexTimeoutError(
+            f"Regex compilation timed out after {timeout}s (pattern may cause catastrophic backtracking)"
+        )
+
+    if "error" in result_container:
+        raise result_container["error"]
+
+    return result_container["result"]
+
+
+def try_compile(
+    pattern: str,
+    flags: int = 0,
+    timeout: float = COMPILE_TIMEOUT,
+    max_length: int = MAX_PATTERN_LENGTH,
+) -> Optional[re.Pattern]:
+    """
+    Try to compile a regex pattern safely, returning None on failure.
+
+    This is a convenience wrapper around safe_compile that catches all
+    exceptions and returns None instead.
+
+    Args:
+        pattern: The regex pattern to compile
+        flags: Optional regex flags
+        timeout: Maximum compilation time
+        max_length: Maximum pattern length
+
+    Returns:
+        Compiled pattern or None if compilation fails
+    """
+    try:
+        return safe_compile(pattern, flags, timeout, max_length)
+    except (RegexTimeoutError, RegexPatternTooLongError, re.error):
+        return None