logtap 0.2.0__py3-none-any.whl

logtap/core/parsers/base.py

@@ -0,0 +1,164 @@
+"""Base classes for log parsers."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from datetime import datetime
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+
+class LogLevel(str, Enum):
+    """Standard log severity levels."""
+
+    EMERGENCY = "EMERGENCY"  # System is unusable
+    ALERT = "ALERT"  # Action must be taken immediately
+    CRITICAL = "CRITICAL"  # Critical conditions
+    ERROR = "ERROR"  # Error conditions
+    WARNING = "WARNING"  # Warning conditions
+    NOTICE = "NOTICE"  # Normal but significant
+    INFO = "INFO"  # Informational
+    DEBUG = "DEBUG"  # Debug-level messages
+
+    @classmethod
+    def from_string(cls, level: str) -> Optional["LogLevel"]:
+        """Parse a log level from string."""
+        level_map = {
+            # Standard names
+            "emergency": cls.EMERGENCY,
+            "emerg": cls.EMERGENCY,
+            "alert": cls.ALERT,
+            "critical": cls.CRITICAL,
+            "crit": cls.CRITICAL,
+            "error": cls.ERROR,
+            "err": cls.ERROR,
+            "warning": cls.WARNING,
+            "warn": cls.WARNING,
+            "notice": cls.NOTICE,
+            "info": cls.INFO,
+            "information": cls.INFO,
+            "informational": cls.INFO,
+            "debug": cls.DEBUG,
+            # Numeric syslog levels
+            "0": cls.EMERGENCY,
+            "1": cls.ALERT,
+            "2": cls.CRITICAL,
+            "3": cls.ERROR,
+            "4": cls.WARNING,
+            "5": cls.NOTICE,
+            "6": cls.INFO,
+            "7": cls.DEBUG,
+        }
+        return level_map.get(level.lower())
+
+    @property
+    def severity(self) -> int:
+        """Get numeric severity (0=most severe, 7=least severe)."""
+        severity_map = {
+            LogLevel.EMERGENCY: 0,
+            LogLevel.ALERT: 1,
+            LogLevel.CRITICAL: 2,
+            LogLevel.ERROR: 3,
+            LogLevel.WARNING: 4,
+            LogLevel.NOTICE: 5,
+            LogLevel.INFO: 6,
+            LogLevel.DEBUG: 7,
+        }
+        return severity_map[self]
+
+
+@dataclass
+class ParsedLogEntry:
+    """A parsed log entry with structured fields."""
+
+    raw: str  # Original log line
+    message: str  # Main message content
+    timestamp: Optional[datetime] = None
+    level: Optional[LogLevel] = None
+    source: Optional[str] = None  # hostname, process, etc.
+    metadata: Dict[str, Any] = field(default_factory=dict)
+
+    def to_dict(self) -> Dict[str, Any]:
+        """Convert to dictionary for JSON serialization."""
+        return {
+            "raw": self.raw,
+            "message": self.message,
+            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
+            "level": self.level.value if self.level else None,
+            "source": self.source,
+            "metadata": self.metadata,
+        }
+
+
+class LogParser(ABC):
+    """Abstract base class for log parsers."""
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """Name of this parser format."""
+        pass
+
+    @abstractmethod
+    def can_parse(self, line: str) -> bool:
+        """
+        Check if this parser can handle the given line.
+
+        Args:
+            line: A log line to check.
+
+        Returns:
+            True if this parser can parse the line.
+        """
+        pass
+
+    @abstractmethod
+    def parse(self, line: str) -> ParsedLogEntry:
+        """
+        Parse a log line into structured format.
+
+        Args:
+            line: The log line to parse.
+
+        Returns:
+            A ParsedLogEntry with extracted fields.
+        """
+        pass
+
+    def parse_many(self, lines: List[str]) -> List[ParsedLogEntry]:
+        """
+        Parse multiple log lines.
+
+        Args:
+            lines: List of log lines to parse.
+
+        Returns:
+            List of ParsedLogEntry objects.
+        """
+        return [self.parse(line) for line in lines]
+
+    def _detect_level_from_content(self, content: str) -> Optional[LogLevel]:
+        """
+        Detect log level from message content.
+
+        Common patterns like "ERROR:", "[error]", etc.
+        """
+        content_lower = content.lower()
+
+        # Check for explicit level indicators
+        level_patterns = [
+            (["emergency", "emerg"], LogLevel.EMERGENCY),
+            (["alert"], LogLevel.ALERT),
+            (["critical", "crit", "fatal"], LogLevel.CRITICAL),
+            (["error", "err", "fail", "failed"], LogLevel.ERROR),
+            (["warning", "warn"], LogLevel.WARNING),
+            (["notice"], LogLevel.NOTICE),
+            (["info"], LogLevel.INFO),
+            (["debug", "trace"], LogLevel.DEBUG),
+        ]
+
+        for patterns, level in level_patterns:
+            for pattern in patterns:
+                if pattern in content_lower:
+                    return level
+
+        return None

logtap/core/parsers/json_parser.py

@@ -0,0 +1,119 @@
+"""JSON log format parser."""
+
+import json
+from datetime import datetime
+from typing import Any, Dict, Optional
+
+from logtap.core.parsers.base import LogLevel, LogParser, ParsedLogEntry
+
+
+class JsonLogParser(LogParser):
+    """
+    Parser for JSON-formatted log lines.
+
+    Handles common JSON log formats with fields like:
+    - message, msg, log
+    - level, severity, loglevel
+    - timestamp, time, @timestamp, ts
+    - source, logger, name
+    """
+
+    # Common field names for each attribute
+    MESSAGE_FIELDS = ["message", "msg", "log", "text", "body"]
+    LEVEL_FIELDS = ["level", "severity", "loglevel", "log_level", "lvl"]
+    TIMESTAMP_FIELDS = ["timestamp", "time", "@timestamp", "ts", "datetime", "date"]
+    SOURCE_FIELDS = ["source", "logger", "name", "service", "app", "application"]
+
+    @property
+    def name(self) -> str:
+        return "json"
+
+    def can_parse(self, line: str) -> bool:
+        """Check if line is valid JSON."""
+        line = line.strip()
+        if not line.startswith("{"):
+            return False
+        try:
+            json.loads(line)
+            return True
+        except (json.JSONDecodeError, ValueError):
+            return False
+
+    def parse(self, line: str) -> ParsedLogEntry:
+        """Parse a JSON log line."""
+        try:
+            data = json.loads(line.strip())
+        except (json.JSONDecodeError, ValueError):
+            return ParsedLogEntry(
+                raw=line,
+                message=line,
+                level=self._detect_level_from_content(line),
+            )
+
+        # Extract message
+        message = self._get_field(data, self.MESSAGE_FIELDS, line)
+
+        # Extract level
+        level_str = self._get_field(data, self.LEVEL_FIELDS)
+        level = None
+        if level_str:
+            level = LogLevel.from_string(str(level_str))
+        if not level:
+            level = self._detect_level_from_content(message)
+
+        # Extract timestamp
+        timestamp_str = self._get_field(data, self.TIMESTAMP_FIELDS)
+        timestamp = self._parse_timestamp(timestamp_str) if timestamp_str else None
+
+        # Extract source
+        source = self._get_field(data, self.SOURCE_FIELDS)
+
+        return ParsedLogEntry(
+            raw=line,
+            message=message,
+            timestamp=timestamp,
+            level=level,
+            source=source,
+            metadata=data,
+        )
+
+    def _get_field(self, data: Dict[str, Any], field_names: list, default: Any = None) -> Any:
+        """Get first matching field from data."""
+        for field in field_names:
+            if field in data:
+                return data[field]
+            # Check case-insensitive
+            for key in data:
+                if key.lower() == field.lower():
+                    return data[key]
+        return default
+
+    def _parse_timestamp(self, value: Any) -> Optional[datetime]:
+        """Parse timestamp from various formats."""
+        if isinstance(value, datetime):
+            return value
+
+        if isinstance(value, (int, float)):
+            # Unix timestamp
+            try:
+                return datetime.fromtimestamp(value)
+            except (ValueError, OSError):
+                pass
+
+        if isinstance(value, str):
+            # Try common formats
+            formats = [
+                "%Y-%m-%dT%H:%M:%S.%fZ",
+                "%Y-%m-%dT%H:%M:%SZ",
+                "%Y-%m-%dT%H:%M:%S.%f",
+                "%Y-%m-%dT%H:%M:%S",
+                "%Y-%m-%d %H:%M:%S.%f",
+                "%Y-%m-%d %H:%M:%S",
+            ]
+            for fmt in formats:
+                try:
+                    return datetime.strptime(value, fmt)
+                except ValueError:
+                    continue
+
+        return None

logtap/core/parsers/nginx.py

@@ -0,0 +1,108 @@
+"""Nginx access log parser."""
+
+import re
+from datetime import datetime
+from typing import Optional
+
+from logtap.core.parsers.base import LogLevel, LogParser, ParsedLogEntry
+
+
+class NginxParser(LogParser):
+    """
+    Parser for Nginx access log format (combined log format).
+
+    Example:
+    192.168.1.1 - - [08/Jan/2024:10:23:45 +0000] "GET /api HTTP/1.1" 200 45
+    """
+
+    # Combined log format pattern
+    PATTERN = re.compile(
+        r"^(\S+)\s+"  # Remote address
+        r"(\S+)\s+"  # Identity (usually -)
+        r"(\S+)\s+"  # Remote user (usually -)
+        r"\[([^\]]+)\]\s+"  # Time
+        r'"([^"]*)"\s+'  # Request
+        r"(\d{3})\s+"  # Status
+        r"(\d+|-)\s*"  # Bytes
+        r'(?:"([^"]*)"\s*)?'  # Referer (optional)
+        r'(?:"([^"]*)")?'  # User agent (optional)
+    )
+
+    @property
+    def name(self) -> str:
+        return "nginx"
+
+    def can_parse(self, line: str) -> bool:
+        """Check if line matches nginx format."""
+        return bool(self.PATTERN.match(line))
+
+    def parse(self, line: str) -> ParsedLogEntry:
+        """Parse an nginx access log line."""
+        match = self.PATTERN.match(line)
+
+        if not match:
+            return ParsedLogEntry(
+                raw=line,
+                message=line,
+                level=self._detect_level_from_content(line),
+            )
+
+        groups = match.groups()
+        remote_addr = groups[0]
+        remote_user = groups[2] if groups[2] != "-" else None
+        time_str = groups[3]
+        request = groups[4]
+        status = int(groups[5])
+        bytes_sent = int(groups[6]) if groups[6] != "-" else 0
+        referer = groups[7] if len(groups) > 7 and groups[7] != "-" else None
+        user_agent = groups[8] if len(groups) > 8 else None
+
+        # Parse timestamp
+        timestamp = self._parse_nginx_time(time_str)
+
+        # Determine level based on status code
+        level = self._status_to_level(status)
+
+        # Parse request method and path
+        request_parts = request.split() if request else []
+        method = request_parts[0] if len(request_parts) > 0 else None
+        path = request_parts[1] if len(request_parts) > 1 else None
+
+        return ParsedLogEntry(
+            raw=line,
+            message=f"{method} {path} -> {status}" if method and path else request,
+            timestamp=timestamp,
+            level=level,
+            source=remote_addr,
+            metadata={
+                "remote_addr": remote_addr,
+                "remote_user": remote_user,
+                "request": request,
+                "method": method,
+                "path": path,
+                "status": status,
+                "bytes_sent": bytes_sent,
+                "referer": referer,
+                "user_agent": user_agent,
+            },
+        )
+
+    def _parse_nginx_time(self, time_str: str) -> Optional[datetime]:
+        """Parse nginx time format: 08/Jan/2024:10:23:45 +0000"""
+        try:
+            # Remove timezone for simpler parsing
+            time_str = time_str.split()[0] if " " in time_str else time_str
+            return datetime.strptime(time_str, "%d/%b/%Y:%H:%M:%S")
+        except ValueError:
+            return None
+
+    def _status_to_level(self, status: int) -> LogLevel:
+        """Convert HTTP status code to log level."""
+        if status >= 500:
+            return LogLevel.ERROR
+        elif status >= 400:
+            return LogLevel.WARNING
+        elif status >= 300:
+            return LogLevel.NOTICE
+        else:
+            return LogLevel.INFO

logtap/core/parsers/syslog.py

@@ -0,0 +1,80 @@
+"""Syslog format parser."""
+
+import re
+from datetime import datetime
+
+from logtap.core.parsers.base import LogParser, ParsedLogEntry
+
+
+class SyslogParser(LogParser):
+    """
+    Parser for standard syslog format.
+
+    Handles formats like:
+    - Jan  8 10:23:45 hostname process[pid]: message
+    - Jan  8 10:23:45 hostname process: message
+    """
+
+    # Syslog pattern: Month Day HH:MM:SS hostname process[pid]: message
+    PATTERN = re.compile(
+        r"^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+"  # Timestamp
+        r"(\S+)\s+"  # Hostname
+        r"(\S+?)(?:\[(\d+)\])?:\s+"  # Process[PID]
+        r"(.*)$"  # Message
+    )
+
+    # Alternative pattern without PID brackets
+    PATTERN_ALT = re.compile(
+        r"^(\w{3}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2})\s+" r"(\S+)\s+" r"(\S+):\s+" r"(.*)$"
+    )
+
+    @property
+    def name(self) -> str:
+        return "syslog"
+
+    def can_parse(self, line: str) -> bool:
+        """Check if line matches syslog format."""
+        return bool(self.PATTERN.match(line) or self.PATTERN_ALT.match(line))
+
+    def parse(self, line: str) -> ParsedLogEntry:
+        """Parse a syslog line."""
+        match = self.PATTERN.match(line)
+
+        if match:
+            timestamp_str, hostname, process, pid, message = match.groups()
+        else:
+            match = self.PATTERN_ALT.match(line)
+            if match:
+                timestamp_str, hostname, process, message = match.groups()
+                pid = None
+            else:
+                # Can't parse, return basic entry
+                return ParsedLogEntry(
+                    raw=line,
+                    message=line,
+                    level=self._detect_level_from_content(line),
+                )
+
+        # Parse timestamp (assume current year)
+        try:
+            timestamp = datetime.strptime(timestamp_str, "%b %d %H:%M:%S")
+            # Set to current year
+            timestamp = timestamp.replace(year=datetime.now().year)
+        except ValueError:
+            timestamp = None
+
+        # Detect level from message
+        level = self._detect_level_from_content(message)
+
+        return ParsedLogEntry(
+            raw=line,
+            message=message,
+            timestamp=timestamp,
+            level=level,
+            source=f"{hostname}/{process}",
+            metadata={
+                "hostname": hostname,
+                "process": process,
+                "pid": pid,
+            },
+        )

logtap/core/reader.py

@@ -0,0 +1,160 @@
+"""
+Core file reading functionality for logtap.
+
+The tail() function is the heart of logtap - an efficient O(n) algorithm that reads
+files from the end backwards in chunks, avoiding the need to load entire files into memory.
+"""
+
+from os import SEEK_END
+from typing import IO, List, Optional, Tuple
+
+import aiofiles
+
+
+def tail(filename: str, lines_limit: int = 50, block_size: int = 1024) -> List[str]:
+    """
+    Reads a file in reverse and returns its last 'lines_limit' lines.
+
+    This is an efficient algorithm that reads from the end of the file backwards
+    in chunks, making it suitable for very large log files.
+
+    Args:
+        filename: The path to the file to be read.
+        lines_limit: The maximum number of lines to be returned. Defaults to 50.
+        block_size: The number of bytes to read at a time. Defaults to 1024.
+
+    Returns:
+        A list of the last 'lines_limit' lines in the file.
+    """
+    lines: List[str] = []
+    with open(filename, "r", encoding="utf-8") as f:
+        # Seek to the end of the file.
+        f.seek(0, SEEK_END)
+        # Get the current position in the file.
+        block_end_byte = f.tell()
+
+        # Continue reading blocks and adding lines until we have enough lines
+        # or reach the start of the file.
+        while len(lines) < lines_limit and block_end_byte > 0:
+            # Read a block from the file, update the block_end_byte position,
+            # and get the new lines.
+            new_lines, block_end_byte = read_block(f, block_end_byte, block_size)
+            lines.extend(new_lines)
+
+        # Return the last 'lines_limit' lines
+        return lines[-lines_limit:]
+
+
+def read_block(file: IO, block_end_byte: int, block_size: int) -> Tuple[List[str], int]:
+    """
+    Reads a block from the end of a file and returns the lines in the block.
+
+    Args:
+        file: The file object to read from.
+        block_end_byte: The current position in the file.
+        block_size: The number of bytes to read at a time.
+
+    Returns:
+        A tuple containing the list of lines in the block,
+        and the updated position in the file.
+    """
+    # Use min() to ensure we only step back as far as we can (start of file)
+    stepback = min(block_size, block_end_byte)
+
+    # Step back and read a block from the file
+    file.seek(block_end_byte - stepback)
+    block = file.read(stepback)
+    block_end_byte -= stepback
+    lines = block.split("\n")
+
+    return lines, block_end_byte
+
+
+async def tail_async(
+    filename: str, lines_limit: int = 50, block_size: int = 1024
+) -> List[str]:
+    """
+    Async version of tail() for use with FastAPI.
+
+    Reads a file in reverse and returns its last 'lines_limit' lines.
+
+    Args:
+        filename: The path to the file to be read.
+        lines_limit: The maximum number of lines to be returned. Defaults to 50.
+        block_size: The number of bytes to read at a time. Defaults to 1024.
+
+    Returns:
+        A list of the last 'lines_limit' lines in the file.
+    """
+    lines: List[str] = []
+    async with aiofiles.open(filename, "r", encoding="utf-8") as f:
+        # Seek to the end of the file.
+        await f.seek(0, SEEK_END)
+        # Get the current position in the file.
+        block_end_byte = await f.tell()
+
+        # Continue reading blocks and adding lines until we have enough lines
+        # or reach the start of the file.
+        while len(lines) < lines_limit and block_end_byte > 0:
+            # Read a block from the file
+            stepback = min(block_size, block_end_byte)
+            await f.seek(block_end_byte - stepback)
+            block = await f.read(stepback)
+            block_end_byte -= stepback
+            new_lines = block.split("\n")
+            lines.extend(new_lines)
+
+        # Return the last 'lines_limit' lines
+        return lines[-lines_limit:]
+
+
+def get_file_lines(
+    filepath: str,
+    search_term: Optional[str] = None,
+    num_lines_to_return: int = 50,
+) -> List[str]:
+    """
+    Retrieves a specified number of lines from a file,
+    optionally filtering for a search term.
+
+    Args:
+        filepath: Path to the file.
+        search_term: Term to filter lines. If None or empty, no filtering is applied.
+        num_lines_to_return: Number of lines to retrieve from the end of the file.
+
+    Returns:
+        List of lines from the file.
+    """
+    lines = tail(filepath, num_lines_to_return)
+
+    if search_term:
+        lines = [line for line in lines if search_term in line]
+
+    return lines
+
+
+async def get_file_lines_async(
+    filepath: str,
+    search_term: Optional[str] = None,
+    num_lines_to_return: int = 50,
+) -> List[str]:
+    """
+    Async version of get_file_lines() for use with FastAPI.
+
+    Retrieves a specified number of lines from a file,
+    optionally filtering for a search term.
+
+    Args:
+        filepath: Path to the file.
+        search_term: Term to filter lines. If None or empty, no filtering is applied.
+        num_lines_to_return: Number of lines to retrieve from the end of the file.
+
+    Returns:
+        List of lines from the file.
+    """
+    lines = await tail_async(filepath, num_lines_to_return)
+
+    if search_term:
+        lines = [line for line in lines if search_term in line]
+
+    return lines