logloglog 0.0.1__py3-none-any.whl

logloglog/__init__.py

@@ -0,0 +1,40 @@
+"""LogLogLog - Efficient scrollback indexing for large log files."""
+
+import logging
+import sys
+from importlib.metadata import version, PackageNotFoundError
+
+
+from .logloglog import LogLogLog
+from .widthview import WidthView
+
+try:
+    __version__ = version("logloglog")
+except PackageNotFoundError:
+    # Package is not installed
+    __version__ = "0.0.0+dev"
+
+__all__ = ["LogLogLog", "WidthView", "configure_logging"]
+
+
+# Configure logging for LogLogLog
+def configure_logging(level=logging.INFO):
+    """Configure logging for LogLogLog."""
+    # Create formatter
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    # Create console handler
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(formatter)
+
+    # Configure logloglog loggers
+    for logger_name in ["logloglog.logloglog", "logloglog.wraptree", "logloglog.index"]:
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(level)
+        # Remove existing handlers to avoid duplicates
+        logger.handlers.clear()
+        logger.addHandler(handler)
+
+
+# Auto-configure with DEBUG level for performance monitoring
+configure_logging(logging.DEBUG)

logloglog/__main__.py

@@ -0,0 +1,8 @@
+"""
+Entry point for the package
+"""
+
+from .program import run_program
+
+if __name__ == "__main__":
+    run_program()

logloglog/cache.py

@@ -0,0 +1,85 @@
+"""Cache management for LogLogLog."""
+
+import os
+import hashlib
+import shutil
+import tempfile
+from pathlib import Path
+import platformdirs
+
+
+# Cache constants
+CACHE_DIR = Path(platformdirs.user_cache_dir("logloglog"))
+TMP_DIR = Path(tempfile.gettempdir()) / "logloglog"
+
+
+class Cache:
+    """Manages cache directories for log files."""
+
+    def __init__(self, cache_dir: Path = None):
+        """Initialize cache manager.
+
+        Args:
+            cache_dir: Cache directory (defaults to CACHE_DIR)
+        """
+        self.cache_dir = cache_dir or CACHE_DIR
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_dir(self, path: Path) -> Path:
+        """Get cache directory for a log file.
+
+        Args:
+            path: Path to the log file
+
+        Returns:
+            Cache directory path
+
+        Raises:
+            OSError: If file cannot be accessed or cache directory cannot be created
+        """
+        # Get file stats for unique identification
+        stat = os.stat(path)
+        # Create hash from device and inode only (removed ctime for stability)
+        hash_input = f"{stat.st_dev}_{stat.st_ino}"
+        hash_digest = hashlib.md5(hash_input.encode()).hexdigest()[:8]
+
+        # Create cache directory name
+        name = path.name
+        cache_name = f"{name}[{hash_digest}]"
+        cache_path = self.cache_dir / cache_name
+
+        # Create directory if it doesn't exist
+        cache_path.mkdir(parents=True, exist_ok=True)
+
+        # Create symlink to original file for reference
+        symlink_path = cache_path / "file"
+        if not symlink_path.exists():
+            symlink_path.symlink_to(path.resolve())
+
+        return cache_path
+
+    def cleanup(self):
+        """Clean up cache directories for files that no longer exist."""
+        if not self.cache_dir.exists():
+            return
+
+        for cache_subdir in self.cache_dir.iterdir():
+            if not cache_subdir.is_dir():
+                continue
+
+            # Check if the symlink exists and points to a valid file
+            symlink_path = cache_subdir / "file"
+            if symlink_path.exists():
+                try:
+                    # Try to resolve the symlink
+                    target = symlink_path.resolve()
+                    if not target.exists():
+                        # Original file is gone, remove cache directory
+                        shutil.rmtree(cache_subdir)
+                except (OSError, FileNotFoundError):
+                    # Symlink is broken, remove cache directory
+                    shutil.rmtree(cache_subdir)
+            else:
+                # No symlink found, directory is orphaned
+                shutil.rmtree(cache_subdir)
+            # TODO: Add inode-based cleanup for more robust file tracking

logloglog/line_index.py

@@ -0,0 +1,274 @@
+"""Simple line indexing with periodic summaries for efficient wrapping calculations."""
+
+import logging
+from pathlib import Path
+from typing import Tuple
+from arrayfile import Array
+
+logger = logging.getLogger(__name__)
+
+# Configuration
+MAX_WIDTH = 512  # Maximum terminal width we support
+SUMMARY_INTERVAL = 1000  # Store summary every N lines
+
+
+class LineIndex:
+    """
+    Indexes log lines with byte positions, widths, and periodic summaries.
+
+    Stores:
+    - line_positions: byte offset of each line in the log file
+    - line_widths: display width of each line
+    - summaries: every 1000 lines, cumulative display rows for each width 1-512
+    """
+
+    def __init__(self, index_path: Path):
+        """Initialize line index with given path."""
+        self.index_path = index_path
+        self._line_positions = None
+        self._line_widths = None
+        self._summaries = None
+        self._line_count = 0
+        self._current_block_width_counts = {}  # Track widths in current 1000-line block
+        self._pending_positions = []  # Batch positions for extend()
+        self._pending_widths = []  # Batch widths for extend()
+
+    def open(self, create: bool = False):
+        """Open index files."""
+        self.index_path.mkdir(parents=True, exist_ok=True)
+
+        mode = "w+b" if create else "r+b"
+
+        # Line positions (uint64 for file offsets)
+        self._line_positions = Array("Q", str(self.index_path / "positions.dat"), mode)
+
+        # Line widths (uint16, capped at 65535)
+        self._line_widths = Array("H", str(self.index_path / "widths.dat"), mode)
+
+        # Summaries (uint32 array, MAX_WIDTH entries per summary)
+        self._summaries = Array("I", str(self.index_path / "summaries.dat"), mode)
+
+        # Count existing lines
+        self._line_count = len(self._line_positions)
+
+    def close(self):
+        """Close all index files."""
+        # Flush any pending data before closing
+        self._flush_pending()
+
+        if self._line_positions:
+            self._line_positions.close()
+            self._line_positions = None
+        if self._line_widths:
+            self._line_widths.close()
+            self._line_widths = None
+        if self._summaries:
+            self._summaries.close()
+            self._summaries = None
+
+    def append_line(self, position: int, width: int):
+        """
+        Append a new line to the index.
+
+        Args:
+            position: Byte offset of line start in log file
+            width: Display width of the line
+        """
+        # Cap width at uint16 max
+        width = min(width, 65535)
+
+        # Batch in memory
+        self._pending_positions.append(position)
+        self._pending_widths.append(width)
+        self._line_count += 1
+
+        # Track width for current block
+        self._current_block_width_counts[width] = self._current_block_width_counts.get(width, 0) + 1
+
+        # Check if we need to flush and store a summary
+        if self._line_count % SUMMARY_INTERVAL == 0:
+            self._flush_pending()
+            self._store_summary()
+            self._current_block_width_counts.clear()
+
+    def _flush_pending(self):
+        """Flush pending positions and widths to disk."""
+        if self._pending_positions:
+            self._line_positions.extend(self._pending_positions)
+            self._pending_positions.clear()
+        if self._pending_widths:
+            self._line_widths.extend(self._pending_widths)
+            self._pending_widths.clear()
+
+    def _store_summary(self):
+        """Store summary using already-tracked width counts."""
+        # Calculate totals for each terminal width
+        width_totals = [0] * MAX_WIDTH
+        for line_width, count in self._current_block_width_counts.items():
+            if line_width == 0:
+                # Empty lines always take 1 row regardless of terminal width
+                for i in range(MAX_WIDTH):
+                    width_totals[i] += count
+            else:
+                # Calculate rows for each terminal width
+                # Ceiling division: (line_width + term_width - 1) // term_width
+                # This is always >= 1 when both operands are positive
+                for term_width in range(1, MAX_WIDTH + 1):
+                    rows = (line_width + term_width - 1) // term_width
+                    width_totals[term_width - 1] += rows * count
+
+        # Store all width totals in summary array (batch append for performance)
+        self._summaries.extend(width_totals)
+
+    def get_line_position(self, line_no: int) -> int:
+        """Get byte position of a line."""
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        # Check if it's in the flushed data or pending batch
+        flushed_count = len(self._line_positions)
+        if line_no < flushed_count:
+            return self._line_positions[line_no]
+        else:
+            # It's in the pending batch
+            pending_idx = line_no - flushed_count
+            return self._pending_positions[pending_idx]
+
+    def get_line_width(self, line_no: int) -> int:
+        """Get display width of a line."""
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        # Check if it's in the flushed data or pending batch
+        flushed_count = len(self._line_widths)
+        if line_no < flushed_count:
+            return self._line_widths[line_no]
+        else:
+            # It's in the pending batch
+            pending_idx = line_no - flushed_count
+            return self._pending_widths[pending_idx]
+
+    def get_total_display_rows(self, width: int) -> int:
+        """
+        Get total display rows for all lines at given terminal width.
+
+        Args:
+            width: Terminal width
+
+        Returns:
+            Total number of display rows
+        """
+        if width <= 0:
+            return 0  # No display possible with zero or negative width
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        total_rows = 0
+
+        # Add up complete summaries
+        complete_summaries = self._line_count // SUMMARY_INTERVAL
+        for i in range(complete_summaries):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            total_rows += self._summaries[summary_offset]
+
+        # Add remaining lines not in a summary
+        start_line = complete_summaries * SUMMARY_INTERVAL
+        for line_idx in range(start_line, self._line_count):
+            line_width = self.get_line_width(line_idx)
+            # Ceiling division is always >= 1 for positive operands
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+            total_rows += rows
+
+        return total_rows
+
+    def get_display_row_for_line(self, line_no: int, width: int) -> int:
+        """
+        Get the display row number where a logical line starts.
+
+        Args:
+            line_no: Logical line number
+            width: Terminal width
+
+        Returns:
+            Display row number where this line starts
+        """
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        if width <= 0:
+            return 0  # No display possible
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        display_row = 0
+
+        # Add complete summaries before this line
+        summary_idx = line_no // SUMMARY_INTERVAL
+        for i in range(summary_idx):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            display_row += self._summaries[summary_offset]
+
+        # Add individual lines from last summary to target line
+        start_line = summary_idx * SUMMARY_INTERVAL
+        for line_idx in range(start_line, line_no):
+            line_width = self.get_line_width(line_idx)
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+            display_row += rows
+
+        return display_row
+
+    def get_line_for_display_row(self, display_row: int, width: int) -> Tuple[int, int]:
+        """
+        Find the logical line containing the given display row.
+
+        Args:
+            display_row: Display row to find
+            width: Terminal width
+
+        Returns:
+            Tuple of (line_number, row_offset_within_line)
+        """
+        if width <= 0:
+            raise IndexError(f"Display row {display_row} out of range")  # No display possible
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        current_row = 0
+
+        # Binary search through summaries to find the right range
+        complete_summaries = self._line_count // SUMMARY_INTERVAL
+        summary_idx = 0
+
+        # Find which summary block contains our display row
+        for i in range(complete_summaries):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            summary_rows = self._summaries[summary_offset]
+            if current_row + summary_rows > display_row:
+                summary_idx = i
+                break
+            current_row += summary_rows
+        else:
+            # It's in the incomplete last block
+            summary_idx = complete_summaries
+
+        # Linear search within the summary block
+        start_line = summary_idx * SUMMARY_INTERVAL
+        end_line = min(start_line + SUMMARY_INTERVAL, self._line_count)
+
+        for line_idx in range(start_line, end_line):
+            line_width = self.get_line_width(line_idx)
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+
+            if current_row + rows > display_row:
+                # Found the line
+                offset_within_line = display_row - current_row
+                return (line_idx, offset_within_line)
+
+            current_row += rows
+
+        # Display row is beyond the end
+        raise IndexError(f"Display row {display_row} out of range")
+
+    def __len__(self) -> int:
+        """Get total number of indexed lines."""
+        return self._line_count

logloglog/log_file.py

@@ -0,0 +1,187 @@
+"""Simple file abstraction for log file operations."""
+
+import asyncio
+from pathlib import Path
+from typing import Optional, Union
+
+
+class LogFile:
+    """
+    Simple file abstraction for reading and writing log files.
+
+    Keeps file handle open during batch operations for performance.
+    Call open() to start a batch read session, close() when done.
+    Individual operations (append, get_size) open/close as needed.
+    """
+
+    def __init__(self, path: Union[Path, str], mode: str = "r"):
+        """
+        Initialize LogFile.
+
+        Args:
+            path: Path to the log file
+            mode: File mode - "r" for read-only, "a" for append, "w" for write
+        """
+        self.path = Path(path)
+        self.mode = mode
+        self._read_position = 0
+        self._file_handle = None
+
+        # Validate mode
+        if mode not in ("r", "a", "w"):
+            raise ValueError(f"Invalid mode: {mode}. Must be 'r', 'a', or 'w'")
+
+        # Create file if it doesn't exist and we're in write/append mode
+        if mode in ("a", "w") and not self.path.exists():
+            self.path.touch()
+
+    def open(self):
+        """Open the file for reading. Call this before batch read operations."""
+        if self._file_handle is None:
+            self._file_handle = open(self.path, "rb")
+            self._file_handle.seek(self._read_position)
+
+    def close(self):
+        """Close the file handle. Call this after batch operations complete."""
+        if self._file_handle is not None:
+            self._file_handle.close()
+            self._file_handle = None
+
+    def read_line(self) -> Optional[str]:
+        """
+        Read the next line from the current position.
+
+        Returns:
+            The next line without trailing newline, or None if no more data.
+        """
+        try:
+            line_bytes = self._file_handle.readline()
+            if line_bytes:
+                # Track position without syscall - we know it's current + bytes read
+                self._read_position += len(line_bytes)
+                return line_bytes.decode("utf-8", errors="replace").rstrip("\r\n")
+        except (IOError, OSError):
+            pass
+        return None
+
+    def read_all_lines(self) -> list[str]:
+        """
+        Read all remaining lines from current position.
+
+        Returns:
+            List of lines without trailing newlines.
+        """
+        lines = []
+        while line := self.read_line():
+            lines.append(line)
+        return lines
+
+    def append_line(self, line: str) -> None:
+        """
+        Append a line to the file.
+
+        Args:
+            line: Line to append (newline will be added automatically)
+
+        Raises:
+            IOError: If file is opened in read-only mode
+        """
+        if self.mode == "r":
+            raise IOError("Cannot write to file opened in read-only mode")
+
+        with open(self.path, "ab") as f:
+            # Ensure line ends with newline
+            if not line.endswith("\n"):
+                line += "\n"
+            f.write(line.encode("utf-8"))
+
+    def append_lines(self, lines: list[str]) -> None:
+        """
+        Append multiple lines to the file.
+
+        Args:
+            lines: Lines to append (newlines will be added as needed)
+        """
+        if self.mode == "r":
+            raise IOError("Cannot write to file opened in read-only mode")
+
+        with open(self.path, "ab") as f:
+            for line in lines:
+                if not line.endswith("\n"):
+                    line += "\n"
+                f.write(line.encode("utf-8"))
+
+    def has_more_data(self) -> bool:
+        """
+        Check if there's more data available to read.
+
+        Returns:
+            True if file has grown beyond current read position.
+        """
+        try:
+            return self.path.stat().st_size > self._read_position
+        except (IOError, OSError):
+            return False
+
+    def get_size(self) -> int:
+        """
+        Get current file size in bytes.
+
+        Returns:
+            File size in bytes, or 0 if file doesn't exist.
+        """
+        try:
+            return self.path.stat().st_size
+        except (IOError, OSError):
+            return 0
+
+    def seek_to(self, position: int) -> None:
+        """
+        Set the read position.
+
+        Args:
+            position: Byte position to seek to
+        """
+        self._read_position = max(0, position)
+        # If file handle is open, seek it too
+        if self._file_handle is not None:
+            self._file_handle.seek(self._read_position)
+
+    def get_position(self) -> int:
+        """
+        Get current read position.
+
+        Returns:
+            Current byte position for reads.
+        """
+        return self._read_position
+
+    def reset(self) -> None:
+        """Reset read position to beginning of file."""
+        self._read_position = 0
+
+    # Async versions for Textual/asyncio compatibility
+
+    async def aread_line(self) -> Optional[str]:
+        """Async version of read_line()."""
+        return await asyncio.to_thread(self.read_line)
+
+    async def aread_all_lines(self) -> list[str]:
+        """Async version of read_all_lines()."""
+        return await asyncio.to_thread(self.read_all_lines)
+
+    async def aappend_line(self, line: str) -> None:
+        """Async version of append_line()."""
+        await asyncio.to_thread(self.append_line, line)
+
+    async def aappend_lines(self, lines: list[str]) -> None:
+        """Async version of append_lines()."""
+        await asyncio.to_thread(self.append_lines, lines)
+
+    async def ahas_more_data(self) -> bool:
+        """Async version of has_more_data()."""
+        return await asyncio.to_thread(self.has_more_data)
+
+    async def aget_size(self) -> int:
+        """Async version of get_size()."""
+        return await asyncio.to_thread(self.get_size)