logloglog 0.0.1__tar.gz

logloglog-0.0.1/LICENSE.md

@@ -0,0 +1,7 @@
+# WTFPL + Warranty
+
+Licensed under the WTFPL with one additional clause:
+
+1. Don't blame me.
+
+Do whatever the fuck you want, just don't blame me.

logloglog-0.0.1/PKG-INFO

@@ -0,0 +1,73 @@
+Metadata-Version: 2.4
+Name: logloglog
+Version: 0.0.1
+Summary: Efficient scrollback indexing for large log files with terminal word wrapping
+Keywords: logging,terminal,scrollback,indexing
+Author-email: Gareth Davidson <gaz@bitplane.net>
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: License :: Public Domain
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: System :: Logging
+Classifier: Topic :: Terminals
+License-File: LICENSE.md
+Requires-Dist: wcwidth~=0.2.5
+Requires-Dist: platformdirs~=3.0
+Requires-Dist: arrayfile~=0.0.1
+Requires-Dist: logloglog[textual] ; extra == "dev"
+Requires-Dist: pre-commit ; extra == "dev"
+Requires-Dist: pytest ; extra == "dev"
+Requires-Dist: pytest-asyncio ; extra == "dev"
+Requires-Dist: coverage ; extra == "dev"
+Requires-Dist: pytest-cov ; extra == "dev"
+Requires-Dist: build ; extra == "dev"
+Requires-Dist: twine ; extra == "dev"
+Requires-Dist: ruff ; extra == "dev"
+Requires-Dist: textual ; extra == "textual"
+Requires-Dist: textual-window ; extra == "textual"
+Provides-Extra: dev
+Provides-Extra: textual
+
+# 🪵🪵🪵
+
+![log](log.webp)
+
+> What rolls down stairs, alone or in pairs, and over your neighbor's dog?
+> What's great for a snack, And fits on your back?
+> It's Log Log Log!
+
+I needed line wrapping in my tty with fast access, so why not make a log that
+supports egregious sizes and low seek times?
+
+## ⏩
+
+```bash
+# makes log.log.log from /var/log
+make log
+```
+
+## TODO
+
+- [ ] Textual demo
+  - [x] stats in window
+  - [ ] slim demo down
+  - [x] follow last line when at end
+- [x] Async/non-blocking design
+  - [x] Make log processing async to avoid blocking
+  - [x] Support streaming updates
+- [ ] Multiple display backends
+- [ ] Python logging integration
+  - [ ] Direct logger handlers
+  - [ ] Log level filtering and formatting
+- [ ] Cache management
+  - [ ] Periodic cleanup of cache directories
+  - [ ] Check file existence by inode lookup
+  - [ ] Handle file rotation and moves
+
+

logloglog-0.0.1/README.md

@@ -0,0 +1,36 @@
+# 🪵🪵🪵
+
+![log](log.webp)
+
+> What rolls down stairs, alone or in pairs, and over your neighbor's dog?
+> What's great for a snack, And fits on your back?
+> It's Log Log Log!
+
+I needed line wrapping in my tty with fast access, so why not make a log that
+supports egregious sizes and low seek times?
+
+## ⏩
+
+```bash
+# makes log.log.log from /var/log
+make log
+```
+
+## TODO
+
+- [ ] Textual demo
+  - [x] stats in window
+  - [ ] slim demo down
+  - [x] follow last line when at end
+- [x] Async/non-blocking design
+  - [x] Make log processing async to avoid blocking
+  - [x] Support streaming updates
+- [ ] Multiple display backends
+- [ ] Python logging integration
+  - [ ] Direct logger handlers
+  - [ ] Log level filtering and formatting
+- [ ] Cache management
+  - [ ] Periodic cleanup of cache directories
+  - [ ] Check file existence by inode lookup
+  - [ ] Handle file rotation and moves
+

logloglog-0.0.1/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "logloglog"
+description = "Efficient scrollback indexing for large log files with terminal word wrapping"
+version = "0.0.1"
+authors = [
+    { name = "Gareth Davidson", email = "gaz@bitplane.net" }
+]
+readme = "README.md"
+requires-python = ">=3.10"
+license = {text = "WTFPL"}
+keywords = ["logging", "terminal", "scrollback", "indexing"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Intended Audience :: Developers",
+    "License :: Public Domain",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+    "Topic :: System :: Logging",
+    "Topic :: Terminals",
+]
+
+dependencies = [
+    "wcwidth~=0.2.5",
+    "platformdirs~=3.0",
+    "arrayfile~=0.0.1"
+]
+
+[project.optional-dependencies]
+dev = [
+    "logloglog[textual]",
+    "pre-commit",
+    "pytest",
+    "pytest-asyncio",
+    "coverage",
+    "pytest-cov",
+    "build",
+    "twine",
+    "ruff"
+]
+textual = [
+    "textual",
+    "textual-window"
+]
+
+[project.scripts]
+stream-logs = "logloglog.tools.stream_logs:main"
+
+[build-system]
+build-backend = "flit_core.buildapi"
+requires = ["flit_core >=3.2,<4"]
+
+[tool.ruff]
+line-length = 120
+target-version = "py310"
+
+[tool.ruff.format]
+docstring-code-format = true

logloglog-0.0.1/src/logloglog/__init__.py

@@ -0,0 +1,40 @@
+"""LogLogLog - Efficient scrollback indexing for large log files."""
+
+import logging
+import sys
+from importlib.metadata import version, PackageNotFoundError
+
+
+from .logloglog import LogLogLog
+from .widthview import WidthView
+
+try:
+    __version__ = version("logloglog")
+except PackageNotFoundError:
+    # Package is not installed
+    __version__ = "0.0.0+dev"
+
+__all__ = ["LogLogLog", "WidthView", "configure_logging"]
+
+
+# Configure logging for LogLogLog
+def configure_logging(level=logging.INFO):
+    """Configure logging for LogLogLog."""
+    # Create formatter
+    formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+
+    # Create console handler
+    handler = logging.StreamHandler(sys.stderr)
+    handler.setFormatter(formatter)
+
+    # Configure logloglog loggers
+    for logger_name in ["logloglog.logloglog", "logloglog.wraptree", "logloglog.index"]:
+        logger = logging.getLogger(logger_name)
+        logger.setLevel(level)
+        # Remove existing handlers to avoid duplicates
+        logger.handlers.clear()
+        logger.addHandler(handler)
+
+
+# Auto-configure with DEBUG level for performance monitoring
+configure_logging(logging.DEBUG)

logloglog-0.0.1/src/logloglog/__main__.py

@@ -0,0 +1,8 @@
+"""
+Entry point for the package
+"""
+
+from .program import run_program
+
+if __name__ == "__main__":
+    run_program()

logloglog-0.0.1/src/logloglog/cache.py

@@ -0,0 +1,85 @@
+"""Cache management for LogLogLog."""
+
+import os
+import hashlib
+import shutil
+import tempfile
+from pathlib import Path
+import platformdirs
+
+
+# Cache constants
+CACHE_DIR = Path(platformdirs.user_cache_dir("logloglog"))
+TMP_DIR = Path(tempfile.gettempdir()) / "logloglog"
+
+
+class Cache:
+    """Manages cache directories for log files."""
+
+    def __init__(self, cache_dir: Path = None):
+        """Initialize cache manager.
+
+        Args:
+            cache_dir: Cache directory (defaults to CACHE_DIR)
+        """
+        self.cache_dir = cache_dir or CACHE_DIR
+        self.cache_dir.mkdir(parents=True, exist_ok=True)
+
+    def get_dir(self, path: Path) -> Path:
+        """Get cache directory for a log file.
+
+        Args:
+            path: Path to the log file
+
+        Returns:
+            Cache directory path
+
+        Raises:
+            OSError: If file cannot be accessed or cache directory cannot be created
+        """
+        # Get file stats for unique identification
+        stat = os.stat(path)
+        # Create hash from device and inode only (removed ctime for stability)
+        hash_input = f"{stat.st_dev}_{stat.st_ino}"
+        hash_digest = hashlib.md5(hash_input.encode()).hexdigest()[:8]
+
+        # Create cache directory name
+        name = path.name
+        cache_name = f"{name}[{hash_digest}]"
+        cache_path = self.cache_dir / cache_name
+
+        # Create directory if it doesn't exist
+        cache_path.mkdir(parents=True, exist_ok=True)
+
+        # Create symlink to original file for reference
+        symlink_path = cache_path / "file"
+        if not symlink_path.exists():
+            symlink_path.symlink_to(path.resolve())
+
+        return cache_path
+
+    def cleanup(self):
+        """Clean up cache directories for files that no longer exist."""
+        if not self.cache_dir.exists():
+            return
+
+        for cache_subdir in self.cache_dir.iterdir():
+            if not cache_subdir.is_dir():
+                continue
+
+            # Check if the symlink exists and points to a valid file
+            symlink_path = cache_subdir / "file"
+            if symlink_path.exists():
+                try:
+                    # Try to resolve the symlink
+                    target = symlink_path.resolve()
+                    if not target.exists():
+                        # Original file is gone, remove cache directory
+                        shutil.rmtree(cache_subdir)
+                except (OSError, FileNotFoundError):
+                    # Symlink is broken, remove cache directory
+                    shutil.rmtree(cache_subdir)
+            else:
+                # No symlink found, directory is orphaned
+                shutil.rmtree(cache_subdir)
+            # TODO: Add inode-based cleanup for more robust file tracking

logloglog-0.0.1/src/logloglog/line_index.py

@@ -0,0 +1,274 @@
+"""Simple line indexing with periodic summaries for efficient wrapping calculations."""
+
+import logging
+from pathlib import Path
+from typing import Tuple
+from arrayfile import Array
+
+logger = logging.getLogger(__name__)
+
+# Configuration
+MAX_WIDTH = 512  # Maximum terminal width we support
+SUMMARY_INTERVAL = 1000  # Store summary every N lines
+
+
+class LineIndex:
+    """
+    Indexes log lines with byte positions, widths, and periodic summaries.
+
+    Stores:
+    - line_positions: byte offset of each line in the log file
+    - line_widths: display width of each line
+    - summaries: every 1000 lines, cumulative display rows for each width 1-512
+    """
+
+    def __init__(self, index_path: Path):
+        """Initialize line index with given path."""
+        self.index_path = index_path
+        self._line_positions = None
+        self._line_widths = None
+        self._summaries = None
+        self._line_count = 0
+        self._current_block_width_counts = {}  # Track widths in current 1000-line block
+        self._pending_positions = []  # Batch positions for extend()
+        self._pending_widths = []  # Batch widths for extend()
+
+    def open(self, create: bool = False):
+        """Open index files."""
+        self.index_path.mkdir(parents=True, exist_ok=True)
+
+        mode = "w+b" if create else "r+b"
+
+        # Line positions (uint64 for file offsets)
+        self._line_positions = Array("Q", str(self.index_path / "positions.dat"), mode)
+
+        # Line widths (uint16, capped at 65535)
+        self._line_widths = Array("H", str(self.index_path / "widths.dat"), mode)
+
+        # Summaries (uint32 array, MAX_WIDTH entries per summary)
+        self._summaries = Array("I", str(self.index_path / "summaries.dat"), mode)
+
+        # Count existing lines
+        self._line_count = len(self._line_positions)
+
+    def close(self):
+        """Close all index files."""
+        # Flush any pending data before closing
+        self._flush_pending()
+
+        if self._line_positions:
+            self._line_positions.close()
+            self._line_positions = None
+        if self._line_widths:
+            self._line_widths.close()
+            self._line_widths = None
+        if self._summaries:
+            self._summaries.close()
+            self._summaries = None
+
+    def append_line(self, position: int, width: int):
+        """
+        Append a new line to the index.
+
+        Args:
+            position: Byte offset of line start in log file
+            width: Display width of the line
+        """
+        # Cap width at uint16 max
+        width = min(width, 65535)
+
+        # Batch in memory
+        self._pending_positions.append(position)
+        self._pending_widths.append(width)
+        self._line_count += 1
+
+        # Track width for current block
+        self._current_block_width_counts[width] = self._current_block_width_counts.get(width, 0) + 1
+
+        # Check if we need to flush and store a summary
+        if self._line_count % SUMMARY_INTERVAL == 0:
+            self._flush_pending()
+            self._store_summary()
+            self._current_block_width_counts.clear()
+
+    def _flush_pending(self):
+        """Flush pending positions and widths to disk."""
+        if self._pending_positions:
+            self._line_positions.extend(self._pending_positions)
+            self._pending_positions.clear()
+        if self._pending_widths:
+            self._line_widths.extend(self._pending_widths)
+            self._pending_widths.clear()
+
+    def _store_summary(self):
+        """Store summary using already-tracked width counts."""
+        # Calculate totals for each terminal width
+        width_totals = [0] * MAX_WIDTH
+        for line_width, count in self._current_block_width_counts.items():
+            if line_width == 0:
+                # Empty lines always take 1 row regardless of terminal width
+                for i in range(MAX_WIDTH):
+                    width_totals[i] += count
+            else:
+                # Calculate rows for each terminal width
+                # Ceiling division: (line_width + term_width - 1) // term_width
+                # This is always >= 1 when both operands are positive
+                for term_width in range(1, MAX_WIDTH + 1):
+                    rows = (line_width + term_width - 1) // term_width
+                    width_totals[term_width - 1] += rows * count
+
+        # Store all width totals in summary array (batch append for performance)
+        self._summaries.extend(width_totals)
+
+    def get_line_position(self, line_no: int) -> int:
+        """Get byte position of a line."""
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        # Check if it's in the flushed data or pending batch
+        flushed_count = len(self._line_positions)
+        if line_no < flushed_count:
+            return self._line_positions[line_no]
+        else:
+            # It's in the pending batch
+            pending_idx = line_no - flushed_count
+            return self._pending_positions[pending_idx]
+
+    def get_line_width(self, line_no: int) -> int:
+        """Get display width of a line."""
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        # Check if it's in the flushed data or pending batch
+        flushed_count = len(self._line_widths)
+        if line_no < flushed_count:
+            return self._line_widths[line_no]
+        else:
+            # It's in the pending batch
+            pending_idx = line_no - flushed_count
+            return self._pending_widths[pending_idx]
+
+    def get_total_display_rows(self, width: int) -> int:
+        """
+        Get total display rows for all lines at given terminal width.
+
+        Args:
+            width: Terminal width
+
+        Returns:
+            Total number of display rows
+        """
+        if width <= 0:
+            return 0  # No display possible with zero or negative width
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        total_rows = 0
+
+        # Add up complete summaries
+        complete_summaries = self._line_count // SUMMARY_INTERVAL
+        for i in range(complete_summaries):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            total_rows += self._summaries[summary_offset]
+
+        # Add remaining lines not in a summary
+        start_line = complete_summaries * SUMMARY_INTERVAL
+        for line_idx in range(start_line, self._line_count):
+            line_width = self.get_line_width(line_idx)
+            # Ceiling division is always >= 1 for positive operands
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+            total_rows += rows
+
+        return total_rows
+
+    def get_display_row_for_line(self, line_no: int, width: int) -> int:
+        """
+        Get the display row number where a logical line starts.
+
+        Args:
+            line_no: Logical line number
+            width: Terminal width
+
+        Returns:
+            Display row number where this line starts
+        """
+        if line_no < 0 or line_no >= self._line_count:
+            raise IndexError(f"Line {line_no} out of range")
+
+        if width <= 0:
+            return 0  # No display possible
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        display_row = 0
+
+        # Add complete summaries before this line
+        summary_idx = line_no // SUMMARY_INTERVAL
+        for i in range(summary_idx):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            display_row += self._summaries[summary_offset]
+
+        # Add individual lines from last summary to target line
+        start_line = summary_idx * SUMMARY_INTERVAL
+        for line_idx in range(start_line, line_no):
+            line_width = self.get_line_width(line_idx)
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+            display_row += rows
+
+        return display_row
+
+    def get_line_for_display_row(self, display_row: int, width: int) -> Tuple[int, int]:
+        """
+        Find the logical line containing the given display row.
+
+        Args:
+            display_row: Display row to find
+            width: Terminal width
+
+        Returns:
+            Tuple of (line_number, row_offset_within_line)
+        """
+        if width <= 0:
+            raise IndexError(f"Display row {display_row} out of range")  # No display possible
+        if width > MAX_WIDTH:
+            width = MAX_WIDTH
+
+        current_row = 0
+
+        # Binary search through summaries to find the right range
+        complete_summaries = self._line_count // SUMMARY_INTERVAL
+        summary_idx = 0
+
+        # Find which summary block contains our display row
+        for i in range(complete_summaries):
+            summary_offset = i * MAX_WIDTH + (width - 1)
+            summary_rows = self._summaries[summary_offset]
+            if current_row + summary_rows > display_row:
+                summary_idx = i
+                break
+            current_row += summary_rows
+        else:
+            # It's in the incomplete last block
+            summary_idx = complete_summaries
+
+        # Linear search within the summary block
+        start_line = summary_idx * SUMMARY_INTERVAL
+        end_line = min(start_line + SUMMARY_INTERVAL, self._line_count)
+
+        for line_idx in range(start_line, end_line):
+            line_width = self.get_line_width(line_idx)
+            rows = (line_width + width - 1) // width if width > 0 and line_width > 0 else 1
+
+            if current_row + rows > display_row:
+                # Found the line
+                offset_within_line = display_row - current_row
+                return (line_idx, offset_within_line)
+
+            current_row += rows
+
+        # Display row is beyond the end
+        raise IndexError(f"Display row {display_row} out of range")
+
+    def __len__(self) -> int:
+        """Get total number of indexed lines."""
+        return self._line_count