logtap 0.3.0__py3-none-any.whl → 0.4.0__py3-none-any.whl

logtap/core/runs.py

@@ -0,0 +1,393 @@
+"""Run store for ingested log streams.
+
+Provides append-only storage with in-memory tail cache and cursor management.
+"""
+
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Dict, Iterator, List, Optional
+
+TAG_KEY_PATTERN = r"^[a-zA-Z0-9_.-]+$"
+TAG_VALUE_MAX_LEN = 256
+
+
+@dataclass
+class RunLine:
+    """A single log line with cursor and timestamp."""
+
+    cursor: int
+    line: str
+    ts: datetime
+
+
+@dataclass
+class RunMetadata:
+    """Metadata for a run."""
+
+    id: str
+    tags: Dict[str, str] = field(default_factory=dict)
+    created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    last_activity: datetime = field(default_factory=lambda: datetime.now(timezone.utc))
+    cursor_start: int = 0  # First cursor ever (immutable)
+    cursor_latest: int = -1  # Latest cursor written
+    lines_count: int = 0
+    bytes_on_disk: int = 0
+    active: bool = True
+
+
+class Run:
+    """A single run with append-only file storage and in-memory tail cache."""
+
+    def __init__(self, run_id: str, data_dir: Path, buffer_lines: int = 100_000):
+        self.id = run_id
+        self.data_dir = data_dir
+        self.buffer_lines = buffer_lines
+        self._lock = threading.RLock()
+
+        # In-memory tail cache (deque for O(1) append and popleft)
+        self._cache: deque[RunLine] = deque(maxlen=buffer_lines)
+        self._cache_start_cursor: int = 0  # Cursor of first item in cache
+
+        # Run directory and files
+        self.run_dir = data_dir / run_id
+        self.log_file = self.run_dir / "log.txt"
+        self.meta_file = self.run_dir / "meta.json"
+
+        # Metadata
+        self.metadata: RunMetadata
+
+        # Initialize or load
+        if self.run_dir.exists():
+            self._load()
+        else:
+            self._create()
+
+    def _create(self) -> None:
+        """Create a new run."""
+        self.run_dir.mkdir(parents=True, exist_ok=True)
+        self.metadata = RunMetadata(id=self.id)
+        self._save_metadata()
+
+    def _load(self) -> None:
+        """Load existing run from disk."""
+        import json
+
+        # Load metadata
+        if self.meta_file.exists():
+            with open(self.meta_file, "r", encoding="utf-8") as f:
+                data = json.load(f)
+                self.metadata = RunMetadata(
+                    id=data["id"],
+                    tags=data.get("tags", {}),
+                    created_at=datetime.fromisoformat(data["created_at"]),
+                    last_activity=datetime.fromisoformat(data["last_activity"]),
+                    cursor_start=data.get("cursor_start", 0),
+                    cursor_latest=data.get("cursor_latest", -1),
+                    lines_count=data.get("lines_count", 0),
+                    bytes_on_disk=data.get("bytes_on_disk", 0),
+                    active=data.get("active", True),
+                )
+        else:
+            self.metadata = RunMetadata(id=self.id)
+
+        # Populate cache from end of log file
+        if self.log_file.exists():
+            self._populate_cache_from_disk()
+
+    def _populate_cache_from_disk(self) -> None:
+        """Load last N lines from disk into cache."""
+        if not self.log_file.exists():
+            return
+
+        lines: List[str] = []
+        with open(self.log_file, "r", encoding="utf-8", errors="replace") as f:
+            # Read all lines (for small files) or tail
+            for line in f:
+                lines.append(line.rstrip("\n"))
+
+        # Only keep last buffer_lines
+        if len(lines) > self.buffer_lines:
+            lines = lines[-self.buffer_lines :]
+            start_cursor = self.metadata.cursor_latest - len(lines) + 1
+        else:
+            start_cursor = 0
+
+        self._cache_start_cursor = start_cursor
+        self._cache.clear()
+        for i, line in enumerate(lines):
+            self._cache.append(
+                RunLine(
+                    cursor=start_cursor + i,
+                    line=line,
+                    ts=self.metadata.last_activity,  # Approximate
+                )
+            )
+
+    def _save_metadata(self) -> None:
+        """Save metadata to disk."""
+        import json
+
+        with open(self.meta_file, "w", encoding="utf-8") as f:
+            json.dump(
+                {
+                    "id": self.metadata.id,
+                    "tags": self.metadata.tags,
+                    "created_at": self.metadata.created_at.isoformat(),
+                    "last_activity": self.metadata.last_activity.isoformat(),
+                    "cursor_start": self.metadata.cursor_start,
+                    "cursor_latest": self.metadata.cursor_latest,
+                    "lines_count": self.metadata.lines_count,
+                    "bytes_on_disk": self.metadata.bytes_on_disk,
+                    "active": self.metadata.active,
+                },
+                f,
+            )
+
+    def append(self, line: str) -> RunLine:
+        """Append a line to the run. Returns the line with assigned cursor."""
+        with self._lock:
+            now = datetime.now(timezone.utc)
+            cursor = self.metadata.cursor_latest + 1
+
+            run_line = RunLine(cursor=cursor, line=line, ts=now)
+
+            # Append to disk
+            with open(self.log_file, "a", encoding="utf-8") as f:
+                written = f.write(line + "\n")
+                self.metadata.bytes_on_disk += written
+
+            # Update cache
+            if len(self._cache) >= self.buffer_lines:
+                self._cache_start_cursor += 1
+            self._cache.append(run_line)
+
+            # Update metadata
+            self.metadata.cursor_latest = cursor
+            self.metadata.lines_count += 1
+            self.metadata.last_activity = now
+
+            return run_line
+
+    def append_batch(self, lines: List[str]) -> List[RunLine]:
+        """Append multiple lines atomically."""
+        with self._lock:
+            result = []
+            for line in lines:
+                result.append(self.append(line))
+            self._save_metadata()
+            return result
+
+    def set_tags(self, tags: Dict[str, str]) -> Optional[str]:
+        """Set tags, merging with existing. Returns error message on conflict, None on success."""
+        import re
+
+        with self._lock:
+            for key, value in tags.items():
+                # Validate key
+                if not re.match(TAG_KEY_PATTERN, key):
+                    return f"Invalid tag key: {key}"
+                # Validate value length
+                if len(value) > TAG_VALUE_MAX_LEN:
+                    return f"Tag value too long: {key}"
+                # Check for conflict
+                if key in self.metadata.tags and self.metadata.tags[key] != value:
+                    existing = self.metadata.tags[key]
+                    return f"Tag conflict for key '{key}': existing='{existing}', new='{value}'"
+
+            # Merge tags
+            self.metadata.tags.update(tags)
+            self._save_metadata()
+            return None
+
+    @property
+    def cursor_earliest(self) -> int:
+        """Earliest cursor available in cache/disk."""
+        with self._lock:
+            if self._cache:
+                return self._cache[0].cursor
+            return 0
+
+    @property
+    def cursor_latest(self) -> int:
+        """Latest cursor written."""
+        with self._lock:
+            return self.metadata.cursor_latest
+
+    def get_lines(
+        self,
+        since: Optional[int] = None,
+        tail: int = 50,
+        limit: int = 1000,
+    ) -> tuple[List[RunLine], bool]:
+        """
+        Get lines from run.
+
+        Args:
+            since: Cursor to start from (exclusive). If None, returns last `tail` lines.
+            tail: Number of recent lines if since is None.
+            limit: Maximum lines to return.
+
+        Returns:
+            Tuple of (lines, gap_detected).
+            gap_detected is True if since < cursor_earliest.
+        """
+        with self._lock:
+            gap = False
+
+            if since is not None:
+                # Resume from cursor
+                if since < self.cursor_earliest:
+                    gap = True
+                    # Start from earliest available
+                    start_cursor = self.cursor_earliest
+                else:
+                    start_cursor = since + 1  # Exclusive
+
+                # Get lines from cache
+                lines = [ln for ln in self._cache if ln.cursor >= start_cursor]
+            else:
+                # Tail mode - get last N lines
+                lines = list(self._cache)[-tail:]
+
+            # Apply limit
+            if len(lines) > limit:
+                lines = lines[:limit]
+
+            return lines, gap
+
+    def tail_iter(self, since: Optional[int] = None) -> Iterator[RunLine]:
+        """
+        Iterator that yields new lines as they arrive.
+
+        Args:
+            since: Cursor to start from (exclusive). If None, starts from latest.
+        """
+        last_cursor = since if since is not None else self.cursor_latest
+
+        while True:
+            with self._lock:
+                new_lines = [ln for ln in self._cache if ln.cursor > last_cursor]
+
+            for line in new_lines:
+                last_cursor = line.cursor
+                yield line
+
+            if not new_lines:
+                time.sleep(0.1)  # Poll interval
+
+    def close(self) -> None:
+        """Mark run as inactive and save metadata."""
+        with self._lock:
+            self.metadata.active = False
+            self._save_metadata()
+
+
+class RunStore:
+    """Manages all runs with disk persistence."""
+
+    def __init__(
+        self,
+        data_dir: Path,
+        buffer_lines: int = 100_000,
+        max_disk_mb: int = 1000,
+        retention_hours: int = 72,
+    ):
+        self.data_dir = Path(data_dir).expanduser()
+        self.buffer_lines = buffer_lines
+        self.max_disk_bytes = max_disk_mb * 1024 * 1024
+        self.retention_seconds = retention_hours * 3600
+        self._runs: Dict[str, Run] = {}
+        self._lock = threading.RLock()
+
+        # Create data directory
+        self.data_dir.mkdir(parents=True, exist_ok=True)
+
+        # Load existing runs
+        self._load_existing_runs()
+
+    def _load_existing_runs(self) -> None:
+        """Load existing runs from disk."""
+        if not self.data_dir.exists():
+            return
+
+        for run_dir in self.data_dir.iterdir():
+            if run_dir.is_dir() and (run_dir / "meta.json").exists():
+                try:
+                    run = Run(run_dir.name, self.data_dir, self.buffer_lines)
+                    self._runs[run_dir.name] = run
+                except Exception:
+                    pass  # Skip corrupted runs
+
+    def get_or_create(self, run_id: str) -> tuple[Run, bool]:
+        """Get existing run or create new one. Returns (run, created)."""
+        with self._lock:
+            if run_id in self._runs:
+                return self._runs[run_id], False
+
+            run = Run(run_id, self.data_dir, self.buffer_lines)
+            self._runs[run_id] = run
+            return run, True
+
+    def get(self, run_id: str) -> Optional[Run]:
+        """Get run by ID, or None if not found."""
+        with self._lock:
+            return self._runs.get(run_id)
+
+    def list_runs(self, since_hours: Optional[int] = None) -> List[Run]:
+        """List all runs, optionally filtered by recent activity."""
+        with self._lock:
+            runs = list(self._runs.values())
+
+            if since_hours is not None:
+                cutoff = datetime.now(timezone.utc).timestamp() - (since_hours * 3600)
+                runs = [r for r in runs if r.metadata.last_activity.timestamp() >= cutoff]
+
+            # Sort by last activity (most recent first)
+            runs.sort(key=lambda r: r.metadata.last_activity, reverse=True)
+            return runs
+
+    def total_disk_usage(self) -> int:
+        """Get total disk usage across all runs in bytes."""
+        with self._lock:
+            return sum(r.metadata.bytes_on_disk for r in self._runs.values())
+
+    def enforce_retention(self) -> None:
+        """Remove runs older than retention period."""
+        with self._lock:
+            cutoff = datetime.now(timezone.utc).timestamp() - self.retention_seconds
+            to_remove = [
+                run_id
+                for run_id, run in self._runs.items()
+                if run.metadata.last_activity.timestamp() < cutoff
+            ]
+
+            for run_id in to_remove:
+                self._delete_run(run_id)
+
+    def enforce_disk_limit(self) -> None:
+        """Remove oldest runs if disk limit exceeded."""
+        with self._lock:
+            while self.total_disk_usage() > self.max_disk_bytes and self._runs:
+                # Find oldest run
+                oldest = min(self._runs.values(), key=lambda r: r.metadata.last_activity)
+                self._delete_run(oldest.id)
+
+    def _delete_run(self, run_id: str) -> None:
+        """Delete a run from disk and memory."""
+        import shutil
+
+        if run_id in self._runs:
+            run = self._runs.pop(run_id)
+            run_dir = run.run_dir
+            if run_dir.exists():
+                shutil.rmtree(run_dir)
+
+    def check_storage(self) -> Optional[str]:
+        """Check if storage is available. Returns error message if not."""
+        if self.total_disk_usage() >= self.max_disk_bytes:
+            return "insufficient_storage"
+        return None

logtap/models/responses.py

@@ -1,6 +1,7 @@
 """Response models for logtap API."""
 
-from typing import List, Optional
+from datetime import datetime
+from typing import Dict, List, Optional
 
 from pydantic import BaseModel, Field
 
@@ -63,3 +64,55 @@ class HealthResponse(BaseModel):
 
     status: str = Field(default="healthy", description="Service status")
     version: str = Field(description="logtap version")
+    mode: Optional[str] = Field(default=None, description="Server mode: serve, collect, or both")
+    features: Optional[List[str]] = Field(default=None, description="Available features")
+    runs: Optional[int] = Field(default=None, description="Number of active runs (collect mode)")
+    uptime_seconds: Optional[int] = Field(default=None, description="Server uptime in seconds")
+
+
+# Run-related models for collector mode
+
+
+class RunInfo(BaseModel):
+    """Information about a single run."""
+
+    id: str = Field(description="Run identifier")
+    lines: int = Field(description="Total lines ingested")
+    cursor_earliest: int = Field(description="Earliest available cursor")
+    cursor_latest: int = Field(description="Latest cursor")
+    tags: Dict[str, str] = Field(default_factory=dict, description="Run tags")
+    created_at: datetime = Field(description="When the run was created")
+    last_activity: datetime = Field(description="Last activity timestamp")
+    active: bool = Field(description="Whether the run is actively receiving data")
+    bytes_on_disk: Optional[int] = Field(default=None, description="Disk usage in bytes")
+
+
+class RunListResponse(BaseModel):
+    """Response for listing runs."""
+
+    runs: List[RunInfo] = Field(description="List of runs")
+
+
+class IngestResponse(BaseModel):
+    """Response after ingest completes."""
+
+    run_id: str = Field(description="Run identifier")
+    lines_ingested: int = Field(description="Number of lines ingested in this request")
+    cursor_end: int = Field(description="Final cursor after ingest")
+
+
+class StreamMetaEvent(BaseModel):
+    """Meta event sent at start of stream."""
+
+    cursor_earliest: int = Field(description="Earliest available cursor")
+    cursor_latest: int = Field(description="Latest cursor")
+    gap: bool = Field(default=False, description="Whether a gap was detected")
+    missed: Optional[int] = Field(default=None, description="Number of missed lines if gap")
+
+
+class StreamLineEvent(BaseModel):
+    """Line event in stream."""
+
+    cursor: int = Field(description="Line cursor")
+    line: str = Field(description="Log line content")
+    ts: datetime = Field(description="Timestamp when line was ingested")

{logtap-0.3.0.dist-info → logtap-0.4.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: logtap
-Version: 0.3.0
+Version: 0.4.0
 Summary: A CLI-first log access tool for Unix systems. Remote log file access without SSH.
 Project-URL: Homepage, https://github.com/cainky/logtap
 Project-URL: Repository, https://github.com/cainky/logtap

{logtap-0.3.0.dist-info → logtap-0.4.0.dist-info}/RECORD

@@ -1,22 +1,27 @@
-logtap/__init__.py,sha256=QGputf7HjtjMkCnDHsPNBBqZ27DYZ-7a2_biF5Rvk6c,179
+logtap/__init__.py,sha256=kPYm2mOUmJEBnDtZ78K-fva7PcnK1J7yjQToY6bEqyc,179
 logtap/__main__.py,sha256=vqJPz3Zf-ICn_4P3B1o7U4NxcCo0qWgfAGEE_j13t-c,138
 logtap/api/__init__.py,sha256=80bP-eIxtAzidgv5nzcfyCEdl8EI2QjVY_eyxjvvhA0,98
-logtap/api/app.py,sha256=BBphxLKvk7yIiLyQ3tdiagqtjc7xFixd_FUzNFjlC0c,1176
+logtap/api/app.py,sha256=6TIwQFjwyW0VYYo3ayRiUO2IuHnPKsK5kAIPO9xzPvo,3094
 logtap/api/dependencies.py,sha256=1cx1qrp0O6v1fHXA2JdEhC8P4caG2oUSCfMk2-8zmGs,1620
 logtap/api/routes/__init__.py,sha256=XYvFyTP4zKywRZH0v97k0EZCYgxdL2PSUaNet20znPE,29
 logtap/api/routes/files.py,sha256=bqZYrX6jrF5-7GzBpUIXXoPVdxUwm6o0LTcJBLtaJUE,991
-logtap/api/routes/health.py,sha256=Ak-z2ChqZZ7FgHdu1JDo3v5aDBPR3VIICyXTLDBf75E,462
+logtap/api/routes/health.py,sha256=s117Hr1E8OcBGPOWq2WwHLZSq35hS7wmLPk6BYq3dq4,1112
 logtap/api/routes/logs.py,sha256=XpRAd4fZmVyylz6bHCHm4y0Y2GofSquH6j5WJP3Jyao,8467
 logtap/api/routes/parsed.py,sha256=XVvkKBE_hQvfJyrDBBPR_PpVxvof-y4B77xKe9Rr0Qk,3367
+logtap/api/routes/runs.py,sha256=Fxb6joJ5FPXPCKTfgD41i0H4UQ4U4fmFxk08SFUxt_s,11355
 logtap/cli/__init__.py,sha256=U4zaUJ1rm0qHXqeArpzC45S5N-5SBdd8K6foe513msk,31
-logtap/cli/main.py,sha256=fWSuQdin9G-RC7Oqzesfp93WZI1-v7227P-WWTsxtIQ,1045
+logtap/cli/main.py,sha256=jfzN-S6dn3bg6yuQ3ovJtaLYb7LnCDg_cl7vqRWTBxw,1230
 logtap/cli/commands/__init__.py,sha256=U4zaUJ1rm0qHXqeArpzC45S5N-5SBdd8K6foe513msk,31
+logtap/cli/commands/collect.py,sha256=8x6LyMrzI79wYtfLZcbQdgpy5nxPZuQOEillE9IfwwE,3002
 logtap/cli/commands/files.py,sha256=WFr8kA0SdgQHz3ZyONTaljxHMcD-nQlndp3UIOwZATc,2455
+logtap/cli/commands/ingest.py,sha256=JaItHHYV3fBmPkseYpubyHryNbuEuxyjRBk-EiiEwyU,4054
 logtap/cli/commands/query.py,sha256=uD9nH5E-7EqJryLf3hHkDbJSQo4kWFGmzzHgTfAKFwk,3418
+logtap/cli/commands/runs.py,sha256=Dweswku19Dj2KOFhT0kaega9KSKmUrvya3eLn0-5lXo,3632
 logtap/cli/commands/serve.py,sha256=9OvfII21q6cel3zZfSsAsiERKwKFt0ZFTXmUd2Psthg,1910
-logtap/cli/commands/tail.py,sha256=dwPRXub1dcRwKulDt_qNa2waQm1YPOxIg0QokAK6Gyw,3648
+logtap/cli/commands/tail.py,sha256=w7P3_1o0OtVtos3kV8w4goShWXzzUDo4ekSye3VSpGo,10015
 logtap/core/__init__.py,sha256=tsoL0XuDrPd5xHEu975WqFHoA7EQgloxrum7CjsWHuk,450
 logtap/core/reader.py,sha256=BuBrEAbS2naCBTtuBNc0Un6thbekzabaHTBzYE1SwKg,5277
+logtap/core/runs.py,sha256=t4JnQvZTi-YB2II8maBIcaJD77gp_CjKVcTGYwHhuU8,13488
 logtap/core/search.py,sha256=rtq8WP96RYUvRkX_R5x_mdD_dw1syDuNkHx3uP_diOg,4574
 logtap/core/validation.py,sha256=Nk86jHqEfI4H96fk-1rjbC5sBwfzls43hyOhnRV6rxI,1359
 logtap/core/parsers/__init__.py,sha256=5f3hFxf_DgNScRDchRT8ocFVgi7Md4xuMN-ShvlssBo,575
@@ -28,9 +33,9 @@ logtap/core/parsers/nginx.py,sha256=j_oILELOM0azDPLc41wXrLu5o_LhnPs9fT0_iaOqqAQ,
 logtap/core/parsers/syslog.py,sha256=gBNQ39QXsigOpfnq3cEdmvFa8NLp_wmiSMDlTt0SIbs,2430
 logtap/models/__init__.py,sha256=tce3Q0QjPhnlAYG8IcwxPedyh1ibBlKIF3CjXe5wwgo,280
 logtap/models/config.py,sha256=8x6OR_y2ZB8SSoQWQGwDB7DXH30UyMNXUcRWOctjUn8,927
-logtap/models/responses.py,sha256=45J-Xw1Gb35uP5188wxy2QZlyy3Fh18fpAFizZnpi3A,1850
-logtap-0.3.0.dist-info/METADATA,sha256=O1zoWpiUNGvVxjSLfAGPfZKnxgy4CFZV1p3_olMe7rM,7466
-logtap-0.3.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-logtap-0.3.0.dist-info/entry_points.txt,sha256=tuAit8kt97yjtACQKvN35wWozp4KhSju_gfDhSS1IrM,47
-logtap-0.3.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
-logtap-0.3.0.dist-info/RECORD,,
+logtap/models/responses.py,sha256=xKdKdS85soxMYGNad3WfF0pOG0Pb5Z7XwVrwK-TCnHs,4084
+logtap-0.4.0.dist-info/METADATA,sha256=_Y9ZSz2BwIF2SOOHYPQZ8YrLIxmpB31tOaabDty1BDY,7466
+logtap-0.4.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+logtap-0.4.0.dist-info/entry_points.txt,sha256=tuAit8kt97yjtACQKvN35wWozp4KhSju_gfDhSS1IrM,47
+logtap-0.4.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+logtap-0.4.0.dist-info/RECORD,,