logtap 0.2.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

logtap/cli/commands/ingest.py

@@ -0,0 +1,123 @@
+"""Ingest command for logtap CLI."""
+
+import sys
+from datetime import datetime
+from typing import List, Optional
+
+import typer
+from rich.console import Console
+
+console = Console(stderr=True)  # Output status to stderr so it doesn't mix with piped data
+
+
+def ingest(
+    name: Optional[str] = typer.Argument(
+        None,
+        help="Run name. If omitted, generates: run-YYYYMMDD-HHMMSS",
+    ),
+    server: str = typer.Option(
+        "http://localhost:8000",
+        "--server",
+        "-s",
+        help="Collector server URL.",
+        envvar="LOGTAP_SERVER",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    tag: Optional[List[str]] = typer.Option(
+        None,
+        "--tag",
+        "-t",
+        help="Tags as key=value (repeatable).",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "--quiet",
+        "-q",
+        help="Suppress status messages.",
+    ),
+) -> None:
+    """
+    Pipe stdin to collector as a named run.
+
+    Reads from stdin and streams lines to the collector server.
+    Designed to be used with pipes for training logs.
+
+    Example:
+        python train.py 2>&1 | logtap ingest run1
+        python train.py 2>&1 | logtap ingest --tag node=gpu1
+        cat training.log | logtap ingest my-run --server http://collector:8000
+    """
+    import httpx
+
+    # Generate run name if not provided
+    if name is None:
+        name = datetime.now().strftime("run-%Y%m%d-%H%M%S")
+
+    # Build headers as list of tuples (supports duplicate keys)
+    headers: List[tuple] = [("Content-Type", "text/plain")]
+    if api_key:
+        headers.append(("X-API-Key", api_key))
+    if tag:
+        for t in tag:
+            headers.append(("X-Logtap-Tag", t))
+
+    url = f"{server.rstrip('/')}/runs/{name}/ingest"
+
+    if not quiet:
+        console.print(f"[dim]Ingesting to[/dim] {name} [dim]@[/dim] {server}")
+
+    try:
+        # Read all stdin
+        content = sys.stdin.read()
+
+        if not quiet:
+            line_count = content.count("\n") + (1 if content and not content.endswith("\n") else 0)
+            console.print(f"[dim]Sending {line_count} lines...[/dim]")
+
+        # Use httpx
+        with httpx.Client(timeout=None) as client:
+            response = client.post(url, content=content, headers=headers)
+
+            if response.status_code == 401:
+                console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code == 409:
+                try:
+                    data = response.json()
+                    msg = data.get("detail", {}).get("message", "Tag conflict")
+                except Exception:
+                    msg = "Tag conflict"
+                console.print(f"[red]Error: {msg}[/red]")
+                raise typer.Exit(1)
+            elif response.status_code == 507:
+                console.print("[red]Error: Server storage limit exceeded.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code >= 400:
+                console.print(f"[red]Error: Server returned {response.status_code}[/red]")
+                try:
+                    console.print(f"[dim]{response.text}[/dim]")
+                except Exception:
+                    pass
+                raise typer.Exit(1)
+
+            data = response.json()
+            if not quiet:
+                console.print(
+                    f"[green]Done.[/green] "
+                    f"Ingested {data['lines_ingested']} lines to [bold]{data['run_id']}[/bold] "
+                    f"(cursor: {data['cursor_end']})"
+                )
+
+    except httpx.ConnectError:
+        console.print(f"[red]Error: Could not connect to {server}[/red]")
+        console.print("[dim]Is the collector running? Start with: logtap collect[/dim]")
+        raise typer.Exit(1)
+    except httpx.TimeoutException:
+        console.print("[red]Error: Connection timed out[/red]")
+        raise typer.Exit(1)

logtap/cli/commands/runs.py

@@ -0,0 +1,116 @@
+"""Runs command for logtap CLI."""
+
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def runs(
+    server: str = typer.Option(
+        "http://localhost:8000",
+        "--server",
+        "-s",
+        help="Collector server URL.",
+        envvar="LOGTAP_SERVER",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    since_hours: Optional[int] = typer.Option(
+        24,
+        "--since-hours",
+        help="Show runs active within N hours.",
+    ),
+) -> None:
+    """
+    List runs on a collector.
+
+    Shows all active and recent runs with their line counts and tags.
+
+    Example:
+        logtap runs
+        logtap runs --server http://gpu-box:8000
+        logtap runs --since-hours 48
+    """
+    from datetime import datetime, timezone
+
+    import httpx
+
+    # Build headers
+    headers = {}
+    if api_key:
+        headers["X-API-Key"] = api_key
+
+    url = f"{server.rstrip('/')}/runs"
+    params = {}
+    if since_hours is not None:
+        params["since_hours"] = since_hours
+
+    try:
+        with httpx.Client(timeout=30) as client:
+            response = client.get(url, headers=headers, params=params)
+
+            if response.status_code == 401:
+                console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code >= 400:
+                console.print(f"[red]Error: Server returned {response.status_code}[/red]")
+                raise typer.Exit(1)
+
+            data = response.json()
+            runs_list = data.get("runs", [])
+
+            if not runs_list:
+                console.print("[dim]No runs found.[/dim]")
+                return
+
+            # Create table
+            table = Table(show_header=True, header_style="bold")
+            table.add_column("RUN", style="cyan")
+            table.add_column("LINES", justify="right")
+            table.add_column("LAST ACTIVITY", style="dim")
+            table.add_column("TAGS")
+
+            now = datetime.now(timezone.utc)
+
+            for run in runs_list:
+                # Format last activity as relative time
+                last_activity = datetime.fromisoformat(run["last_activity"].replace("Z", "+00:00"))
+                delta = now - last_activity
+                if delta.total_seconds() < 60:
+                    time_str = f"{int(delta.total_seconds())}s ago"
+                elif delta.total_seconds() < 3600:
+                    time_str = f"{int(delta.total_seconds() / 60)}m ago"
+                elif delta.total_seconds() < 86400:
+                    time_str = f"{int(delta.total_seconds() / 3600)}h ago"
+                else:
+                    time_str = f"{int(delta.total_seconds() / 86400)}d ago"
+
+                # Format tags
+                tags = run.get("tags", {})
+                tags_str = ", ".join(f"{k}={v}" for k, v in tags.items()) if tags else ""
+
+                table.add_row(
+                    run["id"],
+                    f"{run['lines']:,}",
+                    time_str,
+                    tags_str,
+                )
+
+            console.print(table)
+
+    except httpx.ConnectError:
+        console.print(f"[red]Error: Could not connect to {server}[/red]")
+        console.print("[dim]Is the collector running? Start with: logtap collect[/dim]")
+        raise typer.Exit(1)
+    except httpx.TimeoutException:
+        console.print("[red]Error: Connection timed out[/red]")
+        raise typer.Exit(1)

logtap/cli/commands/tail.py

@@ -1,6 +1,7 @@
 """Tail command for logtap CLI - real-time log streaming."""
 
-from typing import Optional
+import json
+from typing import List, Optional
 
 import typer
 from rich.console import Console
@@ -9,12 +10,12 @@ console = Console()
 
 
 def tail(
-    filename: str = typer.Argument(
-        "syslog",
-        help="Name of the log file to tail.",
+    target: str = typer.Argument(
+        ...,
+        help="Run name or file path to tail.",
     ),
-    server: str = typer.Option(
-        "http://localhost:8000",
+    server: Optional[str] = typer.Option(
+        None,
         "--server",
         "-s",
         help="URL of the logtap server.",
@@ -24,10 +25,10 @@ def tail(
         False,
         "--follow",
         "-f",
-        help="Follow log output (like tail -f). Requires WebSocket support.",
+        help="Follow log output (like tail -f).",
     ),
     lines: int = typer.Option(
-        10,
+        50,
         "--lines",
         "-n",
         help="Number of lines to show initially.",
@@ -39,22 +40,198 @@ def tail(
         help="API key for authentication.",
         envvar="LOGTAP_API_KEY",
     ),
+    mode: str = typer.Option(
+        "auto",
+        "--mode",
+        "-m",
+        help="Resolution mode: auto, runs, or files.",
+    ),
+    since: Optional[int] = typer.Option(
+        None,
+        "--since",
+        help="Resume from cursor (exclusive). For runs mode only.",
+    ),
+    tag: Optional[List[str]] = typer.Option(
+        None,
+        "--tag",
+        "-t",
+        help="Filter by tag (key=value, repeatable). For runs mode only.",
+    ),
+    output: str = typer.Option(
+        "pretty",
+        "--output",
+        "-o",
+        help="Output format: pretty, plain, or jsonl.",
+    ),
 ) -> None:
     """
-    Tail a log file, optionally following new entries.
+    Tail a run or file, optionally following new entries.
+
+    For ML training logs (runs mode):
+        logtap tail run1 -f
+        logtap tail run1 --since 5000 -f
+        logtap tail run1 --tag node=gpu1
 
-    Example:
-        logtap tail syslog
+    For static files (legacy mode):
+        logtap tail syslog --server http://mybox:8000
         logtap tail auth.log -f
-        logtap tail syslog --lines 100
     """
-    import httpx
 
-    # First, get initial lines
+    # If no server specified and target looks like a path, use local file mode
+    if server is None:
+        if "/" in target or "\\" in target or target.startswith("."):
+            console.print("[red]Error: Local file tailing requires --server[/red]")
+            console.print("[dim]For runs: logtap tail run1 --server http://collector:8000[/dim]")
+            console.print(
+                "[dim]For files: logtap tail syslog --server http://fileserver:8000[/dim]"
+            )
+            raise typer.Exit(1)
+        # Default to localhost
+        server = "http://localhost:8000"
+
     headers = {}
     if api_key:
         headers["X-API-Key"] = api_key
 
+    # Determine mode
+    use_runs_mode = False
+    if mode == "runs":
+        use_runs_mode = True
+    elif mode == "files":
+        use_runs_mode = False
+    else:
+        # Auto mode: check server capabilities and run existence
+        use_runs_mode = _detect_runs_mode(server, target, headers)
+
+    if use_runs_mode:
+        _tail_run(server, target, headers, follow, lines, since, tag, output)
+    else:
+        _tail_file(server, target, headers, follow, lines, output)
+
+
+def _detect_runs_mode(server: str, target: str, headers: dict) -> bool:
+    """Detect if server supports runs and target exists as a run."""
+    import httpx
+
+    try:
+        with httpx.Client(timeout=5) as client:
+            # Check health endpoint for capabilities
+            response = client.get(f"{server}/health", headers=headers)
+            if response.status_code == 200:
+                data = response.json()
+                features = data.get("features", [])
+                if "runs" in features:
+                    # Check if run exists
+                    run_response = client.get(f"{server}/runs/{target}", headers=headers)
+                    if run_response.status_code == 200:
+                        return True
+    except Exception:
+        pass
+
+    return False
+
+
+def _tail_run(
+    server: str,
+    run_id: str,
+    headers: dict,
+    follow: bool,
+    lines: int,
+    since: Optional[int],
+    tags: Optional[List[str]],
+    output: str,
+) -> None:
+    """Tail a run using SSE."""
+    import httpx
+
+    # Build URL with query params
+    params = {"tail": lines, "follow": str(follow).lower()}
+    if since is not None:
+        params["since"] = since
+    if tags:
+        params["tag"] = tags
+
+    url = f"{server}/runs/{run_id}/stream"
+
+    try:
+        with httpx.Client(timeout=None) as client:
+            with client.stream("GET", url, headers=headers, params=params) as response:
+                if response.status_code == 401:
+                    console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+                    raise typer.Exit(1)
+                elif response.status_code == 404:
+                    console.print(f"[red]Error: Run '{run_id}' not found.[/red]")
+                    raise typer.Exit(1)
+                elif response.status_code >= 400:
+                    console.print(f"[red]Error: Server returned {response.status_code}[/red]")
+                    raise typer.Exit(1)
+
+                # Track cursor for gap detection message
+                last_cursor = since
+
+                # Parse SSE stream
+                buffer = ""
+                for chunk in response.iter_text():
+                    buffer += chunk
+                    while "\n\n" in buffer:
+                        event_str, buffer = buffer.split("\n\n", 1)
+                        _process_sse_event(event_str, output, last_cursor)
+
+    except httpx.ConnectError:
+        console.print(f"[red]Error: Could not connect to {server}[/red]")
+        console.print("[dim]Is the collector running? Start with: logtap collect[/dim]")
+        raise typer.Exit(1)
+    except KeyboardInterrupt:
+        console.print("\n[dim]Stopped.[/dim]")
+
+
+def _process_sse_event(event_str: str, output: str, last_cursor: Optional[int]) -> None:
+    """Process a single SSE event."""
+    event_type = None
+    data = None
+
+    for line in event_str.split("\n"):
+        if line.startswith(":"):
+            # Comment (heartbeat)
+            continue
+        elif line.startswith("event: "):
+            event_type = line[7:]
+        elif line.startswith("data: "):
+            data = line[6:]
+        # Note: id: lines are parsed by SSE but not used for now
+
+    if event_type == "meta" and data:
+        try:
+            meta = json.loads(data)
+            if meta.get("gap") and meta.get("missed"):
+                console.print(f"[yellow]reconnected (missed {meta['missed']} lines)[/yellow]")
+        except Exception:
+            pass
+    elif event_type == "line" and data:
+        try:
+            line_data = json.loads(data)
+            if output == "jsonl":
+                console.print(data)
+            elif output == "plain":
+                console.print(line_data["line"])
+            else:
+                # Pretty output - just the line
+                console.print(line_data["line"])
+        except Exception:
+            pass
+
+
+def _tail_file(
+    server: str,
+    filename: str,
+    headers: dict,
+    follow: bool,
+    lines: int,
+    output: str,
+) -> None:
+    """Tail a static file (legacy mode)."""
+    import httpx
+
     params = {
         "filename": filename,
         "limit": lines,
@@ -64,9 +241,15 @@ def tail(
         with httpx.Client(timeout=30.0) as client:
             response = client.get(f"{server}/logs", params=params, headers=headers)
 
-        if response.status_code != 200:
-            error_detail = response.json().get("detail", response.text)
-            console.print(f"[bold red]Error:[/bold red] {error_detail}")
+        if response.status_code == 401:
+            console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+            raise typer.Exit(1)
+        elif response.status_code != 200:
+            try:
+                error_detail = response.json().get("detail", response.text)
+            except Exception:
+                error_detail = response.text
+            console.print(f"[red]Error:[/red] {error_detail}")
             raise typer.Exit(1)
 
         data = response.json()
@@ -74,7 +257,10 @@ def tail(
 
         # Print initial lines
         for line in log_lines:
-            console.print(line)
+            if output == "jsonl":
+                console.print(json.dumps({"line": line}))
+            else:
+                console.print(line)
 
         if follow:
             console.print()
@@ -91,13 +277,16 @@ def tail(
                 ws_url = f"{ws_url}/logs/stream?filename={filename}"
 
                 extra_headers = {}
-                if api_key:
-                    extra_headers["X-API-Key"] = api_key
+                if headers.get("X-API-Key"):
+                    extra_headers["X-API-Key"] = headers["X-API-Key"]
 
                 try:
                     async with websockets.connect(ws_url, extra_headers=extra_headers) as ws:
                         async for message in ws:
-                            console.print(message)
+                            if output == "jsonl":
+                                console.print(json.dumps({"line": message}))
+                            else:
+                                console.print(message)
                 except websockets.exceptions.InvalidStatusCode as e:
                     if e.status_code == 404:
                         console.print("[yellow]Streaming not available.[/yellow]")
@@ -113,9 +302,9 @@ def tail(
                 console.print("\n[dim]Stopped.[/dim]")
 
     except httpx.ConnectError:
-        console.print(f"[bold red]Error:[/bold red] Could not connect to {server}")
+        console.print(f"[red]Error:[/red] Could not connect to {server}")
         console.print("[dim]Is the logtap server running? Start it with 'logtap serve'[/dim]")
         raise typer.Exit(1)
     except Exception as e:
-        console.print(f"[bold red]Error:[/bold red] {e}")
+        console.print(f"[red]Error:[/red] {e}")
         raise typer.Exit(1)

logtap/cli/main.py

@@ -3,11 +3,11 @@
 import typer
 
 from logtap import __version__
-from logtap.cli.commands import files, query, serve, tail
+from logtap.cli.commands import collect, files, ingest, query, runs, serve, tail
 
 app = typer.Typer(
     name="logtap",
-    help="A CLI-first log access tool for Unix systems. Remote log file access without SSH.",
+    help="tail -f for GPU clouds. Survives disconnects, aggregates multi-node.",
     rich_markup_mode="rich",
     no_args_is_help=True,
 )
@@ -32,14 +32,20 @@ def main(
     ),
 ) -> None:
     """
-    logtap - Remote log file access without SSH.
+    logtap - tail -f for GPU clouds.
 
-    Start a server with 'logtap serve' or query a remote server with 'logtap query'.
+    Start a collector with 'logtap collect', pipe logs with 'logtap ingest',
+    and tail from anywhere with 'logtap tail'.
     """
     pass
 
 
-# Add commands
+# Primary commands (new ML workflow)
+app.command()(collect.collect)
+app.command()(ingest.ingest)
+app.command()(runs.runs)
+
+# Legacy commands (file-based workflow)
 app.command()(serve.serve)
 app.command()(query.query)
 app.command()(tail.tail)

logtap/core/parsers/base.py

@@ -136,12 +136,14 @@ class LogParser(ABC):
         """
         return [self.parse(line) for line in lines]
 
-    def _detect_level_from_content(self, content: str) -> Optional[LogLevel]:
+    def _detect_level_from_content(self, content: Any) -> Optional[LogLevel]:
         """
         Detect log level from message content.
 
         Common patterns like "ERROR:", "[error]", etc.
         """
+        if not isinstance(content, str):
+            return None
         content_lower = content.lower()
 
         # Check for explicit level indicators

logtap/core/parsers/json_parser.py

@@ -50,8 +50,19 @@ class JsonLogParser(LogParser):
                 level=self._detect_level_from_content(line),
             )
 
+        # Only handle JSON objects (dicts), not arrays or primitives
+        if not isinstance(data, dict):
+            return ParsedLogEntry(
+                raw=line,
+                message=line,
+                level=self._detect_level_from_content(line),
+            )
+
         # Extract message
         message = self._get_field(data, self.MESSAGE_FIELDS, line)
+        # Ensure message is a string for level detection
+        if not isinstance(message, str):
+            message = str(message) if message is not None else line
 
         # Extract level
         level_str = self._get_field(data, self.LEVEL_FIELDS)

logtap/core/reader.py

@@ -27,7 +27,7 @@ def tail(filename: str, lines_limit: int = 50, block_size: int = 1024) -> List[s
         A list of the last 'lines_limit' lines in the file.
     """
     lines: List[str] = []
-    with open(filename, "r", encoding="utf-8") as f:
+    with open(filename, "r", encoding="utf-8", errors="replace") as f:
         # Seek to the end of the file.
         f.seek(0, SEEK_END)
         # Get the current position in the file.
@@ -70,9 +70,7 @@ def read_block(file: IO, block_end_byte: int, block_size: int) -> Tuple[List[str
     return lines, block_end_byte
 
 
-async def tail_async(
-    filename: str, lines_limit: int = 50, block_size: int = 1024
-) -> List[str]:
+async def tail_async(filename: str, lines_limit: int = 50, block_size: int = 1024) -> List[str]:
     """
     Async version of tail() for use with FastAPI.
 
@@ -87,7 +85,7 @@ async def tail_async(
         A list of the last 'lines_limit' lines in the file.
     """
     lines: List[str] = []
-    async with aiofiles.open(filename, "r", encoding="utf-8") as f:
+    async with aiofiles.open(filename, "r", encoding="utf-8", errors="replace") as f:
         # Seek to the end of the file.
         await f.seek(0, SEEK_END)
         # Get the current position in the file.