logtap 0.3.0__py3-none-any.whl → 0.4.1__py3-none-any.whl

logtap/cli/commands/collect.py

@@ -0,0 +1,107 @@
+"""Collector command for logtap CLI."""
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+console = Console()
+
+
+def collect(
+    port: int = typer.Option(
+        8000,
+        "--port",
+        "-p",
+        help="Port to listen on.",
+    ),
+    host: str = typer.Option(
+        "0.0.0.0",
+        "--host",
+        "-H",
+        help="Host to bind to.",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    data_dir: Path = typer.Option(
+        Path("~/.logtap/runs").expanduser(),
+        "--data-dir",
+        "-d",
+        help="Directory for run storage.",
+    ),
+    buffer_lines: int = typer.Option(
+        100_000,
+        "--buffer-lines",
+        help="In-memory cache size per run.",
+    ),
+    max_disk_mb: int = typer.Option(
+        1000,
+        "--max-disk-mb",
+        help="Maximum disk usage across all runs (MB).",
+    ),
+    retention_hours: int = typer.Option(
+        72,
+        "--retention-hours",
+        help="Hours to retain runs before cleanup.",
+    ),
+    reload: bool = typer.Option(
+        False,
+        "--reload",
+        "-r",
+        help="Enable auto-reload for development.",
+    ),
+) -> None:
+    """
+    Start the logtap collector server.
+
+    Accepts ingested log streams over HTTP and serves them for tailing.
+    This is the recommended mode for ML training logs.
+
+    Example:
+        logtap collect
+        logtap collect --port 9000 --api-key secret
+        logtap collect --data-dir /mnt/logs --max-disk-mb 5000
+    """
+    import os
+
+    import uvicorn
+
+    # Expand data_dir
+    data_dir = Path(data_dir).expanduser()
+
+    # Set environment variables for the app
+    os.environ["LOGTAP_HOST"] = host
+    os.environ["LOGTAP_PORT"] = str(port)
+    os.environ["LOGTAP_MODE"] = "collect"
+    os.environ["LOGTAP_DATA_DIR"] = str(data_dir)
+    os.environ["LOGTAP_BUFFER_LINES"] = str(buffer_lines)
+    os.environ["LOGTAP_MAX_DISK_MB"] = str(max_disk_mb)
+    os.environ["LOGTAP_RETENTION_HOURS"] = str(retention_hours)
+    if api_key:
+        os.environ["LOGTAP_API_KEY"] = api_key
+
+    console.print("[bold green]Starting logtap collector[/bold green]")
+    console.print(f"  [dim]Host:[/dim] {host}")
+    console.print(f"  [dim]Port:[/dim] {port}")
+    console.print(f"  [dim]Data directory:[/dim] {data_dir}")
+    console.print(f"  [dim]Buffer lines:[/dim] {buffer_lines:,}")
+    console.print(f"  [dim]Max disk:[/dim] {max_disk_mb} MB")
+    console.print(f"  [dim]Retention:[/dim] {retention_hours} hours")
+    console.print(f"  [dim]Auth:[/dim] {'enabled' if api_key else 'disabled'}")
+    console.print()
+    console.print(f"[dim]API docs available at[/dim] http://{host}:{port}/docs")
+    console.print()
+
+    uvicorn.run(
+        "logtap.api.app:create_collector_app",
+        host=host,
+        port=port,
+        reload=reload,
+        factory=True,
+    )

logtap/cli/commands/doctor.py

@@ -0,0 +1,127 @@
+"""Doctor command for logtap CLI - diagnose connection issues."""
+
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+console = Console()
+
+
+def doctor(
+    server: str = typer.Option(
+        "http://localhost:8000",
+        "--server",
+        "-s",
+        help="Server URL to check.",
+        envvar="LOGTAP_SERVER",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+) -> None:
+    """
+    Check server connectivity and diagnose issues.
+
+    Verifies the server is reachable, auth works, and reports capabilities.
+
+    Example:
+        logtap doctor
+        logtap doctor --server http://gpu-box:8000
+    """
+    import httpx
+
+    console.print(f"[bold]Checking {server}[/bold]\n")
+
+    # Build headers
+    headers = {}
+    if api_key:
+        headers["X-API-Key"] = api_key
+
+    # Step 1: Check if server is reachable
+    console.print("[dim]1. Server reachable?[/dim]", end=" ")
+    try:
+        with httpx.Client(timeout=10) as client:
+            response = client.get(f"{server}/health", headers=headers)
+    except httpx.ConnectError:
+        console.print("[red]NO[/red]")
+        console.print(f"\n[red]Could not connect to {server}[/red]")
+        console.print("\n[dim]Possible causes:[/dim]")
+        console.print("  - Server not running (start with: logtap collect)")
+        console.print("  - Wrong host/port")
+        console.print("  - Firewall blocking connection")
+        console.print(f"\n[dim]Try:[/dim] curl {server}/health")
+        raise typer.Exit(1)
+    except httpx.TimeoutException:
+        console.print("[red]TIMEOUT[/red]")
+        console.print(f"\n[red]Connection timed out to {server}[/red]")
+        raise typer.Exit(1)
+
+    console.print("[green]YES[/green]")
+
+    # Step 2: Check auth
+    console.print("[dim]2. Auth working?[/dim]", end=" ")
+    if response.status_code == 401:
+        console.print("[red]NO (401 Unauthorized)[/red]")
+        console.print("\n[red]API key required or invalid[/red]")
+        if not api_key:
+            console.print("\n[dim]Try:[/dim] logtap doctor --api-key YOUR_KEY")
+            console.print("[dim]Or:[/dim] export LOGTAP_API_KEY=YOUR_KEY")
+        else:
+            console.print("\n[dim]Check that your API key matches the server's --api-key[/dim]")
+        raise typer.Exit(1)
+    elif response.status_code >= 400:
+        console.print(f"[red]NO ({response.status_code})[/red]")
+        raise typer.Exit(1)
+
+    if api_key:
+        console.print("[green]YES (key accepted)[/green]")
+    else:
+        console.print("[green]YES (no auth required)[/green]")
+
+    # Step 3: Parse health response
+    try:
+        health = response.json()
+    except Exception:
+        console.print("\n[yellow]Warning: Could not parse health response[/yellow]")
+        health = {}
+
+    # Step 4: Check features
+    console.print("[dim]3. Features?[/dim]", end=" ")
+    mode = health.get("mode", "unknown")
+    features = health.get("features", [])
+    version = health.get("version", "unknown")
+
+    if "runs" in features:
+        console.print("[green]runs[/green] (collector mode)")
+    elif "files" in features:
+        console.print("[cyan]files[/cyan] (legacy serve mode)")
+    else:
+        console.print(f"[yellow]{mode}[/yellow]")
+
+    # Step 5: Show summary
+    console.print("\n[bold green]Server OK[/bold green]")
+    console.print(f"  Version: {version}")
+    console.print(f"  Mode: {mode}")
+    console.print(f"  Features: {', '.join(features) if features else 'unknown'}")
+
+    if "runs" in features:
+        runs_count = health.get("runs", 0)
+        console.print(f"  Active runs: {runs_count}")
+
+    # Show curl command for debugging
+    console.print("\n[dim]Debug command:[/dim]")
+    if api_key:
+        console.print(f'  curl -H "X-API-Key: {api_key}" {server}/health')
+    else:
+        console.print(f"  curl {server}/health")
+
+    # Show quick start if collector mode
+    if "runs" in features:
+        console.print("\n[dim]Quick start:[/dim]")
+        console.print("  python train.py 2>&1 | logtap ingest run1")
+        console.print("  logtap tail run1 --follow")

logtap/cli/commands/ingest.py

@@ -0,0 +1,123 @@
+"""Ingest command for logtap CLI."""
+
+import sys
+from datetime import datetime
+from typing import List, Optional
+
+import typer
+from rich.console import Console
+
+console = Console(stderr=True)  # Output status to stderr so it doesn't mix with piped data
+
+
+def ingest(
+    name: Optional[str] = typer.Argument(
+        None,
+        help="Run name. If omitted, generates: run-YYYYMMDD-HHMMSS",
+    ),
+    server: str = typer.Option(
+        "http://localhost:8000",
+        "--server",
+        "-s",
+        help="Collector server URL.",
+        envvar="LOGTAP_SERVER",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    tag: Optional[List[str]] = typer.Option(
+        None,
+        "--tag",
+        "-t",
+        help="Tags as key=value (repeatable).",
+    ),
+    quiet: bool = typer.Option(
+        False,
+        "--quiet",
+        "-q",
+        help="Suppress status messages.",
+    ),
+) -> None:
+    """
+    Pipe stdin to collector as a named run.
+
+    Reads from stdin and streams lines to the collector server.
+    Designed to be used with pipes for training logs.
+
+    Example:
+        python train.py 2>&1 | logtap ingest run1
+        python train.py 2>&1 | logtap ingest --tag node=gpu1
+        cat training.log | logtap ingest my-run --server http://collector:8000
+    """
+    import httpx
+
+    # Generate run name if not provided
+    if name is None:
+        name = datetime.now().strftime("run-%Y%m%d-%H%M%S")
+
+    # Build headers as list of tuples (supports duplicate keys)
+    headers: List[tuple] = [("Content-Type", "text/plain")]
+    if api_key:
+        headers.append(("X-API-Key", api_key))
+    if tag:
+        for t in tag:
+            headers.append(("X-Logtap-Tag", t))
+
+    url = f"{server.rstrip('/')}/runs/{name}/ingest"
+
+    if not quiet:
+        console.print(f"[dim]Ingesting to[/dim] {name} [dim]@[/dim] {server}")
+
+    try:
+        # Read all stdin
+        content = sys.stdin.read()
+
+        if not quiet:
+            line_count = content.count("\n") + (1 if content and not content.endswith("\n") else 0)
+            console.print(f"[dim]Sending {line_count} lines...[/dim]")
+
+        # Use httpx
+        with httpx.Client(timeout=None) as client:
+            response = client.post(url, content=content, headers=headers)
+
+            if response.status_code == 401:
+                console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code == 409:
+                try:
+                    data = response.json()
+                    msg = data.get("detail", {}).get("message", "Tag conflict")
+                except Exception:
+                    msg = "Tag conflict"
+                console.print(f"[red]Error: {msg}[/red]")
+                raise typer.Exit(1)
+            elif response.status_code == 507:
+                console.print("[red]Error: Server storage limit exceeded.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code >= 400:
+                console.print(f"[red]Error: Server returned {response.status_code}[/red]")
+                try:
+                    console.print(f"[dim]{response.text}[/dim]")
+                except Exception:
+                    pass
+                raise typer.Exit(1)
+
+            data = response.json()
+            if not quiet:
+                console.print(
+                    f"[green]Done.[/green] "
+                    f"Ingested {data['lines_ingested']} lines to [bold]{data['run_id']}[/bold] "
+                    f"(cursor: {data['cursor_end']})"
+                )
+
+    except httpx.ConnectError:
+        console.print(f"[red]Error: Could not connect to {server}[/red]")
+        console.print("[dim]Is the collector running? Start with: logtap collect[/dim]")
+        raise typer.Exit(1)
+    except httpx.TimeoutException:
+        console.print("[red]Error: Connection timed out[/red]")
+        raise typer.Exit(1)

logtap/cli/commands/runs.py

@@ -0,0 +1,116 @@
+"""Runs command for logtap CLI."""
+
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+console = Console()
+
+
+def runs(
+    server: str = typer.Option(
+        "http://localhost:8000",
+        "--server",
+        "-s",
+        help="Collector server URL.",
+        envvar="LOGTAP_SERVER",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    since_hours: Optional[int] = typer.Option(
+        24,
+        "--since-hours",
+        help="Show runs active within N hours.",
+    ),
+) -> None:
+    """
+    List runs on a collector.
+
+    Shows all active and recent runs with their line counts and tags.
+
+    Example:
+        logtap runs
+        logtap runs --server http://gpu-box:8000
+        logtap runs --since-hours 48
+    """
+    from datetime import datetime, timezone
+
+    import httpx
+
+    # Build headers
+    headers = {}
+    if api_key:
+        headers["X-API-Key"] = api_key
+
+    url = f"{server.rstrip('/')}/runs"
+    params = {}
+    if since_hours is not None:
+        params["since_hours"] = since_hours
+
+    try:
+        with httpx.Client(timeout=30) as client:
+            response = client.get(url, headers=headers, params=params)
+
+            if response.status_code == 401:
+                console.print("[red]Error: Unauthorized. Check your API key.[/red]")
+                raise typer.Exit(1)
+            elif response.status_code >= 400:
+                console.print(f"[red]Error: Server returned {response.status_code}[/red]")
+                raise typer.Exit(1)
+
+            data = response.json()
+            runs_list = data.get("runs", [])
+
+            if not runs_list:
+                console.print("[dim]No runs found.[/dim]")
+                return
+
+            # Create table
+            table = Table(show_header=True, header_style="bold")
+            table.add_column("RUN", style="cyan")
+            table.add_column("LINES", justify="right")
+            table.add_column("LAST ACTIVITY", style="dim")
+            table.add_column("TAGS")
+
+            now = datetime.now(timezone.utc)
+
+            for run in runs_list:
+                # Format last activity as relative time
+                last_activity = datetime.fromisoformat(run["last_activity"].replace("Z", "+00:00"))
+                delta = now - last_activity
+                if delta.total_seconds() < 60:
+                    time_str = f"{int(delta.total_seconds())}s ago"
+                elif delta.total_seconds() < 3600:
+                    time_str = f"{int(delta.total_seconds() / 60)}m ago"
+                elif delta.total_seconds() < 86400:
+                    time_str = f"{int(delta.total_seconds() / 3600)}h ago"
+                else:
+                    time_str = f"{int(delta.total_seconds() / 86400)}d ago"
+
+                # Format tags
+                tags = run.get("tags", {})
+                tags_str = ", ".join(f"{k}={v}" for k, v in tags.items()) if tags else ""
+
+                table.add_row(
+                    run["id"],
+                    f"{run['lines']:,}",
+                    time_str,
+                    tags_str,
+                )
+
+            console.print(table)
+
+    except httpx.ConnectError:
+        console.print(f"[red]Error: Could not connect to {server}[/red]")
+        console.print("[dim]Is the collector running? Start with: logtap collect[/dim]")
+        raise typer.Exit(1)
+    except httpx.TimeoutException:
+        console.print("[red]Error: Connection timed out[/red]")
+        raise typer.Exit(1)