logtap 0.2.2__py3-none-any.whl → 0.4.0__py3-none-any.whl

logtap/__init__.py

@@ -4,5 +4,5 @@ logtap - A CLI-first log access tool for Unix systems.
 Remote log file access without SSH. No database. No complex setup.
 """
 
-__version__ = "0.1.0"
+__version__ = "0.4.0"
 __author__ = "Kyle Cain"

logtap/api/app.py

@@ -1,15 +1,22 @@
 """FastAPI application factory for logtap."""
 
+import os
+import time
+from pathlib import Path
+
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
 from logtap import __version__
-from logtap.api.routes import files, health, logs, parsed
+from logtap.api.routes import files, health, logs, parsed, runs
+from logtap.core.runs import RunStore
 
 
 def create_app() -> FastAPI:
     """
-    Create and configure the FastAPI application.
+    Create and configure the FastAPI application for serve mode.
+
+    Serves static log files from a directory (legacy mode).
 
     Returns:
         Configured FastAPI application instance.
@@ -23,6 +30,10 @@ def create_app() -> FastAPI:
         openapi_url="/openapi.json",
     )
 
+    # Store mode info
+    app.state.mode = "serve"
+    app.state.features = ["files"]
+
     # Configure CORS
     app.add_middleware(
         CORSMiddleware,
@@ -41,5 +52,60 @@ def create_app() -> FastAPI:
     return app
 
 
-# Create default app instance for uvicorn
+def create_collector_app() -> FastAPI:
+    """
+    Create and configure the FastAPI application for collector mode.
+
+    Accepts ingested log streams and serves them for tailing.
+    This is the recommended mode for ML training logs.
+
+    Returns:
+        Configured FastAPI application instance.
+    """
+    app = FastAPI(
+        title="logtap",
+        description="tail -f for GPU clouds. Survives disconnects, aggregates multi-node.",
+        version=__version__,
+        docs_url="/docs",
+        redoc_url="/redoc",
+        openapi_url="/openapi.json",
+    )
+
+    # Store mode info and start time
+    app.state.mode = "collect"
+    app.state.features = ["runs"]
+    app.state.start_time = time.time()
+
+    # Configure CORS
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    # Initialize run store from environment
+    data_dir = Path(os.environ.get("LOGTAP_DATA_DIR", "~/.logtap/runs")).expanduser()
+    buffer_lines = int(os.environ.get("LOGTAP_BUFFER_LINES", "100000"))
+    max_disk_mb = int(os.environ.get("LOGTAP_MAX_DISK_MB", "1000"))
+    retention_hours = int(os.environ.get("LOGTAP_RETENTION_HOURS", "72"))
+
+    run_store = RunStore(
+        data_dir=data_dir,
+        buffer_lines=buffer_lines,
+        max_disk_mb=max_disk_mb,
+        retention_hours=retention_hours,
+    )
+    runs.set_run_store(run_store)
+    app.state.run_store = run_store
+
+    # Include routers
+    app.include_router(health.router, tags=["health"])
+    app.include_router(runs.router, prefix="/runs", tags=["runs"])
+
+    return app
+
+
+# Create default app instance for uvicorn (serve mode)
 app = create_app()

logtap/api/routes/health.py

@@ -1,6 +1,8 @@
 """Health check endpoint for logtap."""
 
-from fastapi import APIRouter
+import time
+
+from fastapi import APIRouter, Request
 
 from logtap import __version__
 from logtap.models.responses import HealthResponse
@@ -9,11 +11,31 @@ router = APIRouter()
 
 
 @router.get("/health", response_model=HealthResponse)
-async def health_check() -> HealthResponse:
+async def health_check(request: Request) -> HealthResponse:
     """
     Check the health of the logtap service.
 
     Returns:
-        Health status and version information.
+        Health status, version, mode, and capability information.
     """
-    return HealthResponse(status="healthy", version=__version__)
+    mode = getattr(request.app.state, "mode", "serve")
+    features = getattr(request.app.state, "features", ["files"])
+
+    # Get run count if in collect mode
+    runs_count = None
+    if hasattr(request.app.state, "run_store"):
+        runs_count = len(request.app.state.run_store.list_runs())
+
+    # Get uptime if start_time is set
+    uptime = None
+    if hasattr(request.app.state, "start_time"):
+        uptime = int(time.time() - request.app.state.start_time)
+
+    return HealthResponse(
+        status="healthy",
+        version=__version__,
+        mode=mode,
+        features=features,
+        runs=runs_count,
+        uptime_seconds=uptime,
+    )

logtap/api/routes/logs.py

@@ -180,7 +180,7 @@ async def stream_logs(
         return
 
     try:
-        async with aiofiles.open(filepath, mode="r", encoding="utf-8") as f:
+        async with aiofiles.open(filepath, mode="r", encoding="utf-8", errors="replace") as f:
             # Seek to end of file
             await f.seek(0, 2)
 
@@ -225,7 +225,7 @@ async def stream_logs_sse(
     filepath = get_filepath(filename, settings)
 
     async def event_generator():
-        async with aiofiles.open(filepath, mode="r", encoding="utf-8") as f:
+        async with aiofiles.open(filepath, mode="r", encoding="utf-8", errors="replace") as f:
             # Seek to end
             await f.seek(0, 2)
 

logtap/api/routes/runs.py

@@ -0,0 +1,351 @@
+"""Routes for run management (collector mode)."""
+
+import asyncio
+from typing import List, Optional
+
+from fastapi import APIRouter, Depends, Header, HTTPException, Query, Request, Response
+from fastapi.responses import StreamingResponse
+
+from logtap.api.dependencies import verify_api_key
+from logtap.core.runs import RunStore
+from logtap.models.responses import (
+    IngestResponse,
+    RunInfo,
+    RunListResponse,
+    StreamLineEvent,
+    StreamMetaEvent,
+)
+
+router = APIRouter()
+
+# Global run store - will be set by app factory
+_run_store: Optional[RunStore] = None
+
+
+def get_run_store() -> RunStore:
+    """Get the run store instance."""
+    if _run_store is None:
+        raise HTTPException(status_code=500, detail="Run store not initialized")
+    return _run_store
+
+
+def set_run_store(store: RunStore) -> None:
+    """Set the global run store instance."""
+    global _run_store
+    _run_store = store
+
+
+def parse_tags(tag_headers: Optional[List[str]]) -> dict:
+    """Parse X-Logtap-Tag headers into dict."""
+    if not tag_headers:
+        return {}
+
+    tags = {}
+    for tag in tag_headers:
+        if "=" in tag:
+            key, value = tag.split("=", 1)
+            tags[key.strip()] = value.strip()
+    return tags
+
+
+@router.get("", response_model=RunListResponse)
+async def list_runs(
+    since_hours: Optional[int] = Query(None, description="Filter to runs active within N hours"),
+    _: None = Depends(verify_api_key),
+    store: RunStore = Depends(get_run_store),
+) -> RunListResponse:
+    """List all runs."""
+    runs = store.list_runs(since_hours=since_hours)
+
+    return RunListResponse(
+        runs=[
+            RunInfo(
+                id=run.id,
+                lines=run.metadata.lines_count,
+                cursor_earliest=run.cursor_earliest,
+                cursor_latest=run.cursor_latest,
+                tags=run.metadata.tags,
+                created_at=run.metadata.created_at,
+                last_activity=run.metadata.last_activity,
+                active=run.metadata.active,
+                bytes_on_disk=run.metadata.bytes_on_disk,
+            )
+            for run in runs
+        ]
+    )
+
+
+@router.get("/{run_id}", response_model=RunInfo)
+async def get_run(
+    run_id: str,
+    _: None = Depends(verify_api_key),
+    store: RunStore = Depends(get_run_store),
+) -> RunInfo:
+    """Get details for a specific run."""
+    run = store.get(run_id)
+    if run is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": "run_not_found", "message": f"Run '{run_id}' does not exist"},
+        )
+
+    return RunInfo(
+        id=run.id,
+        lines=run.metadata.lines_count,
+        cursor_earliest=run.cursor_earliest,
+        cursor_latest=run.cursor_latest,
+        tags=run.metadata.tags,
+        created_at=run.metadata.created_at,
+        last_activity=run.metadata.last_activity,
+        active=run.metadata.active,
+        bytes_on_disk=run.metadata.bytes_on_disk,
+    )
+
+
+@router.post("/{run_id}/ingest", response_model=IngestResponse)
+async def ingest(
+    run_id: str,
+    request: Request,
+    response: Response,
+    x_logtap_tag: Optional[List[str]] = Header(None),
+    _: None = Depends(verify_api_key),
+    store: RunStore = Depends(get_run_store),
+) -> IngestResponse:
+    """
+    Ingest log lines for a run.
+
+    Send lines as plain text with newline delimiters.
+    Supports chunked transfer encoding for streaming.
+    """
+    # Check storage
+    storage_err = store.check_storage()
+    if storage_err:
+        raise HTTPException(
+            status_code=507,
+            detail={"error": "insufficient_storage", "message": "Disk limit exceeded"},
+        )
+
+    # Get or create run
+    run, created = store.get_or_create(run_id)
+
+    # Handle tags
+    tags = parse_tags(x_logtap_tag)
+    if tags:
+        err = run.set_tags(tags)
+        if err:
+            raise HTTPException(
+                status_code=409,
+                detail={"error": "tag_conflict", "message": err},
+            )
+
+    # Read and ingest body
+    lines_ingested = 0
+    buffer = ""
+
+    async for chunk in request.stream():
+        text = chunk.decode("utf-8", errors="replace")
+        buffer += text
+
+        # Process complete lines
+        while "\n" in buffer:
+            line, buffer = buffer.split("\n", 1)
+            run.append(line)
+            lines_ingested += 1
+
+    # Flush remaining partial line
+    if buffer:
+        run.append(buffer)
+        lines_ingested += 1
+
+    # Save metadata
+    run._save_metadata()
+
+    # Set status code: 201 for new run, 200 for existing
+    response.status_code = 201 if created else 200
+
+    return IngestResponse(
+        run_id=run_id,
+        lines_ingested=lines_ingested,
+        cursor_end=run.cursor_latest,
+    )
+
+
+@router.get("/{run_id}/stream")
+async def stream_run(
+    run_id: str,
+    since: Optional[int] = Query(None, description="Cursor to resume from (exclusive)"),
+    tail: int = Query(50, description="Lines to show if since not provided"),
+    follow: bool = Query(False, description="Keep connection open for new lines"),
+    tag: Optional[List[str]] = Query(None, description="Filter by tag (key=value)"),
+    _: None = Depends(verify_api_key),
+    store: RunStore = Depends(get_run_store),
+) -> StreamingResponse:
+    """
+    Stream lines from a run using Server-Sent Events.
+
+    Supports resume via `since` parameter or Last-Event-ID header.
+    """
+    run = store.get(run_id)
+    if run is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": "run_not_found", "message": f"Run '{run_id}' does not exist"},
+        )
+
+    # Tag filtering (if specified, check run has all tags)
+    if tag:
+        required_tags = parse_tags(tag)
+        for key, value in required_tags.items():
+            if run.metadata.tags.get(key) != value:
+                # Run doesn't match filter - return empty stream
+                async def empty_stream():
+                    meta = StreamMetaEvent(
+                        cursor_earliest=run.cursor_earliest,
+                        cursor_latest=run.cursor_latest,
+                        gap=False,
+                    )
+                    yield f"event: meta\ndata: {meta.model_dump_json()}\n\n"
+
+                return StreamingResponse(
+                    empty_stream(),
+                    media_type="text/event-stream",
+                    headers={
+                        "Cache-Control": "no-cache",
+                        "X-Logtap-Earliest-Cursor": str(run.cursor_earliest),
+                        "X-Logtap-Latest-Cursor": str(run.cursor_latest),
+                    },
+                )
+
+    async def generate_sse():
+        # Get initial lines and check for gap
+        lines, gap = run.get_lines(since=since, tail=tail)
+
+        # Send meta event
+        missed = None
+        if gap and since is not None:
+            missed = run.cursor_earliest - since - 1
+            if missed < 0:
+                missed = 0
+
+        meta = StreamMetaEvent(
+            cursor_earliest=run.cursor_earliest,
+            cursor_latest=run.cursor_latest,
+            gap=gap,
+            missed=missed,
+        )
+        yield f"event: meta\ndata: {meta.model_dump_json()}\n\n"
+
+        # Send initial lines
+        last_cursor = since if since is not None else -1
+        for line in lines:
+            event = StreamLineEvent(
+                cursor=line.cursor,
+                line=line.line,
+                ts=line.ts,
+            )
+            yield f"id: {line.cursor}\nevent: line\ndata: {event.model_dump_json()}\n\n"
+            last_cursor = line.cursor
+
+        if not follow:
+            return
+
+        # Follow mode - stream new lines
+        heartbeat_interval = 15  # seconds
+        last_heartbeat = asyncio.get_event_loop().time()
+
+        while True:
+            # Get new lines since last cursor
+            new_lines, _ = run.get_lines(since=last_cursor, limit=100)
+
+            for line in new_lines:
+                event = StreamLineEvent(
+                    cursor=line.cursor,
+                    line=line.line,
+                    ts=line.ts,
+                )
+                yield f"id: {line.cursor}\nevent: line\ndata: {event.model_dump_json()}\n\n"
+                last_cursor = line.cursor
+
+            # Send heartbeat if needed
+            now = asyncio.get_event_loop().time()
+            if now - last_heartbeat >= heartbeat_interval:
+                yield ": heartbeat\n\n"
+                last_heartbeat = now
+
+            # Small delay before checking again
+            await asyncio.sleep(0.1)
+
+    return StreamingResponse(
+        generate_sse(),
+        media_type="text/event-stream",
+        headers={
+            "Cache-Control": "no-cache",
+            "X-Logtap-Earliest-Cursor": str(run.cursor_earliest),
+            "X-Logtap-Latest-Cursor": str(run.cursor_latest),
+        },
+    )
+
+
+@router.get("/{run_id}/query")
+async def query_run(
+    run_id: str,
+    from_cursor: Optional[int] = Query(None, alias="from", description="Start cursor (inclusive)"),
+    to_cursor: Optional[int] = Query(None, alias="to", description="End cursor (inclusive)"),
+    tail: int = Query(50, description="Last N lines (if from/to not provided)"),
+    limit: int = Query(1000, le=10000, description="Maximum lines to return"),
+    search: Optional[str] = Query(None, description="Substring filter"),
+    regex: Optional[str] = Query(None, description="Regex filter"),
+    output: str = Query("jsonl", description="Output format: jsonl or plain"),
+    _: None = Depends(verify_api_key),
+    store: RunStore = Depends(get_run_store),
+) -> StreamingResponse:
+    """Query lines from a run."""
+    # Validate search/regex mutual exclusion
+    if search and regex:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "invalid_query", "message": "Cannot use both search and regex"},
+        )
+
+    run = store.get(run_id)
+    if run is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": "run_not_found", "message": f"Run '{run_id}' does not exist"},
+        )
+
+    # Get lines
+    if from_cursor is not None:
+        # Range query - get lines from cursor onwards
+        lines, _ = run.get_lines(since=from_cursor - 1, limit=limit)
+        if to_cursor is not None:
+            lines = [ln for ln in lines if ln.cursor <= to_cursor]
+    else:
+        # Tail query
+        lines, _ = run.get_lines(tail=tail, limit=limit)
+
+    # Apply search/regex filter
+    if search:
+        lines = [ln for ln in lines if search in ln.line]
+    elif regex:
+        import re2
+
+        try:
+            pattern = re2.compile(regex)
+            lines = [ln for ln in lines if pattern.search(ln.line)]
+        except re2.error:
+            raise HTTPException(
+                status_code=400,
+                detail={"error": "invalid_regex", "message": "Invalid regex pattern"},
+            )
+
+    async def generate():
+        for line in lines:
+            if output == "plain":
+                yield line.line + "\n"
+            else:
+                event = StreamLineEvent(cursor=line.cursor, line=line.line, ts=line.ts)
+                yield event.model_dump_json() + "\n"
+
+    content_type = "text/plain" if output == "plain" else "application/x-ndjson"
+    return StreamingResponse(generate(), media_type=content_type)

logtap/cli/commands/collect.py

@@ -0,0 +1,107 @@
+"""Collector command for logtap CLI."""
+
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+
+console = Console()
+
+
+def collect(
+    port: int = typer.Option(
+        8000,
+        "--port",
+        "-p",
+        help="Port to listen on.",
+    ),
+    host: str = typer.Option(
+        "0.0.0.0",
+        "--host",
+        "-H",
+        help="Host to bind to.",
+    ),
+    api_key: Optional[str] = typer.Option(
+        None,
+        "--api-key",
+        "-k",
+        help="API key for authentication.",
+        envvar="LOGTAP_API_KEY",
+    ),
+    data_dir: Path = typer.Option(
+        Path("~/.logtap/runs").expanduser(),
+        "--data-dir",
+        "-d",
+        help="Directory for run storage.",
+    ),
+    buffer_lines: int = typer.Option(
+        100_000,
+        "--buffer-lines",
+        help="In-memory cache size per run.",
+    ),
+    max_disk_mb: int = typer.Option(
+        1000,
+        "--max-disk-mb",
+        help="Maximum disk usage across all runs (MB).",
+    ),
+    retention_hours: int = typer.Option(
+        72,
+        "--retention-hours",
+        help="Hours to retain runs before cleanup.",
+    ),
+    reload: bool = typer.Option(
+        False,
+        "--reload",
+        "-r",
+        help="Enable auto-reload for development.",
+    ),
+) -> None:
+    """
+    Start the logtap collector server.
+
+    Accepts ingested log streams over HTTP and serves them for tailing.
+    This is the recommended mode for ML training logs.
+
+    Example:
+        logtap collect
+        logtap collect --port 9000 --api-key secret
+        logtap collect --data-dir /mnt/logs --max-disk-mb 5000
+    """
+    import os
+
+    import uvicorn
+
+    # Expand data_dir
+    data_dir = Path(data_dir).expanduser()
+
+    # Set environment variables for the app
+    os.environ["LOGTAP_HOST"] = host
+    os.environ["LOGTAP_PORT"] = str(port)
+    os.environ["LOGTAP_MODE"] = "collect"
+    os.environ["LOGTAP_DATA_DIR"] = str(data_dir)
+    os.environ["LOGTAP_BUFFER_LINES"] = str(buffer_lines)
+    os.environ["LOGTAP_MAX_DISK_MB"] = str(max_disk_mb)
+    os.environ["LOGTAP_RETENTION_HOURS"] = str(retention_hours)
+    if api_key:
+        os.environ["LOGTAP_API_KEY"] = api_key
+
+    console.print("[bold green]Starting logtap collector[/bold green]")
+    console.print(f"  [dim]Host:[/dim] {host}")
+    console.print(f"  [dim]Port:[/dim] {port}")
+    console.print(f"  [dim]Data directory:[/dim] {data_dir}")
+    console.print(f"  [dim]Buffer lines:[/dim] {buffer_lines:,}")
+    console.print(f"  [dim]Max disk:[/dim] {max_disk_mb} MB")
+    console.print(f"  [dim]Retention:[/dim] {retention_hours} hours")
+    console.print(f"  [dim]Auth:[/dim] {'enabled' if api_key else 'disabled'}")
+    console.print()
+    console.print(f"[dim]API docs available at[/dim] http://{host}:{port}/docs")
+    console.print()
+
+    uvicorn.run(
+        "logtap.api.app:create_collector_app",
+        host=host,
+        port=port,
+        reload=reload,
+        factory=True,
+    )