anysite-cli 0.1.0__py3-none-any.whl

anysite/batch/rate_limiter.py

@@ -0,0 +1,98 @@
+"""Token bucket rate limiter for controlling request rates."""
+
+import asyncio
+import time
+
+
+class RateLimiter:
+    """Async rate limiter using the token bucket algorithm.
+
+    Usage:
+        limiter = RateLimiter("10/s")
+        async with limiter:
+            await make_request()
+    """
+
+    def __init__(self, rate_string: str) -> None:
+        """Initialize rate limiter from a rate string.
+
+        Args:
+            rate_string: Rate limit string (e.g., '10/s', '100/m', '1000/h')
+        """
+        self.max_tokens, self.interval = self.parse_rate(rate_string)
+        self.refill_rate = self.max_tokens / self.interval  # tokens per second
+        self.tokens = float(self.max_tokens)
+        self._last_refill = time.monotonic()
+        self._lock = asyncio.Lock()
+
+    @staticmethod
+    def parse_rate(rate_string: str) -> tuple[int, float]:
+        """Parse a rate string into (max_tokens, interval_seconds).
+
+        Args:
+            rate_string: Rate string like '10/s', '100/m', '1000/h'
+
+        Returns:
+            Tuple of (max_tokens, interval_in_seconds)
+
+        Raises:
+            ValueError: If rate string is invalid
+        """
+        rate_string = rate_string.strip()
+
+        parts = rate_string.split("/")
+        if len(parts) != 2:
+            raise ValueError(
+                f"Invalid rate format: '{rate_string}'. "
+                "Expected format: '<number>/<unit>' (e.g., '10/s', '100/m', '1000/h')"
+            )
+
+        try:
+            count = int(parts[0])
+        except ValueError:
+            raise ValueError(f"Invalid rate count: '{parts[0]}'. Must be an integer.") from None
+
+        unit = parts[1].strip().lower()
+        intervals = {"s": 1.0, "m": 60.0, "h": 3600.0}
+
+        if unit not in intervals:
+            raise ValueError(
+                f"Invalid rate unit: '{unit}'. Must be 's' (seconds), 'm' (minutes), or 'h' (hours)."
+            )
+
+        return count, intervals[unit]
+
+    def _refill(self) -> None:
+        """Refill tokens based on elapsed time."""
+        now = time.monotonic()
+        elapsed = now - self._last_refill
+        self.tokens = min(
+            self.max_tokens,
+            self.tokens + elapsed * self.refill_rate,
+        )
+        self._last_refill = now
+
+    async def acquire(self) -> None:
+        """Wait until a token is available, then consume one.
+
+        This method will block (async sleep) if no tokens are available.
+        """
+        async with self._lock:
+            self._refill()
+
+            if self.tokens < 1:
+                # Calculate wait time for next token
+                wait_time = (1 - self.tokens) / self.refill_rate
+                await asyncio.sleep(wait_time)
+                self._refill()
+
+            self.tokens -= 1
+
+    async def __aenter__(self) -> "RateLimiter":
+        """Acquire a token on context entry."""
+        await self.acquire()
+        return self
+
+    async def __aexit__(self, *args: object) -> None:
+        """No-op on context exit."""
+        pass

anysite/cli/__init__.py

@@ -0,0 +1 @@
+"""CLI command modules."""

anysite/cli/config.py

@@ -0,0 +1,176 @@
+"""Configuration management commands."""
+
+from typing import Annotated
+
+import typer
+from rich.table import Table
+
+from anysite.config import get_config_dir, get_config_path
+from anysite.config.settings import get_config_value, list_config, save_config
+from anysite.output.console import console, print_error, print_success
+
+app = typer.Typer(
+    help="Manage Anysite CLI configuration",
+    no_args_is_help=True,
+)
+
+
+@app.command("set")
+def config_set(
+    key: Annotated[str, typer.Argument(help="Configuration key (e.g., api_key, defaults.format)")],
+    value: Annotated[str, typer.Argument(help="Value to set")],
+) -> None:
+    """Set a configuration value.
+
+    \b
+    Examples:
+      anysite config set api_key sk-xxxxx
+      anysite config set defaults.format table
+      anysite config set defaults.count 20
+    """
+    # Convert value types
+    typed_value: str | int | bool = value
+    if value.lower() in ("true", "false"):
+        typed_value = value.lower() == "true"
+    elif value.isdigit():
+        typed_value = int(value)
+
+    try:
+        save_config(key, typed_value)
+        print_success(f"Set {key} = {typed_value}")
+    except Exception as e:
+        print_error(f"Failed to save configuration: {e}")
+        raise typer.Exit(1) from e
+
+
+@app.command("get")
+def config_get(
+    key: Annotated[str, typer.Argument(help="Configuration key to get")],
+) -> None:
+    """Get a configuration value.
+
+    \b
+    Examples:
+      anysite config get api_key
+      anysite config get defaults.format
+    """
+    value = get_config_value(key)
+    if value is None:
+        print_error(f"Configuration key '{key}' not found")
+        raise typer.Exit(1)
+
+    # Mask API key for security
+    if key == "api_key" and isinstance(value, str) and len(value) > 8:
+        masked = value[:4] + "*" * (len(value) - 8) + value[-4:]
+        console.print(f"{key}: {masked}")
+    else:
+        console.print(f"{key}: {value}")
+
+
+@app.command("list")
+def config_list() -> None:
+    """List all configuration values.
+
+    \b
+    Example:
+      anysite config list
+    """
+    config = list_config()
+
+    if not config:
+        console.print("[dim]No configuration set. Run 'anysite config init' to set up.[/dim]")
+        return
+
+    table = Table(show_header=True, header_style="bold")
+    table.add_column("Key", style="cyan")
+    table.add_column("Value")
+
+    def add_items(data: dict, prefix: str = "") -> None:
+        for key, value in data.items():
+            full_key = f"{prefix}.{key}" if prefix else key
+            if isinstance(value, dict):
+                add_items(value, full_key)
+            else:
+                # Mask API key
+                if key == "api_key" and isinstance(value, str) and len(value) > 8:
+                    value = value[:4] + "*" * (len(value) - 8) + value[-4:]
+                table.add_row(full_key, str(value))
+
+    add_items(config)
+    console.print(table)
+
+
+@app.command("path")
+def config_path() -> None:
+    """Show the configuration file path.
+
+    \b
+    Example:
+      anysite config path
+    """
+    path = get_config_path()
+    console.print(f"Config directory: {get_config_dir()}")
+    console.print(f"Config file: {path}")
+    console.print(f"Exists: {path.exists()}")
+
+
+@app.command("init")
+def config_init(
+    api_key: Annotated[
+        str | None,
+        typer.Option(
+            "--api-key",
+            "-k",
+            help="API key to set",
+            prompt="Enter your Anysite API key",
+            hide_input=False,
+        ),
+    ] = None,
+) -> None:
+    """Initialize configuration interactively.
+
+    \b
+    Example:
+      anysite config init
+      anysite config init --api-key sk-xxxxx
+    """
+    if api_key:
+        save_config("api_key", api_key)
+        print_success("Configuration initialized!")
+        console.print(f"\nConfig saved to: {get_config_path()}")
+        console.print("\nYou can now run commands like:")
+        console.print("  [cyan]anysite linkedin user satyanadella[/cyan]")
+
+
+@app.command("reset")
+def config_reset(
+    force: Annotated[
+        bool,
+        typer.Option(
+            "--force",
+            "-f",
+            help="Skip confirmation",
+        ),
+    ] = False,
+) -> None:
+    """Reset configuration to defaults.
+
+    \b
+    Example:
+      anysite config reset
+      anysite config reset --force
+    """
+    config_path = get_config_path()
+
+    if not config_path.exists():
+        console.print("[dim]No configuration file to reset.[/dim]")
+        return
+
+    if not force:
+        confirm = typer.confirm("Are you sure you want to reset all configuration?")
+        if not confirm:
+            console.print("Aborted.")
+            return
+
+    config_path.unlink()
+    print_success("Configuration reset to defaults")

anysite/cli/executor.py

@@ -0,0 +1,388 @@
+"""Shared command execution helpers for CLI commands.
+
+Provides unified execution logic for search/list commands and
+single-item commands with Phase 2 features (streaming, batch,
+progress, enhanced fields).
+"""
+
+import asyncio
+import time
+from pathlib import Path
+from typing import Any
+
+from anysite.api.client import create_client
+from anysite.api.errors import AnysiteError
+from anysite.batch.executor import BatchExecutor, BatchResult
+from anysite.batch.input import InputParser
+from anysite.batch.rate_limiter import RateLimiter
+from anysite.cli.options import ErrorHandling, parse_exclude, parse_fields
+from anysite.output.console import print_error, print_info, print_success
+from anysite.output.formatters import OutputFormat, format_output
+from anysite.output.templates import FilenameTemplate
+from anysite.streaming.progress import ProgressTracker
+from anysite.streaming.writer import StreamingWriter
+from anysite.utils.fields import resolve_fields_preset
+
+
+def _resolve_fields(
+    fields: str | None,
+    exclude: str | None,
+    fields_preset: str | None,
+) -> tuple[list[str] | None, list[str] | None]:
+    """Resolve field selection from various sources.
+
+    Returns:
+        Tuple of (fields_to_include, fields_to_exclude)
+    """
+    include = parse_fields(fields)
+    excl = parse_exclude(exclude)
+
+    if fields_preset and not include:
+        preset_fields = resolve_fields_preset(fields_preset)
+        if preset_fields:
+            include = preset_fields
+
+    return include, excl
+
+
+def _print_stats(stats: dict[str, Any], batch_result: BatchResult | None = None) -> None:
+    """Print execution statistics."""
+    lines = ["", "Statistics:"]
+    lines.append(f"  Total records:    {stats.get('total', 0)}")
+    lines.append(f"  Total time:       {stats.get('elapsed_seconds', 0):.1f}s")
+    lines.append(f"  Records/second:   {stats.get('records_per_second', 0):.1f}")
+
+    if batch_result:
+        lines.append(f"  Succeeded:        {batch_result.succeeded}")
+        if batch_result.failed > 0:
+            lines.append(f"  Failed:           {batch_result.failed}")
+        if batch_result.skipped > 0:
+            lines.append(f"  Skipped:          {batch_result.skipped}")
+
+    from anysite.output.console import error_console
+    for line in lines:
+        error_console.print(f"[dim]{line}[/dim]", style="dim")
+
+
+async def execute_search_command(
+    endpoint: str,
+    payload: dict[str, Any],
+    *,
+    # Phase 1 options
+    format: OutputFormat = OutputFormat.JSON,
+    fields: str | None = None,
+    output: Path | None = None,
+    quiet: bool = False,
+    # Phase 2: Enhanced fields
+    exclude: str | None = None,
+    compact: bool = False,
+    fields_preset: str | None = None,
+    # Phase 2: Streaming
+    stream: bool = False,
+    # Phase 2: Progress & feedback
+    progress: bool | None = None,  # noqa: ARG001
+    stats: bool = False,
+    verbose: bool = False,
+    # Phase 2: Output
+    append: bool = False,
+) -> None:
+    """Execute a search/list command with Phase 2 features.
+
+    Handles:
+    - Streaming output (--stream)
+    - Enhanced field selection (--exclude, --compact, --fields-preset)
+    - Progress bars
+    - Statistics
+    """
+    include_fields, excl_fields = _resolve_fields(fields, exclude, fields_preset)
+
+    async with create_client() as client:
+        if verbose:
+            print_info(f"Requesting {endpoint} with {payload}")
+
+        start_time = time.monotonic()
+        data = await client.post(endpoint, data=payload)
+        elapsed = time.monotonic() - start_time
+
+        if verbose:
+            count = len(data) if isinstance(data, list) else 1
+            print_info(f"Received {count} records in {elapsed:.1f}s")
+
+        # Streaming mode
+        if stream and isinstance(data, list):
+            writer = StreamingWriter(
+                output=output,
+                format=OutputFormat.JSONL,
+                fields=include_fields,
+                exclude=excl_fields,
+                compact=compact,
+                append=append,
+            )
+            with writer:
+                for record in data:
+                    writer.write(record)
+
+            if not quiet and output:
+                print_success(f"Streamed {writer.count} records to {output}")
+
+        else:
+            # Standard output
+            format_output(
+                data,
+                format,
+                include_fields,
+                output,
+                quiet,
+                exclude=excl_fields,
+                compact=compact,
+                append=append,
+            )
+
+        # Show stats
+        if stats and not quiet:
+            total = len(data) if isinstance(data, list) else 1
+            stat_data = {
+                "total": total,
+                "elapsed_seconds": round(elapsed, 2),
+                "records_per_second": round(total / elapsed, 1) if elapsed > 0 else 0,
+            }
+            _print_stats(stat_data)
+
+
+async def execute_single_command(
+    endpoint: str,
+    payload: dict[str, Any],
+    *,
+    # Phase 1 options
+    format: OutputFormat = OutputFormat.JSON,
+    fields: str | None = None,
+    output: Path | None = None,
+    quiet: bool = False,
+    # Phase 2: Batch input
+    from_file: Path | None = None,
+    stdin: bool = False,
+    parallel: int = 1,
+    delay: float = 0.0,
+    on_error: ErrorHandling = ErrorHandling.STOP,
+    # Phase 2: Enhanced fields
+    exclude: str | None = None,
+    compact: bool = False,
+    fields_preset: str | None = None,
+    # Phase 2: Rate limiting
+    rate_limit: str | None = None,
+    # Phase 2: Progress & feedback
+    progress: bool | None = None,
+    stats: bool = False,
+    verbose: bool = False,
+    # Phase 2: Output
+    append: bool = False,
+    output_dir: Path | None = None,
+    filename_template: str = "{id}",
+    # Batch-specific
+    input_key: str = "user",
+    extra_payload: dict[str, Any] | None = None,
+) -> None:
+    """Execute a single-item command with optional batch support.
+
+    Handles:
+    - Batch input from file or stdin (--from-file, --stdin)
+    - Parallel execution (--parallel)
+    - Rate limiting (--rate-limit)
+    - Per-file output (--output-dir)
+    - Progress bars
+    """
+    include_fields, excl_fields = _resolve_fields(fields, exclude, fields_preset)
+
+    # Check for batch mode
+    is_batch = from_file is not None or stdin
+
+    if not is_batch:
+        # Single request (backward compatible path)
+        async with create_client() as client:
+            if verbose:
+                print_info(f"Requesting {endpoint} with {payload}")
+
+            start_time = time.monotonic()
+            data = await client.post(endpoint, data=payload)
+            elapsed = time.monotonic() - start_time
+
+            if verbose:
+                print_info(f"Received response in {elapsed:.1f}s")
+
+            format_output(
+                data,
+                format,
+                include_fields,
+                output,
+                quiet,
+                exclude=excl_fields,
+                compact=compact,
+                append=append,
+            )
+
+            if stats and not quiet:
+                total = len(data) if isinstance(data, list) else 1
+                stat_data = {
+                    "total": total,
+                    "elapsed_seconds": round(elapsed, 2),
+                    "records_per_second": round(total / elapsed, 1) if elapsed > 0 else 0,
+                }
+                _print_stats(stat_data)
+
+        return
+
+    # Batch mode
+    inputs = InputParser.from_file(from_file) if from_file else InputParser.from_stdin()
+
+    if not inputs:
+        if not quiet:
+            print_error("No inputs found")
+        return
+
+    if verbose:
+        print_info(f"Processing {len(inputs)} inputs (parallel={parallel})")
+
+    # Setup rate limiter
+    limiter = RateLimiter(rate_limit) if rate_limit else None
+
+    # Setup progress
+    tracker = ProgressTracker(
+        total=len(inputs),
+        description="Processing...",
+        show=progress,
+        quiet=quiet,
+    )
+
+    # Create the async fetch function
+    async def _fetch_one(inp: str | dict[str, Any]) -> Any:
+        # Determine the input value
+        if isinstance(inp, dict):
+            val = inp.get(input_key, inp.get("value", str(list(inp.values())[0])))
+        else:
+            val = inp
+
+        request_payload = {input_key: val}
+        if extra_payload:
+            request_payload.update(extra_payload)
+
+        async with create_client() as client:
+            return await client.post(endpoint, data=request_payload)
+
+    # Execute batch
+    executor = BatchExecutor(
+        func=_fetch_one,
+        parallel=parallel,
+        delay=delay,
+        on_error=on_error,
+        rate_limiter=limiter,
+        progress_callback=tracker.update,
+    )
+
+    with tracker:
+        batch_result = await executor.execute(inputs)
+
+    # Output results
+    if output_dir:
+        # Per-file output
+        output_dir.mkdir(parents=True, exist_ok=True)
+        ext_map = {
+            OutputFormat.JSON: ".json",
+            OutputFormat.JSONL: ".jsonl",
+            OutputFormat.CSV: ".csv",
+            OutputFormat.TABLE: ".json",
+        }
+        template = FilenameTemplate(
+            filename_template,
+            extension=ext_map.get(format, ".json"),
+        )
+
+        for i, result in enumerate(batch_result.results):
+            inp = inputs[i] if i < len(inputs) else ""
+            input_val = inp if isinstance(inp, str) else str(list(inp.values())[0]) if isinstance(inp, dict) else str(inp)
+            filename = template.resolve(
+                record=result,
+                index=i,
+                input_value=input_val,
+            )
+            filepath = output_dir / filename
+            format_output(
+                result,
+                format,
+                include_fields,
+                filepath,
+                quiet=True,
+                exclude=excl_fields,
+                compact=compact,
+            )
+
+        if not quiet:
+            print_success(f"Saved {len(batch_result.results)} files to {output_dir}/")
+
+    else:
+        # Collect all results and output together
+        all_results = []
+        for result in batch_result.results:
+            if isinstance(result, list):
+                all_results.extend(result)
+            elif isinstance(result, dict) and "data" in result:
+                data = result["data"]
+                if isinstance(data, list):
+                    all_results.extend(data)
+                else:
+                    all_results.append(data)
+            else:
+                all_results.append(result)
+
+        format_output(
+            all_results,
+            format,
+            include_fields,
+            output,
+            quiet,
+            exclude=excl_fields,
+            compact=compact,
+            append=append,
+        )
+
+    # Show stats
+    if stats and not quiet:
+        _print_stats(tracker.get_stats(), batch_result)
+
+
+def run_search_command(
+    endpoint: str,
+    payload: dict[str, Any],
+    **kwargs: Any,
+) -> None:
+    """Sync wrapper for execute_search_command.
+
+    Catches AnysiteError and exits with proper error message.
+    """
+    import typer
+
+    try:
+        asyncio.run(execute_search_command(endpoint, payload, **kwargs))
+    except AnysiteError as e:
+        print_error(str(e))
+        raise typer.Exit(1) from None
+
+
+def run_single_command(
+    endpoint: str,
+    payload: dict[str, Any],
+    **kwargs: Any,
+) -> None:
+    """Sync wrapper for execute_single_command.
+
+    Catches AnysiteError and exits with proper error message.
+    """
+    import typer
+
+    try:
+        asyncio.run(execute_single_command(endpoint, payload, **kwargs))
+    except AnysiteError as e:
+        print_error(str(e))
+        raise typer.Exit(1) from None
+    except (FileNotFoundError, ValueError) as e:
+        print_error(str(e))
+        raise typer.Exit(1) from None