anysite-cli 0.1.0__py3-none-any.whl

anysite/output/formatters.py

@@ -0,0 +1,301 @@
+"""Output formatters for different formats."""
+
+import csv
+import io
+from enum import Enum
+from pathlib import Path
+from typing import Any
+
+import orjson
+from rich.table import Table
+
+from anysite.output.console import console
+
+
+class OutputFormat(str, Enum):
+    """Supported output formats."""
+
+    JSON = "json"
+    JSONL = "jsonl"
+    CSV = "csv"
+    TABLE = "table"
+
+
+def filter_fields(data: dict[str, Any], fields: list[str]) -> dict[str, Any]:
+    """Filter dictionary to only include specified fields.
+
+    Supports nested fields with dot notation (e.g., 'experience.company').
+
+    Args:
+        data: Source dictionary
+        fields: List of field names to include
+
+    Returns:
+        Filtered dictionary
+    """
+    if not fields:
+        return data
+
+    result: dict[str, Any] = {}
+    for field in fields:
+        if "." in field:
+            # Handle nested fields
+            parts = field.split(".")
+            value = data
+            for part in parts:
+                if isinstance(value, dict):
+                    value = value.get(part)
+                elif isinstance(value, list) and part.isdigit():
+                    idx = int(part)
+                    value = value[idx] if idx < len(value) else None
+                else:
+                    value = None
+                    break
+            if value is not None:
+                # Store in nested structure
+                current = result
+                for part in parts[:-1]:
+                    if part not in current:
+                        current[part] = {}
+                    current = current[part]
+                current[parts[-1]] = value
+        elif field in data:
+            result[field] = data[field]
+
+    return result
+
+
+def flatten_for_csv(data: dict[str, Any], prefix: str = "") -> dict[str, Any]:
+    """Flatten nested dictionary for CSV output.
+
+    Args:
+        data: Nested dictionary
+        prefix: Prefix for nested keys
+
+    Returns:
+        Flattened dictionary with dot-notation keys
+    """
+    result: dict[str, Any] = {}
+
+    for key, value in data.items():
+        full_key = f"{prefix}.{key}" if prefix else key
+
+        if isinstance(value, dict):
+            result.update(flatten_for_csv(value, full_key))
+        elif isinstance(value, list):
+            if all(isinstance(item, (str, int, float, bool, type(None))) for item in value):
+                # Simple list - join as string
+                result[full_key] = "; ".join(str(v) for v in value if v is not None)
+            else:
+                # Complex list - take length or first few items
+                result[f"{full_key}_count"] = len(value)
+                for i, item in enumerate(value[:3]):  # Max 3 items
+                    if isinstance(item, dict):
+                        result.update(flatten_for_csv(item, f"{full_key}_{i}"))
+                    else:
+                        result[f"{full_key}_{i}"] = item
+        else:
+            result[full_key] = value
+
+    return result
+
+
+def format_json(data: Any, indent: bool = True) -> str:
+    """Format data as JSON.
+
+    Args:
+        data: Data to format
+        indent: Whether to indent (pretty print)
+
+    Returns:
+        JSON string
+    """
+    option = orjson.OPT_INDENT_2 if indent else 0
+    return orjson.dumps(data, option=option).decode("utf-8")
+
+
+def format_jsonl(data: list[dict[str, Any]]) -> str:
+    """Format data as newline-delimited JSON (JSONL).
+
+    Args:
+        data: List of dictionaries
+
+    Returns:
+        JSONL string with one JSON object per line
+    """
+    lines = [orjson.dumps(item).decode("utf-8") for item in data]
+    return "\n".join(lines)
+
+
+def format_csv_output(data: list[dict[str, Any]], fields: list[str] | None = None) -> str:
+    """Format data as CSV.
+
+    Args:
+        data: List of dictionaries
+        fields: Optional list of fields to include
+
+    Returns:
+        CSV string with headers
+    """
+    if not data:
+        return ""
+
+    # Flatten all records
+    flattened = [flatten_for_csv(item) for item in data]
+
+    # Get all unique keys for headers
+    if fields:
+        headers = fields
+    else:
+        headers = list(dict.fromkeys(key for item in flattened for key in item.keys()))
+
+    # Write CSV
+    output = io.StringIO()
+    writer = csv.DictWriter(output, fieldnames=headers, extrasaction="ignore")
+    writer.writeheader()
+    writer.writerows(flattened)
+    return output.getvalue()
+
+
+def format_table_output(data: list[dict[str, Any]], fields: list[str] | None = None) -> None:
+    """Format data as a Rich table and print to console.
+
+    Args:
+        data: List of dictionaries
+        fields: Optional list of fields to include
+    """
+    if not data:
+        console.print("[dim]No results[/dim]")
+        return
+
+    # For single item, display vertically
+    if len(data) == 1:
+        table = Table(show_header=True, header_style="bold")
+        table.add_column("Field", style="cyan")
+        table.add_column("Value")
+
+        item = data[0]
+        if fields:
+            item = filter_fields(item, fields)
+
+        for key, value in item.items():
+            if isinstance(value, (dict, list)):
+                value = format_json(value, indent=False)
+            table.add_row(key, str(value) if value is not None else "[dim]null[/dim]")
+
+        console.print(table)
+        return
+
+    # For multiple items, display as grid
+    # Flatten for tabular display
+    flattened = [flatten_for_csv(item) for item in data]
+
+    if fields:
+        headers = fields
+    else:
+        # Select most important fields (limit columns for readability)
+        all_keys = list(dict.fromkeys(key for item in flattened for key in item.keys()))
+        # Prioritize common fields
+        priority_fields = ["name", "full_name", "headline", "title", "company", "url", "followers"]
+        headers = [f for f in priority_fields if f in all_keys]
+        headers.extend([f for f in all_keys if f not in headers][:10 - len(headers)])
+
+    # Create table
+    table = Table(show_header=True, header_style="bold")
+    for header in headers:
+        table.add_column(header)
+
+    for item in flattened:
+        row = []
+        for header in headers:
+            value = item.get(header, "")
+            if isinstance(value, (dict, list)):
+                value = "..."
+            row.append(str(value) if value is not None else "")
+        table.add_row(*row)
+
+    console.print(table)
+
+
+def format_output(
+    data: Any,
+    output_format: OutputFormat,
+    fields: list[str] | None = None,
+    output_file: Path | None = None,
+    quiet: bool = False,
+    exclude: list[str] | None = None,
+    compact: bool = False,
+    append: bool = False,
+) -> None:
+    """Format and output data in the specified format.
+
+    Args:
+        data: Data to format (usually list of dicts from API)
+        output_format: Output format (json, jsonl, csv, table)
+        fields: Optional list of fields to include
+        output_file: Optional file path to write output
+        quiet: Suppress non-data output
+        exclude: Optional list of fields to exclude
+        compact: Use compact output (no indentation)
+        append: Append to existing file
+    """
+    # Ensure data is a list
+    if not isinstance(data, list):
+        data = [data]
+
+    # Filter fields if specified
+    if fields:
+        data = [filter_fields(item, fields) for item in data]
+
+    # Exclude fields if specified
+    if exclude:
+        from anysite.utils.fields import exclude_fields
+        data = [exclude_fields(item, exclude) for item in data]
+
+    # Format based on type
+    if output_format == OutputFormat.TABLE:
+        if output_file:
+            # Table can't be written to file, fall back to JSON
+            formatted = format_json(data, indent=not compact)
+            _write_output(formatted, output_file, quiet, append=append)
+        else:
+            format_table_output(data, fields)
+        return
+
+    if output_format == OutputFormat.JSON:
+        formatted = format_json(data, indent=not compact)
+    elif output_format == OutputFormat.JSONL:
+        formatted = format_jsonl(data)
+    elif output_format == OutputFormat.CSV:
+        formatted = format_csv_output(data, fields)
+    else:
+        formatted = format_json(data, indent=not compact)
+
+    _write_output(formatted, output_file, quiet, append=append)
+
+
+def _write_output(
+    content: str,
+    output_file: Path | None,
+    quiet: bool,
+    append: bool = False,
+) -> None:
+    """Write content to file or stdout.
+
+    Args:
+        content: Content to write
+        output_file: Optional file path
+        quiet: Suppress messages
+        append: Append to existing file
+    """
+    if output_file:
+        mode = "a" if append else "w"
+        with open(output_file, mode, encoding="utf-8") as f:
+            f.write(content)
+        if not quiet:
+            from anysite.output.console import print_success
+
+            action = "appended to" if append else "saved to"
+            print_success(f"Output {action} {output_file}")
+    else:
+        print(content)

anysite/output/templates.py

@@ -0,0 +1,76 @@
+"""Filename template resolution for batch output."""
+
+from datetime import datetime
+from typing import Any
+
+
+class FilenameTemplate:
+    """Resolve filename templates with variable substitution.
+
+    Supported variables:
+        {id}        - Record ID or input value
+        {username}  - Username field from record
+        {date}      - Current date (YYYY-MM-DD)
+        {datetime}  - Current date and time (YYYY-MM-DD_HH-MM-SS)
+        {timestamp} - Unix timestamp
+        {index}     - Zero-padded index
+    """
+
+    def __init__(self, template: str, extension: str = ".json") -> None:
+        """Initialize template.
+
+        Args:
+            template: Template string with {variable} placeholders
+            extension: File extension to append
+        """
+        self.template = template
+        self.extension = extension
+
+    def resolve(
+        self,
+        record: dict[str, Any] | None = None,
+        index: int = 0,
+        input_value: str = "",
+    ) -> str:
+        """Resolve template variables to an actual filename.
+
+        Args:
+            record: Data record (for extracting fields)
+            index: Item index in batch
+            input_value: Original input value
+
+        Returns:
+            Resolved filename string with extension
+        """
+        now = datetime.now()
+        record = record or {}
+
+        variables = {
+            "id": input_value or record.get("id", record.get("urn", str(index))),
+            "username": record.get("username", record.get("user", input_value)),
+            "date": now.strftime("%Y-%m-%d"),
+            "datetime": now.strftime("%Y-%m-%d_%H-%M-%S"),
+            "timestamp": str(int(now.timestamp())),
+            "index": f"{index:04d}",
+        }
+
+        filename = self.template
+        for key, value in variables.items():
+            filename = filename.replace(f"{{{key}}}", str(value))
+
+        # Sanitize filename
+        filename = self._sanitize(filename)
+
+        # Add extension if not present
+        if not any(filename.endswith(ext) for ext in [".json", ".jsonl", ".csv"]):
+            filename += self.extension
+
+        return filename
+
+    @staticmethod
+    def _sanitize(filename: str) -> str:
+        """Remove or replace unsafe characters from filename."""
+        unsafe = '<>:"/\\|?*'
+        for char in unsafe:
+            filename = filename.replace(char, "_")
+        return filename.strip(". ")

anysite/streaming/__init__.py

@@ -0,0 +1 @@
+"""Streaming output modules."""

anysite/streaming/progress.py

@@ -0,0 +1,121 @@
+"""Progress tracking with Rich progress bars."""
+
+import sys
+import time
+
+from rich.progress import (
+    BarColumn,
+    MofNCompleteColumn,
+    Progress,
+    SpinnerColumn,
+    TextColumn,
+    TimeElapsedColumn,
+    TimeRemainingColumn,
+)
+
+
+class ProgressTracker:
+    """Rich progress bar wrapper for tracking long operations.
+
+    Automatically shows/hides based on terminal detection and quiet mode.
+    """
+
+    def __init__(
+        self,
+        total: int | None = None,
+        description: str = "Processing...",
+        show: bool | None = None,
+        quiet: bool = False,
+    ) -> None:
+        """Initialize progress tracker.
+
+        Args:
+            total: Total number of items (None for indeterminate)
+            description: Description text
+            show: Force show/hide (None = auto-detect)
+            quiet: Suppress all output
+        """
+        self.total = total
+        self.description = description
+        self.quiet = quiet
+        self._completed = 0
+        self._start_time = time.monotonic()
+        self._progress: Progress | None = None
+
+        # Determine if we should show progress
+        if quiet:
+            self._should_show = False
+        elif show is not None:
+            self._should_show = show
+        else:
+            # Auto: show if stdout is a terminal
+            self._should_show = sys.stderr.isatty()
+
+    def start(self) -> None:
+        """Start the progress bar."""
+        if not self._should_show:
+            return
+
+        columns = [
+            SpinnerColumn(),
+            TextColumn("[bold blue]{task.description}"),
+            BarColumn(),
+            MofNCompleteColumn(),
+            TimeElapsedColumn(),
+            TimeRemainingColumn(),
+        ]
+
+        self._progress = Progress(*columns, transient=True)
+        self._progress.start()
+        self._task_id = self._progress.add_task(
+            self.description,
+            total=self.total,
+        )
+
+    def update(self, n: int = 1) -> None:
+        """Update progress by n items.
+
+        Args:
+            n: Number of items completed
+        """
+        self._completed += n
+        if self._progress is not None:
+            self._progress.update(self._task_id, advance=n)
+
+    def set_status(self, text: str) -> None:
+        """Update the description text.
+
+        Args:
+            text: New description
+        """
+        self.description = text
+        if self._progress is not None:
+            self._progress.update(self._task_id, description=text)
+
+    def finish(self) -> None:
+        """Complete the progress bar."""
+        if self._progress is not None:
+            self._progress.stop()
+            self._progress = None
+
+    def get_stats(self) -> dict[str, float | int]:
+        """Get execution statistics.
+
+        Returns:
+            Dictionary with timing and throughput stats
+        """
+        elapsed = time.monotonic() - self._start_time
+        rate = self._completed / elapsed if elapsed > 0 else 0
+
+        return {
+            "total": self._completed,
+            "elapsed_seconds": round(elapsed, 2),
+            "records_per_second": round(rate, 1),
+        }
+
+    def __enter__(self) -> "ProgressTracker":
+        self.start()
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.finish()

anysite/streaming/writer.py

@@ -0,0 +1,130 @@
+"""Streaming record writer for outputting records one at a time."""
+
+import sys
+from pathlib import Path
+from typing import IO, Any
+
+import orjson
+
+from anysite.output.formatters import OutputFormat
+from anysite.utils.fields import exclude_fields, filter_fields
+
+
+class StreamingWriter:
+    """Write records one at a time to stdout or a file.
+
+    Supports JSONL (primary) and CSV streaming output.
+    """
+
+    def __init__(
+        self,
+        output: Path | None = None,
+        format: OutputFormat = OutputFormat.JSONL,
+        fields: list[str] | None = None,
+        exclude: list[str] | None = None,
+        compact: bool = False,
+        append: bool = False,
+    ) -> None:
+        """Initialize streaming writer.
+
+        Args:
+            output: Output file path (None = stdout)
+            format: Output format (JSONL or CSV)
+            fields: Fields to include
+            exclude: Fields to exclude
+            compact: Compact JSON output
+            append: Append to existing file
+        """
+        self.output = output
+        self.format = format
+        self.fields = fields
+        self.exclude = exclude
+        self.compact = compact
+        self.append = append
+        self._file: IO[str] | None = None
+        self._csv_headers_written = False
+        self._count = 0
+
+    def _get_writer(self) -> IO[str]:
+        """Get or create the output writer."""
+        if self._file is not None:
+            return self._file
+
+        if self.output:
+            mode = "a" if self.append else "w"
+            self._file = open(self.output, mode, encoding="utf-8")  # noqa: SIM115
+            return self._file
+        else:
+            return sys.stdout
+
+    def _process_record(self, record: dict[str, Any]) -> dict[str, Any]:
+        """Apply field filtering to a record."""
+        if self.fields:
+            record = filter_fields(record, self.fields)
+        if self.exclude:
+            record = exclude_fields(record, self.exclude)
+        return record
+
+    def write(self, record: dict[str, Any]) -> None:
+        """Write a single record.
+
+        Args:
+            record: Dictionary to write
+        """
+        record = self._process_record(record)
+        writer = self._get_writer()
+
+        if self.format == OutputFormat.JSONL:
+            line = orjson.dumps(record).decode("utf-8")
+            writer.write(line + "\n")
+            writer.flush()
+
+        elif self.format == OutputFormat.CSV:
+            import csv
+            import io
+
+            if not self._csv_headers_written:
+                # Flatten and write header
+                from anysite.output.formatters import flatten_for_csv
+                flat = flatten_for_csv(record)
+                self._csv_fieldnames = list(flat.keys())
+                output = io.StringIO()
+                csv_writer = csv.DictWriter(output, fieldnames=self._csv_fieldnames, extrasaction="ignore")
+                csv_writer.writeheader()
+                csv_writer.writerow(flat)
+                writer.write(output.getvalue())
+                self._csv_headers_written = True
+            else:
+                from anysite.output.formatters import flatten_for_csv
+                flat = flatten_for_csv(record)
+                output = io.StringIO()
+                csv_writer = csv.DictWriter(output, fieldnames=self._csv_fieldnames, extrasaction="ignore")
+                csv_writer.writerow(flat)
+                writer.write(output.getvalue())
+
+            writer.flush()
+
+        else:
+            # Default to JSONL for streaming
+            line = orjson.dumps(record).decode("utf-8")
+            writer.write(line + "\n")
+            writer.flush()
+
+        self._count += 1
+
+    def close(self) -> None:
+        """Close the output file if opened."""
+        if self._file is not None:
+            self._file.close()
+            self._file = None
+
+    @property
+    def count(self) -> int:
+        """Number of records written."""
+        return self._count
+
+    def __enter__(self) -> "StreamingWriter":
+        return self
+
+    def __exit__(self, *args: object) -> None:
+        self.close()

anysite/utils/__init__.py

@@ -0,0 +1 @@
+"""Utility modules."""