web2cli 0.2.0__py3-none-any.whl

web2cli/executor/http.py

@@ -0,0 +1,113 @@
+"""HTTP request execution via httpx (default) or curl_cffi (TLS impersonation)."""
+
+import sys
+import time
+
+import httpx
+
+from web2cli.types import Request
+
+
+class HttpError(Exception):
+    def __init__(self, status_code: int, message: str):
+        self.status_code = status_code
+        super().__init__(message)
+
+
+async def _execute_httpx(request: Request) -> tuple[int, dict, str]:
+    """Execute via httpx (standard path)."""
+    content_type = (request.content_type or request.headers.get("Content-Type", "")).lower()
+    body_is_form = (
+        isinstance(request.body, dict)
+        and content_type.startswith("application/x-www-form-urlencoded")
+    )
+
+    try:
+        async with httpx.AsyncClient(follow_redirects=True) as client:
+            response = await client.request(
+                method=request.method,
+                url=request.url,
+                params=request.params or None,
+                headers=request.headers,
+                cookies=request.cookies,
+                content=request.body if isinstance(request.body, (str, bytes)) else None,
+                data=request.body if body_is_form else None,
+                json=request.body if isinstance(request.body, dict) and not body_is_form else None,
+            )
+    except httpx.ConnectError:
+        raise HttpError(0, f"Connection failed: could not reach {request.url}")
+    except httpx.TimeoutException:
+        raise HttpError(0, f"Request timed out: {request.url}")
+
+    return response.status_code, dict(response.headers), response.text
+
+
+async def _execute_impersonate(
+    request: Request, impersonate: str
+) -> tuple[int, dict, str]:
+    """Execute via curl_cffi with TLS impersonation."""
+    from curl_cffi.requests import AsyncSession
+
+    content_type = (request.content_type or request.headers.get("Content-Type", "")).lower()
+    body_is_form = (
+        isinstance(request.body, dict)
+        and content_type.startswith("application/x-www-form-urlencoded")
+    )
+
+    try:
+        async with AsyncSession(impersonate=impersonate) as session:
+            response = await session.request(
+                method=request.method,
+                url=request.url,
+                params=request.params or None,
+                headers=request.headers,
+                cookies=request.cookies,
+                data=request.body if body_is_form else (
+                    request.body if isinstance(request.body, (str, bytes)) else None
+                ),
+                json=request.body if isinstance(request.body, dict) and not body_is_form else None,
+                allow_redirects=True,
+            )
+    except ConnectionError:
+        raise HttpError(0, f"Connection failed: could not reach {request.url}")
+    except TimeoutError:
+        raise HttpError(0, f"Request timed out: {request.url}")
+
+    return response.status_code, dict(response.headers), response.text
+
+
+async def execute(
+    request: Request, verbose: bool = False, impersonate: str | None = None
+) -> tuple[int, dict, str]:
+    """Execute HTTP request. Returns (status_code, headers, body)."""
+    if verbose:
+        sys.stderr.write(f"→ {request.method} {request.url}\n")
+        if request.params:
+            sys.stderr.write(f"  params: {request.params}\n")
+        if impersonate:
+            sys.stderr.write(f"  impersonate: {impersonate}\n")
+
+    start = time.monotonic()
+
+    if impersonate:
+        status, headers, body = await _execute_impersonate(request, impersonate)
+    else:
+        status, headers, body = await _execute_httpx(request)
+
+    elapsed = time.monotonic() - start
+
+    if verbose:
+        sys.stderr.write(f"← {status} ({elapsed:.2f}s)\n")
+
+    if status == 429:
+        retry = headers.get("Retry-After", "?")
+        raise HttpError(429, f"Rate limited. Try again in {retry} seconds.")
+    if status == 403:
+        raise HttpError(
+            403,
+            "Access denied. You may need to login: web2cli login <domain>",
+        )
+    if status >= 500:
+        raise HttpError(status, f"Server error ({status})")
+
+    return status, headers, body

web2cli/output/formatter.py

@@ -0,0 +1,116 @@
+"""Output formatting: table, json, csv, plain, md."""
+
+import csv
+import io
+import json
+
+from rich import box
+from rich.console import Console
+from rich.table import Table
+
+
+def format_output(
+    records: list[dict],
+    fmt: str = "table",
+    fields: list[str] | None = None,
+    no_color: bool = False,
+    no_header: bool = False,
+) -> str:
+    """Format records for stdout.
+
+    Args:
+        records: List of dicts to format.
+        fmt: Output format — table, json, csv, plain.
+        fields: Which fields to include (None = all).
+        no_color: Disable colored output.
+        no_header: Omit header row (csv only).
+    """
+    if not records:
+        return ""
+
+    # Resolve fields
+    if not fields:
+        fields = list(records[0].keys())
+
+    # Filter records to only include requested fields
+    filtered = [{k: r.get(k) for k in fields} for r in records]
+
+    if fmt == "json":
+        return _format_json(filtered)
+    if fmt == "csv":
+        return _format_csv(filtered, fields, no_header=no_header)
+    if fmt == "plain":
+        return _format_plain(filtered, fields)
+    if fmt == "md":
+        return _format_markdown(filtered, fields)
+    return _format_table(filtered, fields, no_color)
+
+
+def _format_table(records: list[dict], fields: list[str], no_color: bool) -> str:
+    """Rich table output."""
+    table_box = box.ASCII2 if no_color else box.HEAVY_HEAD
+    table = Table(
+        show_header=True,
+        header_style=None if no_color else "bold",
+        show_lines=False,
+        pad_edge=False,
+        box=table_box,
+    )
+
+    for field in fields:
+        table.add_column(field.upper())
+
+    for record in records:
+        row = []
+        for field in fields:
+            val = record.get(field)
+            cell = str(val) if val is not None else ""
+            row.append(cell)
+        table.add_row(*row)
+
+    console = Console(no_color=no_color, force_terminal=not no_color)
+    with console.capture() as capture:
+        console.print(table)
+    return capture.get().rstrip()
+
+
+def _format_json(records: list[dict]) -> str:
+    """JSON array output."""
+    return json.dumps(records, indent=2, ensure_ascii=False)
+
+
+def _format_csv(records: list[dict], fields: list[str], no_header: bool = False) -> str:
+    """CSV output."""
+    buf = io.StringIO()
+    writer = csv.DictWriter(buf, fieldnames=fields, extrasaction="ignore")
+    if not no_header:
+        writer.writeheader()
+    writer.writerows(records)
+    return buf.getvalue().rstrip()
+
+
+def _format_markdown(records: list[dict], fields: list[str]) -> str:
+    """Markdown table output."""
+    headers = [f.upper() for f in fields]
+    lines = ["| " + " | ".join(headers) + " |"]
+    lines.append("| " + " | ".join("---" for _ in fields) + " |")
+    for record in records:
+        cells = []
+        for field in fields:
+            val = record.get(field)
+            cell = str(val) if val is not None else ""
+            cell = cell.replace("|", "\\|")
+            cells.append(cell)
+        lines.append("| " + " | ".join(cells) + " |")
+    return "\n".join(lines)
+
+
+def _format_plain(records: list[dict], fields: list[str]) -> str:
+    """Plain output — first field only, one per line. Best for piping."""
+    first_field = fields[0]
+    lines = []
+    for record in records:
+        val = record.get(first_field)
+        if val is not None:
+            lines.append(str(val))
+    return "\n".join(lines)

web2cli/parser/custom.py

@@ -0,0 +1,21 @@
+"""Dynamic import and execution of custom parser scripts."""
+
+import importlib.util
+from pathlib import Path
+
+
+def parse_custom(
+    script_path: str,
+    adapter_dir: Path,
+    status_code: int,
+    headers: dict,
+    body: str,
+    args: dict,
+) -> list[dict]:
+    """Import and call a custom parser script."""
+    full_path = adapter_dir / script_path
+    spec = importlib.util.spec_from_file_location("custom_parser", full_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+
+    return module.parse(status_code, headers, body, args)

web2cli/parser/html_parser.py

@@ -0,0 +1,111 @@
+"""HTML response parser using selectolax."""
+
+import sys
+
+from selectolax.parser import HTMLParser
+
+from web2cli.parser.transforms import apply_transform
+
+# Page titles that indicate bot/CAPTCHA blocking
+_BLOCK_SIGNALS = ("human verification", "captcha", "access denied", "just a moment")
+
+
+def parse_html(body: str, response_spec: dict, disable_truncate: bool = False) -> list[dict]:
+    """Parse HTML response using CSS selectors from spec."""
+    tree = HTMLParser(body)
+
+    # Extract items matching the top-level selector
+    extract_selector = response_spec.get("extract", "body")
+    items = tree.css(extract_selector)
+
+    if not items:
+        # Detect CAPTCHA / bot-blocking pages
+        title_el = tree.css_first("title")
+        if title_el:
+            title = title_el.text(strip=True).lower()
+            if any(s in title for s in _BLOCK_SIGNALS):
+                print(
+                    f"Blocked by site ({title_el.text(strip=True)}). "
+                    "Try again later or use `web2cli login` to add cookies.",
+                    file=sys.stderr,
+                )
+        return []
+
+    fields = response_spec.get("fields", [])
+    if not fields:
+        return [{"text": node.text(strip=True)} for node in items]
+
+    records = []
+    for item in items:
+        record = {}
+        for field_spec in fields:
+            name = field_spec["name"]
+            path = field_spec.get("path", "")
+            attribute = field_spec.get("attribute", "text")
+            collect = field_spec.get("collect", False)
+            join_sep = field_spec.get("join", ", ")
+            prefix = field_spec.get("prefix", "")
+            relative = field_spec.get("relative", "self")
+
+            target = item
+            if relative == "next":
+                target = _next_element(item.next)
+            elif relative == "parent":
+                target = item.parent
+
+            if collect:
+                # Collect multiple matching elements
+                sub_nodes = target.css(path) if target else []
+                values = [_extract_attr(n, attribute) for n in sub_nodes]
+                values = [v for v in values if v]
+                value = join_sep.join(values)
+            else:
+                # Single element
+                node = target.css_first(path) if (target and path) else target
+                value = _extract_attr(node, attribute) if node else None
+
+            # Apply prefix
+            if prefix and value:
+                value = prefix + value
+
+            # Apply transform
+            transform = field_spec.get("transform")
+            if transform:
+                value = apply_transform(value, transform, disable_truncate=disable_truncate)
+
+            # Apply truncation (display hint)
+            truncate = field_spec.get("truncate")
+            if (
+                not disable_truncate
+                and truncate
+                and value
+                and isinstance(value, str)
+                and len(value) > truncate
+            ):
+                value = value[:truncate] + "..."
+
+            # Default fallback
+            if value is None:
+                value = field_spec.get("default")
+
+            record[name] = value
+        records.append(record)
+
+    return records
+
+
+def _extract_attr(node, attribute: str) -> str | None:
+    """Extract an attribute value from a selectolax node."""
+    if node is None:
+        return None
+    if attribute == "text":
+        return node.text(strip=True) or None
+    return node.attributes.get(attribute)
+
+
+def _next_element(node):
+    """Return next element-like node, skipping text/whitespace nodes."""
+    cur = node
+    while cur is not None and not hasattr(cur, "css_first"):
+        cur = cur.next
+    return cur

web2cli/parser/transforms.py

@@ -0,0 +1,127 @@
+"""Shared value transforms used by JSON and HTML parsers."""
+
+import math
+import re
+from datetime import datetime, timezone
+
+
+def apply_transform(value, transform: str, disable_truncate: bool = False):
+    """Apply a named transform to a value."""
+    if value is None:
+        return value
+
+    if transform == "round":
+        try:
+            return round(float(value))
+        except (ValueError, TypeError):
+            return value
+
+    if transform == "int":
+        text = str(value).strip().lower().replace(",", "")
+        if text.endswith("k"):
+            try:
+                return int(float(text[:-1]) * 1000)
+            except (ValueError, TypeError):
+                pass
+        if text.endswith("m"):
+            try:
+                return int(float(text[:-1]) * 1000000)
+            except (ValueError, TypeError):
+                pass
+        try:
+            return int(float(text))
+        except (ValueError, TypeError):
+            m = re.search(r"-?\d[\d,]*", text)
+            if m:
+                try:
+                    return int(m.group(0).replace(",", ""))
+                except ValueError:
+                    return value
+            return value
+
+    if transform == "lowercase":
+        return str(value).lower()
+
+    if transform == "uppercase":
+        return str(value).upper()
+
+    if transform == "strip_html":
+        text = re.sub(r"<[^>]+>", " ", str(value))
+        text = re.sub(r"&lt;", "<", text)
+        text = re.sub(r"&gt;", ">", text)
+        text = re.sub(r"&amp;", "&", text)
+        text = re.sub(r"&quot;", '"', text)
+        text = re.sub(r"&#39;", "'", text)
+        text = re.sub(r"&nbsp;", " ", text)
+        return re.sub(r"\s+", " ", text).strip()
+
+    if transform == "timestamp":
+        return _parse_timestamp(value)
+
+    if transform == "x_datetime":
+        return _parse_twitter_datetime(value)
+
+    if transform == "x_date":
+        full = _parse_twitter_datetime(value)
+        if isinstance(full, str) and len(full) >= 10:
+            return full[:10]
+        return full
+
+    if transform.startswith("truncate:"):
+        if disable_truncate:
+            return value
+        try:
+            n = int(transform.split(":")[1])
+            s = str(value)
+            return s[:n] + "..." if len(s) > n else s
+        except (ValueError, IndexError):
+            return value
+
+    return value
+
+
+def _parse_timestamp(value) -> str:
+    """Convert various timestamp formats to readable string."""
+    # Unix timestamp (int or float)
+    if isinstance(value, (int, float)):
+        if value > 1e12:
+            value = value / 1000  # milliseconds
+        try:
+            dt = datetime.fromtimestamp(value, tz=timezone.utc)
+            return dt.strftime("%Y-%m-%d %H:%M")
+        except (OSError, ValueError):
+            return str(value)
+
+    # ISO string
+    if isinstance(value, str):
+        # numeric string (unix seconds / milliseconds)
+        if re.fullmatch(r"\d+(\.\d+)?", value.strip()):
+            try:
+                num = float(value.strip())
+                if num > 1e12:
+                    num = num / 1000
+                dt = datetime.fromtimestamp(num, tz=timezone.utc)
+                return dt.strftime("%Y-%m-%d %H:%M")
+            except (ValueError, OSError):
+                pass
+
+        for fmt in ("%Y-%m-%dT%H:%M:%S.%fZ", "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S"):
+            try:
+                dt = datetime.strptime(value, fmt).replace(tzinfo=timezone.utc)
+                return dt.strftime("%Y-%m-%d %H:%M")
+            except ValueError:
+                continue
+        return value
+
+    return str(value)
+
+
+def _parse_twitter_datetime(value) -> str:
+    """Convert X/Twitter datetime to readable format."""
+    if not isinstance(value, str):
+        return str(value)
+    try:
+        dt = datetime.strptime(value, "%a %b %d %H:%M:%S %z %Y")
+        return dt.strftime("%Y-%m-%d %H:%M")
+    except ValueError:
+        return value

web2cli/pipe.py

@@ -0,0 +1,10 @@
+"""Stdin detection and reading for piped input."""
+
+import sys
+
+
+def read_stdin() -> str | None:
+    """Read from stdin if data is being piped. Returns None if no pipe."""
+    if not sys.stdin.isatty():
+        return sys.stdin.read().strip()
+    return None

web2cli/providers/__init__.py

@@ -0,0 +1,6 @@
+"""Provider plugin interfaces and built-ins."""
+
+from web2cli.providers.base import Provider
+from web2cli.providers.registry import get_provider
+
+__all__ = ["Provider", "get_provider"]

web2cli/providers/base.py

@@ -0,0 +1,22 @@
+"""Provider base class for request generation."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from web2cli.types import AdapterSpec, Request, Session
+
+
+class Provider:
+    """Provider plugin contract."""
+
+    name: str = ""
+
+    def build_request(
+        self,
+        spec: dict[str, Any],
+        ctx: dict[str, Any],
+        adapter: AdapterSpec,
+        session: Session | None,
+    ) -> Request:
+        raise NotImplementedError

web2cli/providers/registry.py

@@ -0,0 +1,86 @@
+"""Provider registry."""
+
+from __future__ import annotations
+
+import importlib.util
+import re
+from pathlib import Path
+
+from web2cli.providers.base import Provider
+from web2cli.types import AdapterSpec
+
+_PROVIDERS: dict[str, Provider] = {}
+_BUILTINS_REGISTERED = False
+_DYNAMIC_MODULES_LOADED: set[str] = set()
+
+_BUILTIN_ADAPTERS_DIR = Path(__file__).resolve().parent.parent / "adapters"
+_USER_ADAPTERS_DIR = Path.home() / ".web2cli" / "adapters"
+
+
+def register_provider(provider: Provider) -> None:
+    if not provider.name:
+        raise ValueError("Provider must have a name")
+    _PROVIDERS[provider.name] = provider
+
+
+def _register_builtins_once() -> None:
+    global _BUILTINS_REGISTERED
+    if _BUILTINS_REGISTERED:
+        return
+
+    _BUILTINS_REGISTERED = True
+
+
+def _safe_ident(value: str) -> str:
+    return re.sub(r"[^A-Za-z0-9_]+", "_", value)
+
+
+def _load_provider_module(module_path: Path, key: str) -> None:
+    if key in _DYNAMIC_MODULES_LOADED:
+        return
+    if not module_path.is_file():
+        return
+
+    module_name = f"web2cli_dynamic_provider_{_safe_ident(key)}"
+    spec = importlib.util.spec_from_file_location(module_name, module_path)
+    if spec is None or spec.loader is None:
+        return
+
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    _DYNAMIC_MODULES_LOADED.add(key)
+
+
+def _load_from_adapter(adapter: AdapterSpec, provider_name: str) -> None:
+    if adapter.adapter_dir is None:
+        return
+    provider_path = adapter.adapter_dir / "providers" / f"{provider_name}.py"
+    key = f"{adapter.meta.domain}:{provider_name}:{provider_path}"
+    _load_provider_module(provider_path, key)
+
+
+def _load_from_known_adapter_dirs(provider_name: str) -> None:
+    for base in (_BUILTIN_ADAPTERS_DIR, _USER_ADAPTERS_DIR):
+        if not base.is_dir():
+            continue
+        for adapter_dir in base.iterdir():
+            provider_path = adapter_dir / "providers" / f"{provider_name}.py"
+            key = f"{adapter_dir}:{provider_name}:{provider_path}"
+            _load_provider_module(provider_path, key)
+
+
+def get_provider(name: str, adapter: AdapterSpec | None = None) -> Provider:
+    _register_builtins_once()
+    provider = _PROVIDERS.get(name)
+
+    if provider is None and adapter is not None:
+        _load_from_adapter(adapter, name)
+        provider = _PROVIDERS.get(name)
+
+    if provider is None:
+        _load_from_known_adapter_dirs(name)
+        provider = _PROVIDERS.get(name)
+
+    if provider is None:
+        raise ValueError(f"Unknown provider: {name}")
+    return provider

web2cli/runtime/__init__.py

@@ -0,0 +1 @@
+"""web2cli runtime modules."""

web2cli/runtime/cache.py

@@ -0,0 +1,42 @@
+"""Small file cache for runtime resources."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from pathlib import Path
+from typing import Any
+
+BASE_CACHE_DIR = Path.home() / ".web2cli" / "cache"
+
+
+def _cache_path(domain: str, key: str) -> Path:
+    digest = hashlib.sha1(key.encode()).hexdigest()  # nosec: non-crypto use
+    return BASE_CACHE_DIR / domain / "runtime" / f"{digest}.json"
+
+
+def load_cache(domain: str, key: str, ttl: int | None = None) -> Any | None:
+    """Load cached payload if present and not expired."""
+    path = _cache_path(domain, key)
+    if not path.is_file():
+        return None
+    try:
+        data = json.loads(path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return None
+
+    ts = data.get("ts")
+    if ttl and isinstance(ts, (int, float)):
+        if time.time() - ts > ttl:
+            return None
+
+    return data.get("payload")
+
+
+def save_cache(domain: str, key: str, payload: Any) -> None:
+    """Persist payload in cache."""
+    path = _cache_path(domain, key)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    doc = {"ts": time.time(), "payload": payload}
+    path.write_text(json.dumps(doc))