proxy-scraper-cli 1.7.1__py3-none-any.whl

proxyscraper/ui/serve.py

@@ -0,0 +1,142 @@
+"""Live view of the rotating proxy server (--serve)."""
+
+from __future__ import annotations
+
+import os
+import time
+
+from rich import box
+from rich.align import Align
+from rich.console import Group
+from rich.panel import Panel
+from rich.table import Table
+from rich.text import Text
+
+from ..server import RotatingServer
+from . import widgets
+from .widgets import (
+    ACCENT,
+    BAD,
+    GOOD,
+    MUTED,
+    TYPE_STYLE,
+    WARN,
+    card,
+    fmt,
+    fmt_duration,
+    pct,
+    row,
+    shown_proxy,
+    table,
+)
+
+
+def _mb(n: int) -> str:
+    return f"{n / 2**20:.1f} MB"
+
+
+class ServeDashboard:
+    def __init__(self, server: RotatingServer):
+        self.server = server
+
+    def __rich__(self):
+        server, st, pool = self.server, self.server.stats, self.server.pool
+        # if it listens on all addresses (Docker), it can still be reached locally via 127.0.0.1
+        shown_host = {"0.0.0.0": "127.0.0.1", "": "127.0.0.1", "::": "[::1]"}.get(server.host, server.host)
+        if ":" in shown_host and not shown_host.startswith("["):
+            shown_host = f"[{shown_host}]"  # IPv6 in URLs only in brackets
+        address = f"{shown_host}:{server.port}"
+        width = widgets.console.size.width
+        uptime = max(time.perf_counter() - st.started, 1e-6)
+        usable = pool.usable
+
+        title = Table.grid(expand=True)
+        title.add_column()
+        title.add_column(justify="right")
+        title.add_row(
+            Text.assemble(("● ", f"bold {GOOD}"), ("Proxy server running on ", "bold"), (address, f"bold {ACCENT}")),
+            Text(f"⏱ {fmt_duration(uptime)}", style=MUTED),
+        )
+
+        usage = Table.grid(padding=(0, 2))
+        usage.add_column(style=MUTED, no_wrap=True)
+        usage.add_column(overflow="fold")
+        # with a password every example needs the login; the password itself is never shown
+        from_env = server.password and os.environ.get("PROXY_SCRAPER_SERVE_PASSWORD") == server.password
+        pw = "$PROXY_SCRAPER_SERVE_PASSWORD" if from_env else "PASSWORD" if server.password else "x"
+        login = f"any:{pw}@" if server.password else ""
+        usage.add_row("Test", Text(f"curl -x http://{login}{address} https://api.ipify.org", style="bold"))
+        usage.add_row("SOCKS5", Text(f"curl -x socks5h://{login}{address} https://api.ipify.org"))
+        usage.add_row("One country", Text(f"curl -x http://country-de:{pw}@{address} https://api.ipify.org"))
+        usage.add_row("Fixed session", Text(f"curl -x http://session-abc:{pw}@{address} …  (keeps the same proxy)"))
+        usage.add_row("Terminal", Text(f"export http_proxy=http://{login}{address} https_proxy=http://{login}{address}"))
+        usage.add_row("Status (JSON)", Text(f"curl {'-u any:' + pw + ' ' if server.password else ''}"
+                                            f"http://{address}/__proxy-scraper/status"))
+        usage.add_row("Prometheus", Text(f"http://{'any:' + pw + '@' if server.password else ''}"
+                                         f"{address}/__proxy-scraper/metrics"))
+        mode = pool.strategy + (f" · sticky {pool.sticky_seconds:g} s" if pool.sticky_seconds else "")
+        if server.revived:
+            mode += f" · {fmt(server.revived)} brought back"
+        if server.last_refill:
+            ago = fmt_duration(time.time() - server.last_refill)
+            mode += f" · {fmt(server.refilled)} added by refills (last {ago} ago)"
+        usage.add_row("Rotation", Text(mode, style=MUTED))
+
+        cards = row(
+            card("Requests", fmt(st.requests), f"{st.requests / uptime * 60:.1f} per minute"),
+            card("Successful", pct(st.ok, st.requests) if st.requests else "–",
+                 f"{fmt(st.failed)} failed", f"bold {GOOD}" if not st.failed else f"bold {WARN}"),
+            card("Active", fmt(st.active), "open connections", f"bold {ACCENT}"),
+            card("Pool", f"{fmt(len(usable))} / {fmt(len(pool.entries))}",
+                 f"{fmt(len(pool.tls_capable))} for HTTPS · {fmt(len(pool.entries) - len(usable))} out"),
+            card("Traffic", _mb(st.bytes_down), f"↑ {_mb(st.bytes_up)}"),
+        )
+
+        recent = table()
+        recent.add_column("", width=1)
+        recent.add_column("Target", ratio=2, no_wrap=True, overflow="ellipsis")
+        recent.add_column("via proxy", ratio=2, no_wrap=True, overflow="ellipsis")
+        if width >= 100:
+            recent.add_column("Client", style=MUTED, no_wrap=True)
+        recent.add_column("Attempts", justify="right", width=8)
+        recent.add_column("Time", justify="right", width=8)
+        for log in reversed(st.recent):
+            ptype = log.via.split("://", 1)[0]
+            cells = [
+                Text("✔", style=GOOD) if log.ok else Text("✘", style=BAD),
+                log.target,
+                Text(shown_via(log.via), style=TYPE_STYLE.get(ptype, MUTED)),
+            ]
+            if width >= 100:
+                cells.append(log.client)
+            cells += [str(log.attempts), f"{fmt(log.ms)} ms"]
+            recent.add_row(*cells)
+
+        busiest = table()
+        busiest.add_column("Proxy", no_wrap=True, overflow="ellipsis")
+        busiest.add_column("OK", justify="right", style=GOOD)
+        busiest.add_column("Errors", justify="right", style=MUTED)
+        busiest.add_column("Latency", justify="right")
+        for entry in sorted(pool.entries, key=lambda e: (-e.ok, e.result.latency))[:5]:
+            r = entry.result
+            style = MUTED if entry.disabled else TYPE_STYLE.get(r.ptype, "")
+            name = Text(f"{r.ptype}://{shown_proxy(r.proxy)}", style=style)
+            busiest.add_row(name, fmt(entry.ok), fmt(entry.fail), f"{fmt(r.latency)} ms")
+
+        waiting = Align.center(Text("waiting for the first connection …", style=MUTED))
+        return Group(
+            Panel(Group(title, Text(""), usage), box=box.HEAVY, border_style=ACCENT, padding=(0, 1)),
+            cards,
+            row(
+                widgets.panel(recent if st.recent else waiting, "Recent connections", GOOD),
+                widgets.panel(busiest, "Most used proxies"),
+                ratios=(3, 2),
+            ),
+            Text("  Ctrl+C stops the server", style=MUTED),
+        )
+
+
+def shown_via(via: str) -> str:
+    """'socks5://user:pass@1.2.3.4:1080' -> password masked; '–' (no proxy) stays."""
+    scheme, sep, proxy = via.partition("://")
+    return f"{scheme}://{shown_proxy(proxy)}" if sep else via

proxyscraper/ui/widgets.py

@@ -0,0 +1,250 @@
+"""Colors, formatting and reusable building blocks for the terminal UI."""
+
+from __future__ import annotations
+
+import time
+from datetime import datetime
+from typing import Optional, Sequence
+from urllib.parse import unquote
+
+from rich import box
+from rich.align import Align
+from rich.console import Console, Group
+from rich.panel import Panel
+from rich.progress import ProgressColumn
+from rich.table import Table
+from rich.text import Text
+
+from .. import __version__
+from ..geo import flag
+
+console = Console(highlight=False)
+
+# One palette for everything (the lime is the brand color of the logo and the website). rich
+# downsamples it automatically on terminals with fewer colors.
+ACCENT = "#D4F77A"
+GOOD = "#34D399"
+WARN = "#FBBF24"
+BAD = "#F87171"
+MUTED = "#8B949E"
+BORDER = "#30363D"
+TYPE_STYLE = {"http": "#60A5FA", "socks4": "#C084FC", "socks5": "#34D399"}
+ANON_STYLE = {"elite": ("E", GOOD), "anonymous": ("A", WARN), "transparent": ("T", BAD)}
+ANON_LABEL = {"elite": "Elite", "anonymous": "Anonymous", "transparent": "Transp."}
+PHASES = ("Sources", "Collect", "Check", "Done")
+SPARK = "▁▂▃▄▅▆▇█"
+LATENCY_EDGES = (300, 700, 1500, 3000, 6000)
+LATENCY_LABELS = ("< 0.3 s", "< 0.7 s", "< 1.5 s", "< 3 s", "< 6 s", "≥ 6 s")
+# below a 0.2 % hit rate the network is probably blocking proxy connections (firewall)
+BLOCKED_HIT_RATE = 0.002
+
+
+# --------------------------------------------------------------------------- #
+# Formatting
+# --------------------------------------------------------------------------- #
+
+def fmt(n: float) -> str:
+    """Thousands separators: 12345 -> 12,345"""
+    return f"{n:,.0f}"
+
+
+def pct(part: float, whole: float) -> str:
+    return f"{part / whole * 100:.1f}%" if whole else "–"
+
+
+def fmt_duration(seconds: float) -> str:
+    seconds = int(seconds)
+    if seconds < 60:
+        return f"{seconds} s"
+    m, s = divmod(seconds, 60)
+    if m < 60:
+        return f"{m} min {s:02d} s"
+    h, m = divmod(m, 60)
+    return f"{h} h {m:02d} min"
+
+
+def latency_style(ms: int) -> str:
+    return GOOD if ms < 1000 else WARN if ms < 3000 else BAD
+
+
+def bar(value: float, maximum: float, width: int, style: str) -> Text:
+    """Bar with eighth steps for fine resolution in little space."""
+    width = max(width, 1)
+    filled = 0 if maximum <= 0 else min(value / maximum, 1.0) * width
+    full = int(filled)
+    rest = int((filled - full) * 8)
+    text = Text("█" * full, style=style)
+    if rest and full < width:
+        text.append(" ▏▎▍▌▋▊▉"[rest], style=style)
+        full += 1
+    text.append("·" * (width - full), style=MUTED)
+    return text
+
+
+def sparkline(values: Sequence[float], width: int) -> Text:
+    values = list(values)[-width:]
+    if not values:
+        return Text("")
+    hi = max(values) or 1
+    return Text("".join(SPARK[min(int(v / hi * (len(SPARK) - 1) + 0.5), len(SPARK) - 1)] for v in values), style=ACCENT)
+
+
+def type_badge(ptype: str) -> Text:
+    return Text(ptype, style=f"bold {TYPE_STYLE.get(ptype, 'white')}")
+
+
+def country_cell(cc: str) -> Text:
+    return Text(f"{flag(cc)} {cc}" if cc else "  ··", style="" if cc else MUTED)
+
+
+def https_cell(value: Optional[bool]) -> Text:
+    if value is None:
+        return Text("·", style=MUTED)
+    return Text("✔", style=GOOD) if value else Text("✘", style=BAD)
+
+
+def anon_cell(level: str) -> Text:
+    letter, style = ANON_STYLE.get(level, ("·", MUTED))
+    return Text(letter, style=f"bold {style}" if level else style)
+
+
+def short_url(url: str) -> str:
+    gh = "https://raw.githubusercontent.com/"
+    if url.startswith(gh):
+        return "gh:" + url[len(gh):]
+    return url.split("://", 1)[-1]
+
+
+class CountColumn(ProgressColumn):
+    """12,345 / 1,000,000 instead of 12345/1000000."""
+
+    def render(self, task) -> Text:
+        return Text.assemble((fmt(task.completed), "bold"), (f" / {fmt(task.total or 0)}", MUTED))
+
+
+# --------------------------------------------------------------------------- #
+# Building blocks
+# --------------------------------------------------------------------------- #
+
+def banner() -> Panel:
+    grid = Table.grid(expand=True)
+    grid.add_column()
+    grid.add_column(justify="right")
+    grid.add_row(
+        Text.assemble(("◆ proxy-scraper", f"bold {ACCENT}"), (f"  v{__version__}", MUTED)),
+        Text(datetime.now().strftime("%Y-%m-%d  %H:%M"), style=MUTED),
+    )
+    grid.add_row(Text("Free proxies that actually work – collected, checked, learned.", style=MUTED), "")
+    return Panel(grid, box=box.ROUNDED, border_style=ACCENT, padding=(0, 2))
+
+
+def phase_bar(current: int) -> Text:
+    text = Text()
+    for i, name in enumerate(PHASES):
+        if i:
+            text.append("  ─  ", style=GOOD if i <= current else BORDER)
+        if i < current:
+            text.append(f"✔ {name}", style=GOOD)
+        elif i == current:
+            text.append(f"● {i + 1} {name}", style=f"bold {ACCENT}")
+        else:
+            text.append(f"○ {i + 1} {name}", style=MUTED)
+    return text
+
+
+def header(current: int, started: float) -> Table:
+    grid = Table.grid(expand=True, padding=(0, 1))
+    grid.add_column()
+    grid.add_column(justify="right")
+    grid.add_row(phase_bar(current), Text(f"⏱ {fmt_duration(time.perf_counter() - started)}", style=MUTED))
+    return grid
+
+
+_section_open = False
+
+
+def section(title: str) -> None:
+    """Starts a section; the following info()/note() lines hang off its left border line."""
+    global _section_open
+    if _section_open:
+        section_end()
+    line = Text()
+    line.append("  ╭─ ", style=BORDER)
+    line.append(title, style=f"bold {ACCENT}")
+    line.append(" ", style=BORDER)
+    line.append("─" * max(console.size.width - line.cell_len - 2, 4), style=BORDER)
+    console.print(line)
+    _section_open = True
+
+
+def section_end() -> None:
+    global _section_open
+    if _section_open:
+        console.print(Text("  ╰─", style=BORDER))
+        _section_open = False
+
+
+def _gutter() -> Text:
+    # color only for the border character – not as the base style, which would bleed into the whole line
+    gutter = Text()
+    gutter.append("  │  " if _section_open else "  ", style=BORDER)
+    return gutter
+
+
+def info(label: str, value, style: str = "") -> None:
+    """Uniform info line – with a border line inside a section, indented otherwise."""
+    line = _gutter()
+    line.append(f"{label:<14}", style=MUTED)
+    line.append_text(value if isinstance(value, Text) else Text(str(value), style=style or "bold"))
+    console.print(line)
+
+
+def note(message: str, style: str = WARN, icon: str = "⚠") -> None:
+    console.print(_gutter() + Text(f"{icon} ", style=style) + Text.from_markup(message))
+
+
+def card(label: str, value: str, sub, style: str = "bold") -> Panel:
+    # all cards in a row should stay the same height – long subtitles are shortened instead of wrapped
+    sub = sub if isinstance(sub, Text) else Text(sub, style=MUTED)
+    sub.no_wrap, sub.overflow = True, "ellipsis"
+    body = Group(Text(label.upper(), style=f"bold {MUTED}"), Text(value, style=style), sub)
+    return Panel(body, box=box.ROUNDED, border_style=BORDER, padding=(0, 1))
+
+
+def table(**kwargs) -> Table:
+    """Uniform, airy table without border lines (the frame comes from the panel)."""
+    kwargs.setdefault("expand", True)
+    return Table(box=None, header_style=f"bold {MUTED}", pad_edge=False, **kwargs)
+
+
+def panel_title(title: str, style: str = BORDER) -> Text:
+    """Keep titles readable: with a subtle frame, muted-light instead of the dark frame color."""
+    return Text(f" {title} ", style=f"bold {MUTED if style == BORDER else style}")
+
+
+def panel(renderable, title: str, style: str = BORDER, **kwargs) -> Panel:
+    return Panel(renderable, title=panel_title(title, style), title_align="left", box=box.ROUNDED,
+                 border_style=style, **kwargs)
+
+
+def row(*renderables, ratios: Optional[Sequence[int]] = None) -> Table:
+    grid = Table.grid(expand=True)
+    for i, _ in enumerate(renderables):
+        grid.add_column(ratio=(ratios[i] if ratios else 1))
+    grid.add_row(*renderables)
+    return grid
+
+
+def centered(text: str, style: str = MUTED) -> Align:
+    return Align.center(Text(text, style=style))
+
+
+def shown_proxy(proxy: str) -> str:
+    """Proxy for the terminal: passwords are masked ('alice:•••@1.2.3.4:1080'), the files keep them."""
+    auth, sep, address = proxy.rpartition("@")
+    if not sep:
+        return proxy
+    user = unquote(auth.partition(":")[0])
+    # user names come from third-party lists – never let control characters (newline, ESC) into the terminal
+    user = "".join(c if c.isprintable() else "?" for c in user)
+    return f"{user}:•••@{address}"