twidr 0.1__py3-none-any.whl

twidr/lib/cli.py

@@ -0,0 +1,121 @@
+"""Constructs for command-line argument parsing."""
+
+import re
+from argparse import ArgumentParser, ArgumentTypeError
+from pathlib import Path
+from typing import Final
+
+from . import defaults, meta
+
+HANDLE_RE: Final = re.compile(r"[A-Za-z0-9_]{1,15}")
+DOMAIN_RE: Final = re.compile(r"[a-z0-9\-]+(?:\.[a-z0-9\-]+)+")
+
+
+def handle(value: str) -> str:
+    h = value.lstrip("@")
+
+    if not HANDLE_RE.fullmatch(h):
+        raise ArgumentTypeError(
+            f"{value!r} is not a valid handle; must be 1–15 letters, digits, or underscores"
+        )
+
+    return h
+
+
+def domain(value: str) -> str:
+    d = value.lower()
+
+    if not DOMAIN_RE.fullmatch(d):
+        raise ArgumentTypeError(
+            f"{value!r} is not a valid domain (expected something like twitter.com)"
+        )
+
+    return d
+
+
+def positive_float(value: str) -> float:
+    f = float(value)
+
+    if f <= 0:
+        raise ArgumentTypeError("Must be > 0")
+
+    return f
+
+
+def positive_int(value: str) -> int:
+    i = int(value)
+
+    if i < 1:
+        raise ArgumentTypeError("Must be >= 1")
+
+    return i
+
+
+def parser() -> ArgumentParser:
+    """Get an ArgumentParser instance preconfigured for this program."""
+
+    p = ArgumentParser(description=meta.APP_DESCRIPTION)
+
+    p.add_argument(
+        "-v",
+        "--version",
+        action="version",
+        version=f"🐦️ {meta.APP_NAME}, {meta.PROJECT_DESCRIPTION} ({meta.ORG_DOMAIN}) {meta.VERSION}",
+    )
+
+    p.add_argument("username", type=handle, help="account handle, with or without @")
+
+    p.add_argument(
+        "-o",
+        "--output",
+        type=Path,
+        default=None,
+        help="directory for media and state files (default: a directory named after the handle)",
+    )
+
+    p.add_argument(
+        "--domain",
+        type=domain,
+        action="append",
+        default=None,
+        help=(
+            "archived domain to search (repeat the argument for multiple; "
+            f"domains tried in order; default {", ".join(defaults.DOMAINS)})"
+        ),
+    )
+
+    p.add_argument(
+        "--max-attempts",
+        type=positive_int,
+        default=defaults.MAX_ATTEMPTS,
+        help=f"attempts per request before giving up (default {defaults.MAX_ATTEMPTS})",
+    )
+
+    p.add_argument(
+        "--refresh-index",
+        action="store_true",
+        help="requery the Wayback Machine's CDX index for the account",
+    )
+
+    p.add_argument(
+        "--retry-failed",
+        action="store_true",
+        help="retry tweets or images that failed or yielded no media in previous runs",
+    )
+
+    p.add_argument(
+        "-t",
+        "--threads",
+        type=positive_int,
+        default=defaults.NUM_WORKERS,
+        help=f"number of parallel worker threads (default {defaults.NUM_WORKERS})",
+    )
+
+    p.add_argument(
+        "--rate",
+        type=positive_float,
+        default=defaults.RATE,
+        help=f"max total requests per second across all workers (default {defaults.RATE})",
+    )
+
+    return p

twidr/lib/client.py

@@ -0,0 +1,91 @@
+"""Machinery for session-bound, threaded, and rate-limited HTTP GET requests."""
+
+import threading
+import time
+from threading import Event, Lock
+from typing import Any, Final
+
+from requests import RequestException, Response, Session
+
+from . import meta
+
+USER_AGENT: Final = f"{meta.APP_NAME}/{meta.VERSION} ({meta.ORG_DOMAIN})"
+
+
+class RateLimiter:
+    """A self-contained construct for rate-limiting an HTTP client atomically across threads."""
+
+    def __init__(self, per_second: float, stop: Event) -> None:
+        """Create a new RateLimiter with a per-second maximum rate and an Event to trigger pauses."""
+
+        self._lock = Lock()
+        self._interval = 1 / per_second
+        self._next = 0.0
+        self._stop = stop
+
+    def wait(self) -> None:
+        """Atomically trigger a pause."""
+
+        with self._lock:
+            now = time.monotonic()
+            slot = max(now, self._next)
+            self._next = slot + self._interval
+
+        delay = slot - now
+
+        if delay > 0:
+            self._stop.wait(delay)  # returns early on Ctrl-C
+
+    def penalize(self, seconds: float) -> None:
+        """Atomically add to the current pause delay."""
+
+        with self._lock:
+            self._next = max(self._next, time.monotonic() + seconds)
+
+
+class Client:
+    """A generic, rate-limited HTTP client implementing GET."""
+
+    def __init__(self, rate: float, max_attempts: int) -> None:
+        """Create a new Client with a rate limit and maximum number of attempts per request."""
+
+        self.stop = Event()
+        self._limiter = RateLimiter(rate, self.stop)
+        self._max_attempts = max_attempts
+        self._local = threading.local()
+
+    def _session(self) -> Session:
+        session: Session | None = getattr(self._local, "session", None)
+
+        if session is None:
+            session = Session()
+            session.headers["User-Agent"] = USER_AGENT
+
+            self._local.session = session
+
+        return session
+
+    def get(self, url: str, **kw: Any) -> Response | None:
+        """Send an HTTP GET request with arbitrary parameters."""
+
+        for attempt in range(self._max_attempts):
+            self._limiter.wait()
+
+            if self.stop.is_set():
+                return None
+
+            try:
+                response = self._session().get(url, timeout=30, **kw)
+            except RequestException:
+                self._limiter.penalize(2**attempt)
+
+                continue
+
+            if response.status_code == 429 or response.status_code >= 500:
+                self._limiter.penalize(5 * (attempt + 1))
+
+                continue
+
+            return response
+
+        return None

twidr/lib/defaults.py

@@ -0,0 +1,8 @@
+"""Default program config values."""
+
+from typing import Final
+
+DOMAINS: Final = ("twitter.com", "x.com")
+MAX_ATTEMPTS: Final = 4
+NUM_WORKERS: Final = 4
+RATE: Final = 3.0

twidr/lib/meta.py

@@ -0,0 +1,15 @@
+"""Configurable and pyproject.toml-derived information about this program."""
+
+from importlib.metadata import metadata
+from typing import Final
+
+APP_NAME: Final = "twidr"
+APP_DESCRIPTION: Final = (
+    "download images from the Wayback Machine's snapshots of a Twitter (X) account"
+)
+ORG_DOMAIN: Final = "dandelion.computer"
+
+_meta = metadata(APP_NAME)
+
+VERSION: Final = _meta["Version"]
+PROJECT_DESCRIPTION: Final = _meta["Summary"]

twidr/lib/wayback.py

@@ -0,0 +1,123 @@
+"""A specialized, HTTP-based client for Twitter media stored in the Wayback Machine."""
+
+from threading import Event
+from typing import Final, Literal, NamedTuple
+
+from requests import Response
+
+from .client import Client
+
+
+class Capture(NamedTuple):
+    """An archived copy of a URL on the Wayback Machine."""
+
+    timestamp: str
+    original: str
+
+
+def resolution_rank(capture: Capture) -> tuple[int, str]:
+    """Rank captures of an image by resolution, then by timestamp."""
+
+    url = capture.original
+
+    ordinal = (
+        0 if "orig" in url else 1 if "large" in url else 2 if "medium" in url else 3
+    )
+
+    return (ordinal, capture.timestamp)
+
+
+class Wayback:
+    """A specialized, rate-limited client for retrieving Twitter media from the Wayback Machine."""
+
+    CDX_URL: Final = "https://web.archive.org/cdx/search/cdx"
+    WEB_URL: Final = "https://web.archive.org/web"
+
+    def __init__(self, client: Client) -> None:
+        """Create a new Wayback client, wrapping a Client providing generic, rate-limited HTTP."""
+
+        self._client = client
+
+    @property
+    def stop(self) -> Event:
+        return self._client.stop
+
+    def _cdx(self, **params: str) -> list[list[str]] | None:
+        """Get raw CDX rows (header skipped) or return None on request failure."""
+
+        response = self._client.get(self.CDX_URL, params={"output": "json", **params})
+
+        # failure at request time
+        if response is None or not response.ok:
+            return None
+
+        # empty
+        if not response.text.strip():
+            return []
+
+        # success
+        try:
+            rows: list[list[str]] = response.json()
+
+        # failure at parse time
+        except ValueError:
+            return None
+
+        return rows[1:]
+
+    def _replay(
+        self, capture: Capture, modifier: Literal["id", "im"]
+    ) -> Response | None:
+        """Fetch a capture; "id" gets a raw capture and "im" gets an image."""
+
+        return self._client.get(
+            f"{self.WEB_URL}/{capture.timestamp}{modifier}_/{capture.original}"
+        )
+
+    def captures(self, user: str, domain: str) -> list[Capture] | None:
+        """Get every capture of a user's tweet pages on a domain."""
+
+        rows = self._cdx(
+            url=f"{domain}/{user}/status/*",
+            fl="timestamp,original",
+            filter="!statuscode:404",
+        )
+
+        return None if rows is None else [Capture(row[0], row[1]) for row in rows]
+
+    def variants(self, media_id: str) -> list[Capture]:
+        """Get captures of a piece of media, best resolution first."""
+
+        rows = self._cdx(
+            url=f"pbs.twimg.com/media/{media_id}*",
+            fl="timestamp,original,mimetype",
+            filter="statuscode:200",
+        )
+
+        if rows is None:
+            return []
+
+        return sorted(
+            (Capture(row[0], row[1]) for row in rows if not row[2].startswith("text/")),
+            key=resolution_rank,
+        )
+
+    def page(self, capture: Capture) -> str | None:
+        """Get a captured document as HTML."""
+
+        response = self._replay(capture, "id")
+
+        if response is None or response.status_code != 200:
+            return None
+
+        return response.text
+
+    def image(self, capture: Capture) -> Response | None:
+        """Get a captured image as a Response."""
+
+        response = self._replay(capture, "im")
+
+        if response is None or not response.ok:
+            return None
+
+        return response