tikstractormf 1.0.0__py3-none-any.whl

tikstractormf/__init__.py

@@ -0,0 +1,7 @@
+"""tikstractormf — TikTok media downloader (gallery-dl cookies + post_id naming)."""
+
+from tikstractormf.client import Tiktokscraper
+from tikstractormf.models import Media, Post, Profile
+
+__all__ = ["Tiktokscraper", "Post", "Media", "Profile"]
+__version__ = "1.0.0"

tikstractormf/_backend.py

@@ -0,0 +1,283 @@
+"""Shared helpers for talking to Meta's servers through two HTTP backends.
+
+Two independent backends are supported:
+
+* **httpx** — the default. No browser TLS impersonation.
+* **curl_cffi** — opted into with ``--impersonate chrome``. Meta's WAF inspects
+  the JA3/JA4 TLS fingerprint and the HTTP/2 SETTINGS frame, which httpx cannot
+  fake; curl_cffi impersonates a real Chrome byte-for-byte, the same mechanism
+  ``yt-dlp --impersonate`` uses.
+
+Everything else (rate limiting, retries, streaming, the GraphQL client and the
+media downloader) is written against the tiny surface both libraries share, so
+the httpx path stays byte-identical and curl_cffi is just an opt-in branch.
+
+Also holds the cookie converter used by the three Playwright call sites, which
+used to duplicate it three times (and all three dropped the leading dot, which
+made every auth cookie host-only).
+"""
+
+from __future__ import annotations
+
+import socket
+from typing import TYPE_CHECKING, Any, Protocol, cast
+
+import httpx
+
+if TYPE_CHECKING:  # pragma: no cover
+    # curl_cffi is an optional extra; the alias is only needed for typing.
+    from curl_cffi.requests.session import HttpMethod
+else:
+    HttpMethod = str
+
+# ─── Retry classification ────────────────────────────────────────────────────
+#
+# Shared by the GraphQL client and the media downloader. Both exception
+# families must be understood here, because curl_cffi replaces httpx wholesale
+# when --impersonate is active.
+
+
+def is_retryable_status(status: int) -> bool:
+    """True for statuses worth retrying: 429 (rate limited) and any 5xx.
+
+    Every other 4xx is permanent and must surface immediately. A 403 from
+    Meta's WAF will not fix itself, so retrying it only burns the full
+    exponential backoff (~60s) before reporting the very same error.
+    """
+    return status == 429 or status >= 500
+
+
+def _curl_exc() -> Any:
+    """curl_cffi's exception module, or None when the extra is not installed."""
+    try:
+        from curl_cffi.requests import exceptions as curl_exceptions
+    except ImportError:
+        return None
+    return curl_exceptions
+
+
+def is_transient_error(exc: BaseException) -> bool:
+    """True when a failure is worth retrying (flaky network, DNS, CDN hiccup).
+
+    Handles both backends. Order matters: curl_cffi's ``RequestException``
+    subclasses ``OSError``, so the curl branch must be tested *before* the
+    generic ``OSError`` branch or every curl error would be misclassified as a
+    plain OSError with no errno and never retried.
+    """
+    curl = _curl_exc()
+    if curl is not None:
+        # Raised by Response.raise_for_status(): retry only if the status allows.
+        if isinstance(exc, curl.HTTPError):
+            status = getattr(getattr(exc, "response", None), "status_code", None)
+            return status is not None and is_retryable_status(int(status))
+        # ConnectTimeout, ReadTimeout, ConnectionError (incl. DNSError, SSLError).
+        if isinstance(exc, (curl.Timeout, curl.ConnectionError)):
+            return True
+        # Any other curl RequestException (InvalidURL, TooManyRedirects, ...)
+        # is a permanent programming/URL problem.
+        if isinstance(exc, curl.RequestException):
+            return False
+
+    if isinstance(exc, httpx.HTTPStatusError):
+        return is_retryable_status(exc.response.status_code)
+    if isinstance(exc, httpx.TransportError):
+        # ConnectError (wraps gaierror/[Errno -3]), ConnectTimeout, ReadTimeout,
+        # RemoteProtocolError, PoolTimeout, ...
+        return True
+    if isinstance(exc, httpx.HTTPError):
+        # InvalidURL, DecodingError, TooManyRedirects: permanent.
+        return False
+
+    if isinstance(exc, OSError):
+        # EAI_AGAIN lives in `socket`, not `errno`; referencing errno.EAI_AGAIN
+        # raised AttributeError on *any* OSError reaching here, including the
+        # very DNS flap this branch exists to detect.
+        return exc.errno in (-3, socket.EAI_AGAIN)
+    return False
+
+
+# ─── HTTP backend ────────────────────────────────────────────────────────────
+
+
+class StreamClient(Protocol):
+    """The minimum surface the media downloader needs.
+
+    Both ``httpx.Client`` and :class:`CurlClient` satisfy it structurally,
+    which is what lets ``download_media`` accept either backend. The signature
+    lists only the argument the downloader actually passes, because a protocol
+    declaring ``**kwargs`` would require the implementation to accept arbitrary
+    keywords — which ``httpx.Client.stream`` does not.
+    """
+
+    def stream(
+        self, method: str, url: str, *, headers: dict[str, str] | None = None
+    ) -> Any: ...
+
+    def close(self) -> None: ...
+
+
+class _CurlStreamResponse:
+    """Presents a curl_cffi streamed response through the httpx surface.
+
+    Only one method actually differs: curl_cffi names it ``iter_content``
+    while httpx calls it ``iter_bytes``. Everything else (``status_code``,
+    ``raise_for_status``) already matches.
+    """
+
+    __slots__ = ("_resp",)
+
+    def __init__(self, resp: Any) -> None:
+        self._resp = resp
+
+    @property
+    def status_code(self) -> int:
+        return int(self._resp.status_code)
+
+    def raise_for_status(self) -> None:
+        self._resp.raise_for_status()
+
+    def iter_bytes(self, chunk_size: int | None = None) -> Any:
+        # chunk_size is intentionally dropped: curl pulls what it wants off the
+        # socket and warns loudly if we pretend otherwise.
+        return self._resp.iter_content()
+
+
+def known_impersonate_targets() -> set[str]:
+    """Every impersonation target curl_cffi understands (generic + versioned).
+
+    curl_cffi does **not** validate ``impersonate`` at construction time — it
+    accepts any string and only misbehaves later, at request time. Validating up
+    front turns a confusing runtime failure into an actionable error.
+    """
+    from typing import get_args
+
+    from curl_cffi.requests.impersonate import REAL_TARGET_MAP, BrowserTypeLiteral
+
+    return set(REAL_TARGET_MAP) | set(get_args(BrowserTypeLiteral))
+
+
+class CurlClient:
+    """``httpx``-shaped wrapper around a curl_cffi Session.
+
+    Only instantiated when ``impersonate`` is requested. Three differences from
+    ``httpx.Client`` are papered over here:
+
+    * cookies are *assigned* (``session.cookies = jar``), not passed as a kwarg
+    * ``timeout`` is ``(connect, total)`` — verified empirically: the second
+      element is a whole-transfer budget, **not** a per-phase timeout, so it has
+      to be generous enough for large media or slow CDN links get truncated
+      mid-download (the same failure the httpx path fixed via a per-chunk read)
+    * HTTP/2 must be requested explicitly
+    """
+
+    def __init__(
+        self,
+        *,
+        impersonate: str,
+        cookies: Any = None,
+        headers: dict[str, str] | None = None,
+        connect_timeout: float = 15.0,
+        total_timeout: float = 600.0,
+        http2: bool = True,
+    ) -> None:
+        try:
+            from curl_cffi import requests as curl_requests
+        except ImportError as exc:  # pragma: no cover - depends on env
+            raise ImportError(
+                "--impersonate requires curl_cffi. "
+                "Install it with: pip install 'tikstractormf[antibot]'"
+            ) from exc
+
+        self.impersonate = impersonate
+        if impersonate not in known_impersonate_targets():
+            examples = ", ".join(sorted(known_impersonate_targets())[:6])
+            raise ValueError(
+                f"unknown impersonation target: {impersonate!r}. "
+                f"Valid targets include: {examples} (and versioned forms like "
+                "'chrome131' or 'safari18_0')"
+            )
+
+        kwargs: dict[str, Any] = {"timeout": (connect_timeout, total_timeout)}
+        if http2:
+            from curl_cffi.const import CurlHttpVersion
+
+            kwargs["http_version"] = CurlHttpVersion.V2_0
+        self._session = curl_requests.Session(impersonate=impersonate, **kwargs)
+
+        # A CookieJar instance is converted by curl_cffi on assignment.
+        if cookies is not None:
+            self._session.cookies = cookies
+        if headers:
+            self._session.headers.update(headers)
+
+    def request(self, method: str, url: str, **kwargs: Any) -> Any:
+        kwargs.setdefault("allow_redirects", True)
+        return self._session.request(cast("HttpMethod", method), url, **kwargs)
+
+    def stream(self, method: str, url: str, **kwargs: Any) -> Any:
+        from contextlib import contextmanager
+
+        verb = cast("HttpMethod", method)
+
+        @contextmanager
+        def _ctx() -> Any:
+            with self._session.stream(verb, url, **kwargs) as resp:
+                yield _CurlStreamResponse(resp)
+
+        return _ctx()
+
+    def close(self) -> None:
+        self._session.close()
+
+
+# ─── Playwright ──────────────────────────────────────────────────────────────
+
+
+def chromium_launch_args() -> list[str]:
+    """Args for ``chromium.launch``.
+
+    ``--no-sandbox`` is required in many CI/container environments but reduces
+    isolation. It can be disabled by setting ``TIKTOK_USE_SANDBOX=1`` when the
+    environment supports an unprivileged sandbox.
+    """
+    import os
+
+    args = ["--disable-blink-features=AutomationControlled"]
+    if os.environ.get("TIKTOK_USE_SANDBOX") != "1":
+        args.insert(0, "--no-sandbox")
+    return args
+
+
+def to_playwright_cookies(jar: Any) -> list[dict[str, Any]]:
+    """Convert a CookieJar into the dict shape ``BrowserContext.add_cookies`` wants.
+
+    The leading dot on the domain **must be preserved**. Chromium treats a
+    domain without one as host-only, so ``threads.net`` would never be sent to
+    ``www.threads.net`` — i.e. every auth cookie silently dropped and the
+    browser left logged out. Verified: with ``.threads.net`` the cookie arrives,
+    with ``threads.net`` no ``Cookie`` header is sent at all.
+
+    Shared by ``TiktokAPI`` (profile pic + post scraping) and ``scripts/sniff.py``
+    so the three former copies of this conversion cannot drift apart again.
+    """
+    out: list[dict[str, Any]] = []
+    for cookie in jar or ():
+        try:
+            domain = getattr(cookie, "domain", "") or ""
+            if not domain:
+                # Host-only cookies have no domain and cannot be added without a
+                # URL; skip rather than let add_cookies() fail on the whole batch.
+                continue
+            out.append(
+                {
+                    "name": cookie.name,
+                    "value": cookie.value,
+                    "domain": domain,
+                    "path": getattr(cookie, "path", "/") or "/",
+                    "secure": bool(getattr(cookie, "secure", True)),
+                }
+            )
+        except Exception:
+            continue
+    return out
+

tikstractormf/_parsing.py

@@ -0,0 +1,267 @@
+"""Parsing helpers for TikTok API responses."""
+
+from __future__ import annotations
+
+import json as _json
+import re
+from datetime import datetime, timezone
+from urllib.parse import urlparse
+
+from tikstractormf.models import Media, Post
+
+
+def _guess_ext(url: str, format_hint: str | None = None) -> str:
+    # format_hint comes from the JSON (video.format, mime_type, etc.)
+    if format_hint:
+        fh = format_hint.lower().strip(".")
+        if fh in {"jpg", "jpeg", "png", "webp", "gif", "avif", "heic", "bmp",
+                  "mp4", "webm", "mov", "m4v"}:
+            return "jpeg" if fh == "jpeg" else fh
+        if "mp4" in fh:
+            return "mp4"
+        if "webp" in fh:
+            return "webp"
+        if "png" in fh:
+            return "png"
+    path = urlparse(url).path.lower()
+    last_segment = path.rsplit("/", 1)[-1]
+    suffix = last_segment.rsplit(".", 1)[-1] if "." in last_segment else ""
+    if suffix in {"jpg", "jpeg", "png", "webp", "gif", "avif", "heic", "bmp"}:
+        return suffix
+    video = {"mp4", "webm", "mov", "m4v"}
+    if suffix in video:
+        return suffix
+    # inspect query string for mime_type / format
+    from urllib.parse import parse_qs
+    qs = parse_qs(urlparse(url).query)
+    mime = (qs.get("mime_type") or qs.get("mime") or [""])[0].lower()
+    fmt = (qs.get("format") or qs.get("file_format") or [""])[0].lower()
+    if "video" in mime or fmt in {"mp4", "webm", "mov", "m4v"}:
+        if "webm" in mime or fmt == "webm":
+            return "webm"
+        if "mov" in mime or fmt == "mov":
+            return "mov"
+        return "mp4"
+    if "webp" in mime or fmt == "webp":
+        return "webp"
+    if "png" in mime or fmt == "png":
+        return "png"
+    low = url.lower()
+    if ".mp4" in low or ".webm" in low or ".mov" in low:
+        return "mp4" if ".mp4" in low else ("webm" if ".webm" in low else "mov")
+    return "jpg"
+
+
+def parse_item(item: dict, fallback_username: str | None = None,
+               extract_audio: bool = False) -> Post | None:
+    """Parse one TikTok item (from item_list) into a Post."""
+    try:
+        vid = str(item.get("id") or "")
+        if not vid:
+            return None
+        author = item.get("author") or {}
+        username = author.get("uniqueId") or fallback_username or "unknown"
+        desc = item.get("desc")
+        create_time = item.get("createTime")
+        datetime_iso = None
+        if create_time:
+            try:
+                datetime_iso = datetime.fromtimestamp(int(create_time), tz=timezone.utc).isoformat()
+            except Exception:
+                pass
+        permalink = f"https://www.tiktok.com/@{username}/video/{vid}"
+
+        medias: list[Media] = []
+        video = item.get("video") or {}
+        # Prefer best quality from bitrateInfo
+        play_addr = None
+        format_hint = video.get("format")
+        bitrate_info = video.get("bitrateInfo")
+        bitrate_candidates: list[str] = []
+        if isinstance(bitrate_info, list) and bitrate_info:
+            # pick highest width*height
+            best_score = -1
+            best_url_list: list[str] = []
+            for bi in bitrate_info:
+                pa = bi.get("PlayAddr") or {}
+                url_list = pa.get("UrlList") or []
+                if not url_list:
+                    continue
+                for u in url_list:
+                    if u and u not in bitrate_candidates:
+                        bitrate_candidates.append(u)
+                w = int(pa.get("Width") or 0)
+                h = int(pa.get("Height") or 0)
+                score = w * h
+                if score > best_score:
+                    best_score = score
+                    play_addr = url_list[0]
+                    best_url_list = list(url_list)
+                    format_hint = pa.get("DataSize") and format_hint
+            if not play_addr and bitrate_info:
+                first = bitrate_info[0].get("PlayAddr", {}).get("UrlList") or []
+                if first:
+                    play_addr = first[0]
+                    best_url_list = list(first)
+            if best_url_list:
+                for u in best_url_list:
+                    if u and u not in bitrate_candidates:
+                        bitrate_candidates.append(u)
+        if not play_addr:
+            play_addr = video.get("playAddr") or video.get("downloadAddr")
+        cover = video.get("cover") or video.get("originCover") or video.get("dynamicCover")
+        download_addr = video.get("downloadAddr")
+        # Prefer downloadAddr unless it is a watermarked "logo" variant, in
+        # which case playAddr is usually the clean source.
+        if download_addr and "logo" in download_addr.lower() and play_addr:
+            primary = play_addr
+        else:
+            primary = download_addr or play_addr
+
+        # Collect all download candidates, preferring downloadAddr, then playAddr,
+        # then every UrlList entry from bitrateInfo. Dedup while preserving order.
+        all_candidates: list[str] = []
+        for cand in (
+            download_addr,
+            play_addr,
+            video.get("playAddr"),
+            *bitrate_candidates,
+        ):
+            if cand and cand not in all_candidates:
+                all_candidates.append(cand)
+
+        fallback_urls: list[str] = []
+        if primary:
+            for cand in all_candidates:
+                if cand and cand != primary and cand not in fallback_urls:
+                    fallback_urls.append(cand)
+            ext = _guess_ext(primary, format_hint or video.get("format"))
+            medias.append(
+                Media(
+                    id=vid, post_id=vid, index=1, type="video", url=primary,
+                    fallback_urls=fallback_urls, ext=ext,
+                )
+            )
+        # photo posts
+        image_post = item.get("imagePost") or {}
+        images = image_post.get("images") or []
+        if images and not play_addr:
+            for idx, img in enumerate(images):
+                iu = None
+                img_fallbacks: list[str] = []
+                if isinstance(img, dict):
+                    raw = img.get("imageURL")
+                    ul: list[str] = []
+                    if isinstance(raw, dict):
+                        ul = raw.get("urlList") or raw.get("UrlList") or []
+                        if ul:
+                            iu = ul[0]
+                            img_fallbacks = [u for u in ul[1:] if u]
+                    elif isinstance(raw, str):
+                        iu = raw
+                    iu = iu or img.get("displayImage") or img.get("url")
+                    if not iu and isinstance(img.get("image"), dict):
+                        iu2 = img["image"].get("url") or img["image"].get("imageURL")
+                        if isinstance(iu2, dict):
+                            ul2 = iu2.get("urlList") or iu2.get("UrlList") or []
+                            if ul2:
+                                iu = ul2[0]
+                                img_fallbacks = [u for u in ul2[1:] if u]
+                        elif isinstance(iu2, str):
+                            iu = iu2
+                    # displayImage may also carry a urlList
+                    di = img.get("displayImage")
+                    if isinstance(di, dict):
+                        dul = di.get("urlList") or di.get("UrlList") or []
+                        if dul and not iu:
+                            iu = dul[0]
+                            img_fallbacks = [u for u in dul[1:] if u]
+                        elif dul:
+                            for u in dul:
+                                if u and u != iu and u not in img_fallbacks:
+                                    img_fallbacks.append(u)
+                if not iu:
+                    continue
+                ext = _guess_ext(iu)
+                mid = vid if len(images) == 1 else f"{vid}_{idx+1}"
+                medias.append(
+                    Media(
+                        id=mid, post_id=vid, index=idx + 1, type="image", url=iu,
+                        fallback_urls=img_fallbacks, ext=ext,
+                    )
+                )
+        # Audio track attached to a photo carousel (gallery-dl extracts it when
+        # audio=True). Only add when the item has no playable video.
+        music = item.get("music") or {}
+        music_url = music.get("playUrl")
+        if extract_audio and images and not play_addr and music_url:
+            ext = _guess_ext(music_url)
+            medias.append(
+                Media(
+                    id=f"{vid}_audio",
+                    post_id=vid,
+                    index=len(images) + 1,
+                    type="audio",
+                    url=music_url,
+                    ext=ext if ext in {"mp3", "m4a", "aac", "wav", "ogg"} else "mp3",
+                )
+            )
+        # cover as image if no media? skip
+        if not medias and cover:
+            ext = _guess_ext(cover)
+            medias.append(Media(id=vid, post_id=vid, index=1, type="image", url=cover, ext=ext))
+
+        if not medias:
+            return None
+        return Post(
+            id=vid,
+            permalink=permalink,
+            username=username,
+            caption=desc,
+            datetime_iso=datetime_iso,
+            media=medias,
+        )
+    except Exception:
+        return None
+
+
+def _pick_best_image(candidates):
+    if not candidates:
+        return None
+    try:
+        c = max(candidates, key=lambda x: int(x.get("width", 0)))
+        return c.get("url")
+    except Exception:
+        return candidates[0].get("url") if candidates else None
+
+
+def extract_user_id(html: str) -> str | None:
+    m = re.search(r'"uniqueId"\s*:\s*"([^"]+)"', html)
+    return m.group(1) if m else None
+
+
+def _unescape_json_string(value: str) -> str:
+    """Decode the escape sequences found in a JSON string literal."""
+    try:
+        result = _json.loads('"' + value + '"')
+        return str(result)
+    except Exception:
+        return (
+            value.replace(r"\u0026", "&")
+            .replace(r"\u002F", "/")
+            .replace(r"\u002f", "/")
+            .replace(r"\/", "/")
+        )
+
+
+def parse_profile_html(html: str):
+    """Return the profile pic URL (or None)."""
+    pic = None
+    m = re.search(r'"avatarLarger"\s*:\s*"((?:[^"\\]|\\.)*)"', html)
+    if m:
+        pic = _unescape_json_string(m.group(1))
+    if not pic:
+        m = re.search(r'"avatarMedium"\s*:\s*"((?:[^"\\]|\\.)*)"', html)
+        if m:
+            pic = _unescape_json_string(m.group(1))
+    return pic