threadstractormf 1.4.0__py3-none-any.whl

threadstractormf/_discovery.py

@@ -0,0 +1,445 @@
+"""Discover the current GraphQL request parameters from the live web app.
+
+Why this module exists
+----------------------
+The Threads profile-media query is a *persisted query*: the client sends no
+query text, only ``fb_api_req_friendly_name`` plus the ``doc_id`` that indexes
+the server-side copy. When Meta edits the query it registers a **new** doc_id
+and the old one stops resolving — the server answers
+
+    {"errors": [{"message": "execution error", "severity": "CRITICAL"}]}
+
+which says nothing about what went wrong. That is exactly what a hardcoded
+constant cannot survive: v1.2.0 shipped ``doc_id 37598244946487292`` and it
+went stale, so every scrape silently fell through to the slow DOM fallback.
+
+The id is a server-side index. It cannot be derived, guessed, or computed — the
+only source of truth is the request the web app itself issues. So we watch that
+request once, keep it, and replay it.
+
+Measured, not assumed
+---------------------
+Comparing the page-1 request against page-3 of the same scroll (2026-10):
+
+* identical across pages: ``doc_id``, ``fb_dtsg``, ``lsd``, ``jazoest``,
+  ``__rev``, ``__s``, ``__hsi``, ``av``, ``__crn``, ``__spin_*``, ``__user``
+* different per request: ``__csr``, ``__dyn``, ``__hsdp``, ``__sjsp``,
+  ``__req`` (a counter) and ``variables``
+
+Replaying with only the stable set — the per-request nonces dropped, cursor
+moved to page 2 — returned 10 edges and 0 errors. So a captured template is
+reusable for every page of the walk.
+
+The nonces are still stored. They are anti-abuse tokens rather than part of the
+query, and keeping them costs nothing; if a future deploy starts demanding them
+per request, re-using the captured values is the first thing to try.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any, cast
+from urllib.parse import parse_qs
+
+# The query we need a template for. Stable across doc_id rotations: Meta reuses
+# the friendly name and issues a new id.
+MEDIA_QUERY_NAME = "BarcelonaProfileMediaTabRefetchableDirectQuery"
+
+_GRAPHQL_PATH = "https://www.threads.com/graphql/query"
+
+# Where the capture lives. XDG, with the documented fallbacks, so the template
+# survives across CLI invocations — the whole point is to pay for the browser
+# once, not once per process.
+_CACHE_SUBDIR = "threadstractormf"
+_CACHE_FILENAME = "request_template.json"
+
+# How long an unused template is kept. The tokens in it are session-bound and
+# therefore dead long before this, so it is a grace period for convenience,
+# not for correctness: recapturing costs one browser launch.
+_CACHE_MAX_AGE_DAYS = 14
+
+_UA = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+    "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+)
+
+# Only these are replayed. Verified stable across pages; see the module
+# docstring. Anything else the browser sends is a per-request nonce.
+STABLE_FIELDS = (
+    "doc_id",
+    "fb_api_req_friendly_name",
+    "fb_api_caller_class",
+    "fb_dtsg",
+    "lsd",
+    "jazoest",
+    "av",
+    "__user",
+    "__a",
+    "__rev",
+    "__s",
+    "__hsi",
+    "__hs",
+    "__crn",
+    "__spin_r",
+    "__spin_b",
+    "__spin_t",
+    "dpr",
+    "__ccg",
+    "server_timestamps",
+)
+
+# Kept but not replayed by default: re-used as a base if the stable set ever
+# stops being enough.
+NONCE_FIELDS = ("__csr", "__dyn", "__hsdp", "__sjsp", "__hblp", "__comet_req")
+
+
+def session_fingerprint(cookies: Any) -> str:
+    """A stable key for *which account* a captured template belongs to.
+
+    Why this exists: the captured request carries session-bound tokens — ``lsd``,
+    ``fb_dtsg``, ``__s``, ``__hsi``, and ``av`` (the caller's own id). Replaying
+    them under a different account's cookies gets the query refused, and each
+    refusal rediscovered and overwrote the file, so two users sharing a machine
+    traded the cache back and forth and every scrape paid the browser.
+
+    Keyed by a hash of the session cookie rather than the raw value: the cache
+    file is user-readable, and a sessionid is a live credential.
+    """
+    values: list[str] = []
+    for cookie in cookies or ():
+        name = getattr(cookie, "name", None)
+        if name in {"sessionid", "ds_user_id"}:
+            values.append(f"{name}={getattr(cookie, 'value', '')}")
+    if not values:
+        # Anonymous: the tokens are not tied to an account, so one shared
+        # bucket is correct and still avoids a recapture per profile.
+        return "anonymous"
+    digest = hashlib.sha256("\n".join(sorted(values)).encode()).hexdigest()[:16]
+    return digest
+
+
+def _cache_path(fingerprint: str = "anonymous", username: str = "") -> Path:
+    """XDG cache location for the captured template.
+
+    One file per (session, account). Both parts matter: the tokens are bound to
+    the session, and the captured ``variables.userID`` — the account the browser
+    actually queried — belongs to the profile that was visited. A template
+    captured for @a is meaningless for @b.
+    """
+    base = os.environ.get("XDG_CACHE_HOME")
+    if base:
+        root = Path(base)
+    else:
+        home = Path.home()
+        root = home / ".cache" if os.name == "posix" else home / "AppData" / "Local"
+    suffix = f".{fingerprint}"
+    if username:
+        safe = re.sub(r"[^A-Za-z0-9_.-]", "", username)[:40]
+        suffix = f".{safe}{suffix}"
+    return root / _CACHE_SUBDIR / f"{_CACHE_FILENAME}{suffix}"
+
+
+def legacy_cache_path() -> Path:
+    """The pre-1.3 single-file cache, kept so it can be cleaned up or ignored."""
+    base = os.environ.get("XDG_CACHE_HOME")
+    if base:
+        root = Path(base)
+    else:
+        home = Path.home()
+        root = home / ".cache" if os.name == "posix" else home / "AppData" / "Local"
+    return root / _CACHE_SUBDIR / _CACHE_FILENAME
+
+
+@dataclass
+class RequestTemplate:
+    """A replayable Threads GraphQL request body.
+
+    ``params`` holds the query's own fields; ``nonces`` the per-request
+    anti-abuse tokens, kept for the day they become mandatory.
+
+    ``variables`` is the captured ``variables`` blob, and it is **not**
+    optional decoration. The query carries ~38
+    ``__relay_internal__pv__Barcelona*relayprovider`` feature flags, and Threads
+    refuses the query with ``execution error`` when they are missing — measured,
+    not assumed: the same captured body answered with 12 edges when they were
+    replayed and 0 edges from the 7-key set this module used to send. So the
+    flags travel with the template and only the fields we actually control
+    (``userID``, the cursor, the page size) are overwritten per request.
+    """
+
+    params: dict[str, str] = field(default_factory=dict)
+    nonces: dict[str, str] = field(default_factory=dict)
+    variables: dict[str, Any] = field(default_factory=dict)
+    username: str = ""
+    user_id: str = ""
+    endpoint: str = _GRAPHQL_PATH
+    captured_at: float = 0.0
+
+    @property
+    def doc_id(self) -> str:
+        return self.params.get("doc_id", "")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "params": self.params,
+            "nonces": self.nonces,
+            "variables": self.variables,
+            "username": self.username,
+            "user_id": self.user_id,
+            "endpoint": self.endpoint,
+            "captured_at": self.captured_at,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> RequestTemplate:
+        variables = data.get("variables")
+        return cls(
+            params={str(k): str(v) for k, v in (data.get("params") or {}).items()},
+            nonces={str(k): str(v) for k, v in (data.get("nonces") or {}).items()},
+            variables=variables if isinstance(variables, dict) else {},
+            username=str(data.get("username") or ""),
+            user_id=str(data.get("user_id") or ""),
+            endpoint=str(data.get("endpoint") or _GRAPHQL_PATH),
+            captured_at=float(data.get("captured_at") or 0.0),
+        )
+
+    def is_usable(self) -> bool:
+        """A template is replayable only with an id, an lsd, and its variables.
+
+        The variables are part of the requirement, not a nicety: a template
+        without the relay flags replays into ``execution error`` every time, so
+        loading one without them would produce a template that looks fine and
+        never works.
+        """
+        return bool(
+            self.params.get("doc_id")
+            and self.params.get("lsd")
+            and self.variables
+        )
+
+
+def load_template(fingerprint: str = "anonymous", username: str = "") -> RequestTemplate | None:
+    """Read the cached template for this session, or None when absent/corrupt.
+
+    A cache that cannot be parsed is treated as absent rather than fatal: the
+    caller's response to a missing template is to capture a new one, and that
+    path must not itself raise.
+    """
+    path = _cache_path(fingerprint, username)
+    try:
+        data = json.loads(path.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return None
+    if not isinstance(data, dict):
+        return None
+    template = RequestTemplate.from_dict(data)
+    return template if template.is_usable() else None
+
+
+def save_template(
+    template: RequestTemplate, fingerprint: str = "anonymous", username: str = ""
+) -> None:
+    """Persist the template so the next run skips the browser.
+
+    Best effort: a read-only or full home must not break a scrape that is
+    otherwise working in memory.
+
+    Written to a temporary file and renamed, so a crash or a second process
+    reading mid-write cannot leave truncated JSON behind. Parallel scrapes on
+    one machine are normal for scrapmf, and a corrupt cache would send every
+    one of them back to the browser.
+
+    Old templates are pruned: one file per (session, account), so every
+    logout/login and every new profile adds a file, and nothing was ever
+    removing them. The tokens in them are dead by then, so they are pure
+    clutter. Only files untouched for ``_CACHE_MAX_AGE_DAYS`` go, and the one
+    just written is excluded from the sweep.
+    """
+    path = _cache_path(fingerprint, username or template.username)
+    try:
+        path.parent.mkdir(parents=True, exist_ok=True)
+        try:
+            path.parent.chmod(0o700)
+        except OSError:
+            pass
+        tmp = path.with_suffix(f"{path.suffix}.{os.getpid()}.tmp")
+        # Crear el temporal con modo 0600 desde el inicio, evitando la ventana
+        # en la que existiría con la umask por defecto.
+        fd = os.open(str(tmp), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+        with os.fdopen(fd, "w", encoding="utf-8") as fh:
+            fh.write(json.dumps(template.to_dict(), indent=2))
+        try:
+            tmp.chmod(0o600)
+        except OSError:
+            pass
+        os.replace(tmp, path)
+    except OSError:
+        return
+    _prune_old_templates(path)
+
+
+def _prune_old_templates(keep: Path) -> None:
+    """Delete cached templates nobody has looked at in a while.
+
+    Best effort and deliberately conservative: an error here must never matter
+    to a scrape that already has its template, and a file is only removed once
+    it is clearly stale. Anything that is not a plain file is left alone.
+    """
+    cutoff = time.time() - _CACHE_MAX_AGE_DAYS * 86400
+    try:
+        entries = list(keep.parent.iterdir())
+    except OSError:
+        return
+    for entry in entries:
+        if entry == keep:
+            continue
+        try:
+            if not entry.is_file():
+                continue
+            # Solo se podan archivos de caché de esta herramienta; cualquier
+            # otro archivo del directorio se preserva.
+            if not entry.name.startswith("request_template.json"):
+                continue
+            if entry.stat().st_mtime > cutoff:
+                continue
+            entry.unlink()
+        except OSError:
+            continue
+
+
+def clear_template(fingerprint: str = "anonymous", username: str = "") -> None:
+    """Drop the cached template after the server rejected it."""
+    try:
+        _cache_path(fingerprint, username).unlink()
+    except OSError:
+        pass
+
+
+def build_body(template: RequestTemplate, variables: dict[str, Any]) -> dict[str, str]:
+    """The form body for one page: captured params + the merged variables.
+
+    ``variables`` overrides the captured blob rather than replacing it. Threads
+    validates the whole ``variables`` object against the registered query —
+    dropping the ``__relay_internal__pv__Barcelona*`` feature flags that came
+    with it turns a working request into ``execution error`` — so the capture
+    is the baseline and only the keys passed in here are changed (the target
+    account, the cursor, the page size).
+    """
+    body = dict(template.params)
+    merged = dict(template.variables)
+    merged.update(variables)
+    body["variables"] = json.dumps(merged)
+    return body
+
+
+def _extract_user_id(body: dict[str, str]) -> str:
+    """Pull the target account's id out of the captured variables."""
+    try:
+        parsed = json.loads(body.get("variables") or "{}")
+    except ValueError:
+        return ""
+    uid = parsed.get("userID")
+    return str(uid) if uid else ""
+
+
+def _parse_variables(raw: str | None) -> dict[str, Any]:
+    """Decode the captured ``variables`` blob.
+
+    Kept whole, relay flags included — see :class:`RequestTemplate`. A blob that
+    will not decode yields an empty dict, which makes the template unusable
+    rather than replayable-but-wrong.
+    """
+    try:
+        parsed = json.loads(raw or "{}")
+    except ValueError:
+        return {}
+    return parsed if isinstance(parsed, dict) else {}
+
+
+def discover(username: str, cookies: Any) -> RequestTemplate:
+    """Capture a replayable template by watching the web app's own request.
+
+    Raises RuntimeError when no matching request was observed — the caller
+    decides whether that is fatal or just a reason to fall back.
+    """
+    from playwright.sync_api import sync_playwright
+
+    from threadstractormf._backend import chromium_launch_args, to_playwright_cookies
+
+    captured: list[dict[str, str]] = []
+    url = f"https://www.threads.com/@{username}/media"
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(
+            headless=True,
+            args=chromium_launch_args(),
+        )
+        try:
+            context = browser.new_context(user_agent=_UA)
+            pw = to_playwright_cookies(cookies)
+            if pw:
+                # Best effort: an anonymous capture is still useful, it just
+                # may not carry a session-bound lsd.
+                try:
+                    context.add_cookies(cast("Any", pw))
+                except Exception:
+                    pass
+            page = context.new_page()
+
+            def on_request(request: Any) -> None:
+                try:
+                    post_data = request.post_data or ""
+                except Exception:
+                    return
+                if MEDIA_QUERY_NAME not in post_data:
+                    return
+                if "graphql" not in request.url.lower():
+                    return
+                captured.append({k: v[0] for k, v in parse_qs(post_data).items()})
+
+            page.on("request", on_request)
+            page.goto(url, wait_until="domcontentloaded", timeout=45000)
+            # The profile tab fires its own query on load; the wait is generous
+            # because a slow first paint must not read as "no query happened".
+            page.wait_for_timeout(8000)
+            # One scroll in case the first page was served from cache and the
+            # query only fires on the next batch.
+            try:
+                page.mouse.wheel(0, 4000)
+                page.wait_for_timeout(3000)
+            except Exception:
+                pass
+        finally:
+            browser.close()
+
+    for body in captured:
+        if body.get("doc_id") and body.get("lsd"):
+            template = RequestTemplate(
+                params={k: body[k] for k in STABLE_FIELDS if k in body},
+                nonces={k: body[k] for k in NONCE_FIELDS if k in body},
+                variables=_parse_variables(body.get("variables")),
+                username=username.lstrip("@"),
+                user_id=_extract_user_id(body),
+                endpoint=_GRAPHQL_PATH,
+                captured_at=time.time(),
+            )
+            if template.is_usable():
+                return template
+
+    raise RuntimeError(
+        f"no {MEDIA_QUERY_NAME} request observed while loading {url}; "
+        "Threads may have changed its client, or the profile is unavailable"
+    )
+
+
+def discover_and_cache(username: str, cookies: Any) -> RequestTemplate:
+    """Capture a template and persist it for subsequent runs."""
+    template = discover(username, cookies)
+    save_template(template, session_fingerprint(cookies), template.username)
+    return template

threadstractormf/_parsing.py

@@ -0,0 +1,244 @@
+"""Parsing helpers for Threads API responses.
+
+Extracted from ``api.py`` to keep that module focused on orchestration (GraphQL
+walk + Playwright fallback). Re-exported by ``api.py`` for backwards compat.
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from threadstractormf.models import Media, MediaType, Post
+
+
+def _guess_ext(url: str) -> str:
+    """Best-effort file extension from the URL *path*.
+
+    Extension is read from the path segment itself rather than by searching the
+    whole URL: an image like ".../video_cover.jpg" used to be tagged `.mp4`
+    because the path contained the word "video", which produced files whose
+    extension disagreed with their contents.
+    """
+    path = urlparse(url).path.lower()
+    last_segment = path.rsplit("/", 1)[-1]
+    suffix = last_segment.rsplit(".", 1)[-1] if "." in last_segment else ""
+    # "jpg" is kept as-is (not folded into "jpeg") so filenames stay identical
+    # to previous versions and existing archives still dedup by name.
+    if suffix in {"jpg", "jpeg", "png", "webp", "gif", "avif", "heic", "bmp"}:
+        return suffix
+    video = {"mp4", "webm", "mov", "m4v"}
+    if suffix in video:
+        return suffix
+    # No usable extension in the path: fall back to the URL as a whole, which
+    # is how Threads serves some CDN URLs.
+    low = url.lower()
+    if ".mp4" in low or ".webm" in low or ".mov" in low:
+        return "mp4" if ".mp4" in low else ("webm" if ".webm" in low else "mov")
+    return "jpg"
+
+
+def _pick_best_image(candidates: list[dict]) -> str | None:
+    if not candidates:
+        return None
+    # mayor width primero
+    try:
+        c = max(candidates, key=lambda x: int(x.get("width", 0)))
+        return c.get("url")
+    except Exception:
+        return candidates[0].get("url")
+
+
+def _pick_best_video(versions: Any) -> str | None:
+    """Best MP4 URL out of a ``video_versions`` list.
+
+    Instagram orders these ascending by bitrate, so index 0 is the *worst*
+    rendition — picking it silently threw away resolution. Prefer the entry
+    with the largest pixel area, falling back to pixel count and then to the
+    last element (the highest rung in the ladder).
+    """
+    if not isinstance(versions, list) or not versions:
+        return None
+    candidates = [v for v in versions if isinstance(v, dict) and v.get("url")]
+    if not candidates:
+        return None
+
+    def score(item: dict) -> tuple[int, int]:
+        return (int(item.get("width") or 0) * int(item.get("height") or 0),
+                int(item.get("width") or 0))
+
+    best = max(candidates, key=score)
+    if score(best)[0] == 0 and len(candidates) > 1:
+        # No dimensions reported: trust the ordering and take the top rung.
+        best = candidates[-1]
+    url = best.get("url")
+    return str(url) if url else None
+
+
+def _parse_profile_html(html: str) -> tuple[str | None, str | None, str | None]:
+    """Extract ``(user_id, lsd, profile_pic_url)`` from a profile page.
+
+    A single pass over the document, shared by ``_resolve_user_id`` and
+    ``get_profile`` so one page fetch serves both.
+
+    Only ``userID`` identifies the account. The previous fallback to ``pk``
+    matched the *first* ``pk`` in the document, which belongs to a post — the
+    same field ``_parse_post_node`` treats as a post id — and then used it as
+    the GraphQL ``userID``. That produced empty results and a silent, very slow
+    fallback to the DOM scraper, with no way to tell the user why.
+    """
+    user_id = None
+    lsd = None
+    pic = None
+
+    m = re.search(r'"userID"\s*:\s*"(\d+)"', html)
+    if m:
+        user_id = m.group(1)
+
+    # Threads stopped shipping ``"lsd":"<token>"`` and now nests it in the
+    # Relay require payload as ``["LSD",[],{"token":"<token>"},<prio>]``. The
+    # old regex silently matched nothing, so every request went out with an
+    # empty X-FB-LSD and the query was rejected. Both shapes are accepted
+    # because which one ships depends on the rollout, not on our side.
+    m = re.search(r'\["LSD",\s*\[\],\s*\{"token":"([^"]+)"', html)
+    if m:
+        lsd = m.group(1)
+    else:
+        m = re.search(r'"lsd"\s*:\s*"([^"]+)"', html)
+        if m:
+            lsd = m.group(1)
+
+    m = re.search(r'"profile_pic_url"\s*:\s*"((?:[^"\\]|\\.)*)"', html)
+    if m:
+        # The value is JSON-escaped inside the HTML blob. Only ``\u0026`` and
+        # ``\/`` matter for a URL, and leaving ``\/`` in place produces a URL the
+        # CDN will not serve.
+        pic = m.group(1).replace(r"\u0026", "&").replace(r"\/", "/")
+
+    return user_id, lsd, pic
+
+
+def _parse_post_node(node: dict, fallback_username: str | None = None) -> Post | None:
+    # node = {"thread_items": [{"post": {...}}]}
+    try:
+        items = node.get("thread_items") or []
+        if not items:
+            return None
+        post = items[0].get("post")
+        if not post:
+            return None
+        # reposts: if post.user.username != fallback_username and excluded, filtered above
+        pk = str(post.get("pk") or "")
+        code = post.get("code")
+        # code is the short post_id like DKxyz (threads.net/@user/post/<code>)
+        post_id = code if code else pk
+        if not post_id:
+            return None
+        username = post.get("user", {}).get("username") or fallback_username or "unknown"
+        permalink = f"https://www.threads.com/@{username}/post/{post_id}"
+        # caption
+        cap = post.get("caption", {})
+        caption = None
+        if isinstance(cap, dict):
+            caption = cap.get("text")
+        # datetime from taken_at (unix)
+        taken = post.get("taken_at")
+        datetime_iso = None
+        if taken:
+            try:
+                datetime_iso = datetime.fromtimestamp(int(taken), tz=timezone.utc).isoformat()
+            except Exception:
+                pass
+        # counts (no vienen en mediaData, set 0)
+        like_count = 0
+        # carousel
+        medias: list[Media] = []
+        vtype: MediaType = "image"
+        carousel = post.get("carousel_media")
+        # single media case
+        if carousel and isinstance(carousel, list) and len(carousel) > 0:
+            for idx, cm in enumerate(carousel):
+                # cm has image_versions2 or video_versions
+                url = None
+                vtype = "image"
+                if cm.get("video_versions"):
+                    url = _pick_best_video(cm["video_versions"])
+                    vtype = "video"
+                if not url:
+                    iv2 = cm.get("image_versions2", {})
+                    url = _pick_best_image(iv2.get("candidates", []))
+                if not url:
+                    continue
+                ext = _guess_ext(url)
+                mid = post_id if len(carousel) == 1 else f"{post_id}_{idx+1}"
+                medias.append(
+                    Media(id=mid, post_id=post_id, index=idx + 1, type=vtype, url=url, ext=ext)
+                )
+        else:
+            # single image/video
+            url = None
+            vtype = "image"
+            if post.get("video_versions"):
+                url = _pick_best_video(post["video_versions"])
+                vtype = "video"
+            if not url:
+                iv2 = post.get("image_versions2", {})
+                url = _pick_best_image(iv2.get("candidates", []))
+            if url:
+                ext = _guess_ext(url)
+                medias.append(
+                    Media(id=post_id, post_id=post_id, index=1, type=vtype, url=url, ext=ext)
+                )
+            else:
+                # text only? skip if no media (we do not want text-only posts)
+                pass
+        # no medias: skip entirely (user asked for photos/videos only)
+        if not medias:
+            return None
+        return Post(
+            id=post_id,
+            permalink=permalink,
+            username=username,
+            caption=caption,
+            datetime_iso=datetime_iso,
+            like_count=like_count,
+            media=medias,
+        )
+    except Exception:
+        return None
+
+
+def _walk_xhr_videos(node: Any, found: dict[str, list[str]]) -> None:
+    """Recursively collect {post_code: [mp4 urls]} from threads site API JSON.
+
+    Post objects carry "code" plus video_versions at their own level or inside
+    carousel_media items; this walks arbitrary response trees.
+    """
+    if isinstance(node, dict):
+        code = node.get("code")
+        if isinstance(code, str):
+            urls: list[str] = []
+            vv = node.get("video_versions")
+            if isinstance(vv, list):
+                for v in vv:
+                    if isinstance(v, dict) and ".mp4" in (v.get("url") or ""):
+                        urls.append(str(v["url"]))
+            car = node.get("carousel_media")
+            if isinstance(car, list):
+                for item in car:
+                    if (
+                        isinstance(item, dict)
+                        and isinstance(item.get("video_versions"), list)
+                    ):
+                        for v in item["video_versions"]:
+                            if isinstance(v, dict) and ".mp4" in (v.get("url") or ""):
+                                urls.append(str(v["url"]))
+            if urls:
+                found[code].extend(urls)
+        for v in node.values():
+            _walk_xhr_videos(v, found)
+    elif isinstance(node, list):
+        for v in node:
+            _walk_xhr_videos(v, found)