threadstractormf 1.4.0__py3-none-any.whl

threadstractormf/downloader.py

@@ -0,0 +1,353 @@
+"""Downloader — port of background.js:787-904 with post_id naming.
+
+User rules:
+  - media.id = post_id if post has a single media
+  - media.id = f"{post_id}_{i}" for carousels (1-index)
+  - filename = f"{media.id}.{ext}" (no datetime)
+  - profile pic = f"{username}_profile.{ext}"
+  - URL validated with isValidMediaUrl (CDN allowlist)
+  - filename sanitized
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum
+from pathlib import Path
+from urllib.parse import urlparse
+
+import httpx
+
+from threadstractormf._backend import (
+    CurlClient,
+    StreamClient,
+    is_transient_error,
+)
+from threadstractormf.archive import DownloadLedger, find_existing_for_post
+from threadstractormf.models import Media, sanitize_filename
+from threadstractormf.rate_limit import BatchCooldownLimiter
+
+# Allowlist same as background.js:112-116
+# Hostname suffixes Meta serves media from.
+#
+# Matched as a *domain suffix*, never as a substring. Two reasons:
+#   * the CDN rotates regional hosts (instagram.fpei1-1.fna.fbcdn.net,
+#     scontent-xx.cdninstagram.com), so pinning literal hosts would go stale,
+#     while a suffix keeps working for a POP we have never seen;
+#   * a substring match accepts any host that merely embeds the name, which is
+#     what let https://instagram.evil.test/x.jpg and
+#     https://scontent.evil.com/v/t51.1/a.jpg through.
+#
+# "instagram" and "threads" are deliberately absent: those are brand names, not
+# hostnames.
+_CDN_HOST_SUFFIXES = (
+    ".cdninstagram.com",
+    ".cdninstagram.net",
+    ".fbcdn.net",
+)
+
+# Global limiter por defecto (port background.js:16-17). Se puede inyectar otro por llamada.
+_default_limiter = BatchCooldownLimiter(
+    cooldown_ms=2000, batch_size=100, batch_cooldown_ms=120000, jitter=0.2, enabled=True
+)
+
+
+# Directories that already represent one media type. Landing in one of them is
+# taken as "the caller decided where things go", so no further split happens.
+CONTENT_DIRS = frozenset({"photos", "videos", "profile", "posts"})
+
+# Browser-like headers: video endpoints (/v/t65.*) reject python-httpx UA with 403.
+CDN_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+    ),
+    "Accept": "*/*",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Referer": "https://www.threads.net/",
+}
+
+
+def _split_subfolder(media: Media, dest: Path) -> str | None:
+    """Subdirectory for this media, or None to write straight into ``dest``.
+
+    Behaviour (unchanged, previously inlined):
+
+    ==================  ==========  ==========================
+    media                dest.name   result
+    ==================  ==========  ==========================
+    avatar               any but     ``profile``
+                        ``profile``
+    any                  one of      written into dest
+                        CONTENT_DIRS
+    video                anything    ``videos``
+                        else
+    image                anything    ``photos``
+                        else
+    ==================  ==========  ==========================
+
+    Note the folder comes from ``media.type`` (what the API declares), while the
+    file extension comes from the URL. The two are independent: a ``.webm`` video
+    is ``type="video"`` and therefore lands in ``videos/``.
+    """
+    if media.id.endswith("_profile"):
+        return None if dest.name == "profile" else "profile"
+    if dest.name in CONTENT_DIRS:
+        return None
+    return "videos" if media.type == "video" else "photos"
+
+
+def is_valid_media_url(url: str) -> bool:
+    """Port background.js:96-144 isValidMediaUrl, with a domain-suffix host check."""
+    if not url or not isinstance(url, str):
+        return False
+    try:
+        u = urlparse(url)
+        if u.scheme != "https":
+            return False
+        # A fully-qualified name may arrive with a trailing dot.
+        host = u.netloc.lower().rstrip(".")
+        # s[1:] allows the bare registrable domain (cdninstagram.com); otherwise
+        # the dot must match exactly one label boundary, so cdninstagram.com.evil.io
+        # and notcdninstagram.com are both rejected.
+        if not any(host == s[1:] or host.endswith(s) for s in _CDN_HOST_SUFFIXES):
+            return False
+        path = u.path.lower()
+        has_ext = bool(re.search(r"\.(jpg|jpeg|png|webp|gif|mp4|webm|mov|avi)(\?|$)", path))
+        has_media_path = (
+            "/v/t51." in path or "/image/" in path or "/video/" in path or "/media/" in path
+        )
+        has_qs = bool(u.query)
+        return has_ext or has_media_path or has_qs
+    except Exception:
+        return False
+
+
+# Retry policy for transient network errors (DNS resolution flaps, timeouts,
+# CDN hiccups). Extra attempts give the resolver/network time to recover.
+_DOWNLOAD_ATTEMPTS = 3
+_DOWNLOAD_BACKOFF_S = (2.0, 5.0)
+
+# Generous CDN timeouts: slow-yet-active transfers stay alive (read timeout is
+# per-chunk, not total) and DNS/TLS setup gets breathing room on bad networks.
+_CDN_TIMEOUT = httpx.Timeout(connect=15.0, read=45.0, write=45.0, pool=30.0)
+
+# curl_cffi takes (connect, total): the second value bounds the whole transfer,
+# so it must be well above the per-chunk httpx read budget or big videos on slow
+# links get cut mid-download.
+_CDN_CURL_TIMEOUT = (15.0, 900.0)
+
+
+def _build_download_client(
+    *, impersonate: str | None, headers: dict[str, str]
+) -> StreamClient:
+    """Create the CDN client: Chrome impersonation when asked, else httpx."""
+    if impersonate:
+        return CurlClient(
+            impersonate=impersonate,
+            headers=headers,
+            connect_timeout=_CDN_CURL_TIMEOUT[0],
+            total_timeout=_CDN_CURL_TIMEOUT[1],
+        )
+    return httpx.Client(follow_redirects=True, timeout=_CDN_TIMEOUT, http2=True)
+
+
+class DownloadStatus(str, Enum):
+    """Whether a media call actually fetched bytes.
+
+    download_media used to return a bare Path from every branch, so a file the
+    ledger already knew about was indistinguishable from a fresh download: the
+    CLI announced both as "downloaded" and any summary built on it would have
+    been fiction.
+    """
+
+    DOWNLOADED = "downloaded"
+    SKIPPED = "skipped"  # the ledger already had this (post_id, index)
+    ADOPTED = "adopted"  # found on disk under another name
+
+
+@dataclass(frozen=True)
+class DownloadOutcome:
+    """Result of one media download: where it ended up, and how it got there."""
+
+    path: Path
+    status: DownloadStatus
+
+
+def download_media(
+    media: Media,
+    dest: str | Path,
+    *,
+    client: StreamClient | None = None,
+    impersonate: str | None = None,
+    overwrite: bool = False,
+    limiter: BatchCooldownLimiter | None = None,
+    rate_limit: bool = True,
+    filename_template: str | None = None,
+    username: str | None = None,
+    date_iso: str | None = None,
+    ledger: DownloadLedger | None = None,
+    adopt_existing: bool = True,
+) -> DownloadOutcome:
+    """Download a Media to dest.
+
+    Returns where it landed plus whether bytes were actually fetched: a media
+    the ledger already knew about returns SKIPPED, not DOWNLOADED.
+
+    Applies anti rate-limit if ``rate_limit=True``. When ``filename_template`` is
+    given it generates the filename (gallery-dl style).
+
+    Skipping happens at three levels, cheapest first:
+
+    1. ``(post_id, index)`` already in ``ledger`` — immune to the filename
+       changing (a fixed extension guess, a different template, a date that
+       rolled over);
+    2. the exact destination file exists — records it in the ledger;
+    3. with ``adopt_existing``, a file for the same ``post_id`` under a different
+       name is adopted — this is what saves a historical archive from being
+       re-downloaded when the naming scheme is corrected.
+
+    Levels 2 and 3 make the ledger self-seeding: the first run after upgrading
+    behaves exactly as before (filename dedup) while filling the ledger.
+    ``overwrite`` bypasses all three.
+    """
+    if not is_valid_media_url(media.url):
+        raise ValueError(f"invalid download URL: {media.url}")
+
+    root = Path(dest).expanduser().resolve()
+    # Split photos/videos/profile only when dest is not already one of them.
+    # The ledger key is derived from `root`, BEFORE the split, so one account
+    # keeps a single ledger instead of one per media type.
+    subfolder = _split_subfolder(media, root)
+    dest = root / subfolder if subfolder else root
+    dest.mkdir(parents=True, exist_ok=True)
+
+    if filename_template:
+        # For profile, ignore date template and use simple username
+        if media.id.endswith("_profile") and "{date" in filename_template:
+            filename = media.filename(
+                template="{username}_profile.{extension}", username=username, date_iso=date_iso
+            )
+        else:
+            filename = media.filename(
+                template=filename_template, username=username, date_iso=date_iso
+            )
+    else:
+        filename = media.filename()
+    filename = sanitize_filename(filename)
+    out = dest / filename
+
+    if not overwrite:
+        # 1. the ledger knows about it under any name
+        if ledger is not None and ledger.has(media.post_id, media.index):
+            return DownloadOutcome(out, DownloadStatus.SKIPPED)
+        # 2. the exact file is already there -> adopt it into the ledger
+        if out.exists():
+            if ledger is not None:
+                ledger.record(media.post_id, media.index, out)
+            return DownloadOutcome(out, DownloadStatus.ADOPTED)
+        # 3. the same post under a *different* name (extension fix, template
+        #    change, date rollover). The ledger starts empty on the first run
+        #    after such a change, so without this the whole archive re-downloads.
+        if adopt_existing and ledger is not None:
+            adopted = find_existing_for_post(dest, media.post_id, media.index)
+            if adopted is not None:
+                ledger.record(media.post_id, media.index, adopted)
+                return DownloadOutcome(adopted, DownloadStatus.ADOPTED)
+
+    # Anti rate-limit: wait cooldown before downloading (port background.js:789-792)
+    use_limiter = limiter if limiter is not None else _default_limiter
+    # allow disabling globally
+    should_wait = rate_limit and use_limiter.enabled
+    if should_wait:
+        use_limiter.wait()
+
+    close_client = False
+    if client is None:
+        client = _build_download_client(impersonate=impersonate, headers=CDN_HEADERS)
+        close_client = True
+
+    # Retry loop for transient network errors; writes go to a .part temp file
+    # so a failed attempt never leaves a truncated media at the final path.
+    import sys
+    import time as _time
+
+    tmp = out.with_name(out.name + ".part")
+    last_exc: BaseException | None = None
+    try:
+        for attempt in range(1, _DOWNLOAD_ATTEMPTS + 1):
+            try:
+                with client.stream("GET", media.url, headers=CDN_HEADERS) as r:
+                    r.raise_for_status()
+                    with tmp.open("wb") as f:
+                        for chunk in r.iter_bytes(chunk_size=8192):
+                            f.write(chunk)
+                tmp.replace(out)
+                # El cooldown de lote se cuenta solo tras una descarga exitosa.
+                if hasattr(use_limiter, "increment"):
+                    use_limiter.increment()
+                if ledger is not None:
+                    ledger.record(media.post_id, media.index, out)
+                return DownloadOutcome(out, DownloadStatus.DOWNLOADED)
+            except Exception as e:
+                last_exc = e
+                tmp.unlink(missing_ok=True)
+                if attempt < _DOWNLOAD_ATTEMPTS and is_transient_error(e):
+                    wait_s = _DOWNLOAD_BACKOFF_S[
+                        min(attempt - 1, len(_DOWNLOAD_BACKOFF_S) - 1)
+                    ]
+                    print(
+                        f"  {media.id} retrying ({attempt}/{_DOWNLOAD_ATTEMPTS - 1}) after "
+                        f"{wait_s:.0f}s: {e}",
+                        file=sys.stderr,
+                        flush=True,
+                    )
+                    _time.sleep(wait_s)
+                    continue
+                raise
+        raise RuntimeError("download loop exited without result") from last_exc
+    finally:
+        if close_client:
+            client.close()
+
+
+def download_profile_pic(
+    url: str,
+    username: str,
+    dest: str | Path,
+    *,
+    client: StreamClient | None = None,
+    impersonate: str | None = None,
+    limiter: BatchCooldownLimiter | None = None,
+    rate_limit: bool = True,
+    filename_template: str | None = None,
+    ledger: DownloadLedger | None = None,
+    adopt_existing: bool = True,
+) -> DownloadOutcome:
+    """Download profile pic with id f"{username}_profile"."""
+    # Derivar la extension con el mismo criterio que el resto del scraper,
+    # para que Media.ext y el archivo escrito coincidan.
+    from threadstractormf._parsing import _guess_ext
+    from threadstractormf.models import Media
+
+    ext = _guess_ext(url)
+    media = Media(
+        id=f"{sanitize_filename(username)}_profile",
+        post_id=f"{sanitize_filename(username)}_profile",
+        index=1,
+        type="image",
+        url=url,
+        ext=ext,
+    )
+    # profile uses filename_template if given, else {username}_{media_id}.{ext}
+    if filename_template:
+        return download_media(
+            media, dest, client=client, impersonate=impersonate, limiter=limiter,
+            rate_limit=rate_limit, filename_template=filename_template, username=username,
+            ledger=ledger, adopt_existing=adopt_existing,
+        )
+    return download_media(
+        media, dest, client=client, impersonate=impersonate, limiter=limiter,
+        rate_limit=rate_limit, ledger=ledger, adopt_existing=adopt_existing,
+    )

threadstractormf/models.py

@@ -0,0 +1,123 @@
+"""Pydantic models — map the extension post_metadata but with post_id naming.
+
+Extension (content.js:678-695):
+  username, datetime_iso, datetime_display, post_permalink, media_urls,
+  post_content, like_count, reply_count
+Library:
+  Post.id = post_id extracted from permalink /post/<ID> (regex content.js:311)
+  Media.id = post_id if single media, or f"{post_id}_{i}" for carousels (user spec)
+  Profile.profile_pic with id f"{username}_profile"
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field, field_validator
+
+MediaType = Literal["image", "video"]
+
+
+class Media(BaseModel):
+    id: str = Field(description="media_id: post_id or post_id_N for carousels")
+    post_id: str = Field(description="base post_id (/post/<ID>)")
+    index: int = Field(ge=1, description="1-index within the carousel")
+    type: MediaType
+    url: str
+    ext: str = Field(description="jpg|mp4|webp... derived from URL")
+    width: int | None = None
+    height: int | None = None
+
+    @field_validator("ext")
+    @classmethod
+    def normalize_ext(cls, v: str) -> str:
+        return v.lower().lstrip(".")
+
+    def filename(
+        self,
+        *,
+        prefix: str | None = None,
+        template: str | None = None,
+        username: str | None = None,
+        category: str = "threads",
+        subcategory: str = "posts",
+        date_iso: str | None = None,
+    ) -> str:
+        """Generate filename. If template is None uses simple post_id naming,
+        otherwise gallery-dl style template."""
+        if template:
+            from threadstractormf.template import render_filename
+
+            return render_filename(
+                template,
+                post_id=self.post_id,
+                media_id=self.id,
+                username=username or self.post_id,
+                category=category,
+                subcategory=subcategory,
+                extension=self.ext,
+                num=self.index,
+                date_iso=date_iso,
+            )
+        # For profile pics, id is already username_profile
+        if self.post_id.endswith("_profile"):
+            return f"{self.id}.{self.ext}"
+        # Detect carousel by id != post_id
+        if self.id != self.post_id:
+            return f"{self.id}.{self.ext}"
+        return f"{self.post_id}.{self.ext}"
+
+
+class Post(BaseModel):
+    id: str = Field(description="post_id = short code from /post/<ID>")
+    permalink: str
+    username: str
+    caption: str | None = None
+    datetime_iso: str | None = None
+    datetime_display: str | None = None
+    like_count: int = 0
+    reply_count: int = 0
+    media: list[Media] = Field(default_factory=list)
+
+    @property
+    def is_carousel(self) -> bool:
+        return len(self.media) > 1
+
+    def primary_media(self) -> Media | None:
+        return self.media[0] if self.media else None
+
+
+class Profile(BaseModel):
+    """What ``get_profile`` can actually resolve today.
+
+    Only fields the scraper fills in are declared: a field that always came back
+    ``None`` would be an API promise the library cannot keep. Bio, follower count
+    and privacy state are available in the page HTML but not parsed yet; add
+    them here together with the extractor that populates them.
+    """
+
+    username: str
+    user_id: str | None = None
+    profile_pic_url: str | None = None
+    profile_pic_media: Media | None = None
+
+
+# Helpers portados de background.js
+def extract_post_id(permalink: str) -> str | None:
+    """Extract post_id from permalink. Replicates content.js:311 regex /post/([^/]+)."""
+    import re
+
+    m = re.search(r"/post/([^/?#]+)", permalink)
+    return m.group(1) if m else None
+
+
+def sanitize_filename(name: str) -> str:
+    """Port of background.js:147-154 sanitizeFilename."""
+    import re
+
+    s = re.sub(r"[\/\\\?\*\|<>:\"\"\x00-\x1f\x7f]", "_", name)
+    s = s.replace("..", "_")
+    s = s.lstrip(".")
+    s = re.sub(r"_+", "_", s)
+    s = s.strip("_")
+    return s[:100]

threadstractormf/rate_limit.py

@@ -0,0 +1,201 @@
+"""Anti rate-limit — port of background.js cooldown + API backoff.
+
+background.js: cooldownMs=2000, cooldownAfter100=120000, lastCooldownMilestone
+Python: FixedCooldownLimiter + BatchCooldownLimiter + ExponentialBackoff
+Compatible with gallery-dl --sleep-request / yt-dlp --sleep-interval
+"""
+
+from __future__ import annotations
+
+import random
+import time
+from dataclasses import dataclass
+
+
+@dataclass
+class RateLimitConfig:
+    cooldown_ms: int = 2000  # between downloads (background.js:16)
+    cooldown_after_100_ms: int = 120000  # every 100 (background.js:17)
+    jitter: float = 0.2  # 20% random to avoid thundering herd
+    rps: float = 0.5  # for GraphQL API: 1 req / 2s
+    max_retries: int = 5
+    backoff_base: float = 2.0
+    enabled: bool = True
+
+
+class FixedCooldownLimiter:
+    """Wait cooldown_ms between consecutive calls (CDN downloads)."""
+
+    def __init__(self, cooldown_ms: int = 2000, jitter: float = 0.2, enabled: bool = True):
+        self.cooldown_ms = cooldown_ms
+        self.jitter = jitter
+        self.enabled = enabled
+        self._last: float | None = None
+
+    def wait(self) -> None:
+        if not self.enabled or self.cooldown_ms <= 0:
+            return
+        now = time.monotonic() * 1000
+        if self._last is None:
+            self._last = now
+            return
+        elapsed = now - self._last
+        wait_ms = self.cooldown_ms - elapsed
+        if wait_ms > 0:
+            # jitter ±
+            if self.jitter:
+                wait_ms *= 1 + random.uniform(-self.jitter, self.jitter)
+                wait_ms = max(0, wait_ms)
+            time.sleep(wait_ms / 1000)
+        self._last = time.monotonic() * 1000
+
+
+class BatchCooldownLimiter(FixedCooldownLimiter):
+    """Extends Fixed with a long cooldown every N downloads (background.js batch)."""
+
+    def __init__(
+        self,
+        cooldown_ms: int = 2000,
+        batch_size: int = 100,
+        batch_cooldown_ms: int = 120000,
+        jitter: float = 0.2,
+        enabled: bool = True,
+    ):
+        super().__init__(cooldown_ms, jitter, enabled)
+        self.batch_size = batch_size
+        self.batch_cooldown_ms = batch_cooldown_ms
+        self._count = 0
+        self._last_batch: int = 0
+
+    def increment(self) -> None:
+        self._count += 1
+        # if it is a batch multiple and not the same milestone
+        milestone = (self._count // self.batch_size) * self.batch_size
+        if milestone > 0 and milestone != self._last_batch and self._count % self.batch_size == 0:
+            self._last_batch = milestone
+            if self.enabled and self.batch_cooldown_ms > 0:
+                wait = self.batch_cooldown_ms / 1000
+                if self.jitter:
+                    wait *= 1 + random.uniform(-self.jitter, self.jitter)
+                time.sleep(max(0, wait))
+
+    def wait(self) -> None:
+        # Solo pacing: el conteo (y el cooldown de lote) se hace tras una
+        # descarga exitosa, para no penalizar intentos fallidos.
+        super().wait()
+
+
+class ApiRateLimiter:
+    """Paced limiter for GraphQL requests, with adaptive back-pressure.
+
+    Base pacing is a plain interval (``rps=0.5`` -> one request every 2s), which
+    is deliberately slow because Meta answers a burst with 429s and, eventually,
+    with a WAF block.
+
+    On top of that the limiter reacts to what the server actually says:
+
+    * a 429 **doubles** the interval (capped at ``max_interval``), so repeated
+      complaints space requests out exponentially instead of hammering;
+    * a streak of successful requests **relaxes** it back toward the base rate,
+      one step at a time, so a single 429 does not permanently slow a crawl.
+
+    Net effect: a full profile still gets walked to the end, but the request rate
+    backs off exactly as much as the server demands and recovers on its own.
+    """
+
+    def __init__(
+        self,
+        rps: float = 0.5,
+        enabled: bool = True,
+        *,
+        max_interval: float = 60.0,
+        penalty_factor: float = 2.0,
+        relax_after: int = 5,
+        relax_factor: float = 0.75,
+    ):
+        self.rps = rps
+        self.enabled = enabled
+        self.max_interval = max_interval
+        self.penalty_factor = penalty_factor
+        self.relax_after = relax_after
+        self.relax_factor = relax_factor
+        self._last: float | None = None
+        self._base_interval = 1.0 / rps if rps > 0 else 0.0
+        self._penalty = 1.0
+        self._ok_streak = 0
+
+    @property
+    def base_interval(self) -> float:
+        return self._base_interval
+
+    @property
+    def penalty(self) -> float:
+        return self._penalty
+
+    def current_interval(self) -> float:
+        """Effective seconds to wait between requests right now."""
+        if self._base_interval <= 0:
+            return 0.0
+        return min(self._base_interval * self._penalty, self.max_interval)
+
+    def _max_penalty(self) -> float:
+        if self._base_interval <= 0:
+            return 1.0
+        return self.max_interval / self._base_interval
+
+    def penalize(self) -> None:
+        """Call on 429 (or any rate-limit signal): back off harder."""
+        if not self.enabled:
+            return
+        self._penalty = min(self._penalty * self.penalty_factor, self._max_penalty())
+        self._ok_streak = 0
+
+    def relax(self) -> None:
+        """Call on success: decay the penalty back toward the base rate.
+
+        Decay is multiplicative so recovery from a deep penalty converges in a
+        handful of streaks. An additive step needed ~1160 successful requests to
+        climb back from the 60s cap, which in practice never happened.
+        """
+        if not self.enabled or self._penalty <= 1.0:
+            return
+        self._ok_streak += 1
+        if self._ok_streak >= self.relax_after:
+            self._penalty = max(1.0, self._penalty * self.relax_factor)
+            self._ok_streak = 0
+
+    def wait(self) -> None:
+        if not self.enabled or self._base_interval <= 0:
+            return
+        now = time.monotonic()
+        if self._last is None:
+            self._last = now
+            return
+        elapsed = now - self._last
+        wait = self.current_interval() - elapsed
+        if wait > 0:
+            time.sleep(wait)
+        self._last = time.monotonic()
+
+
+def backoff_sleep(
+    attempt: int, *, base: float = 2.0, max_sleep: float = 60.0, jitter: float = 0.3
+) -> float:
+    """Compute exponential sleep for 429/500. Returns seconds slept."""
+    sleep = min(max_sleep, (base**attempt))
+    if jitter:
+        sleep *= 1 + random.uniform(-jitter, jitter)
+    sleep = max(0, sleep)
+    time.sleep(sleep)
+    return sleep
+
+
+def parse_retry_after(headers: dict) -> float | None:
+    """Extract Retry-After if present (seconds)."""
+    for k, v in headers.items():
+        if k.lower() == "retry-after":
+            try:
+                return float(str(v).split(",")[0].strip())
+            except Exception:
+                return None
+    return None