threadstractormf 1.4.0__py3-none-any.whl

threadstractormf/archive.py

@@ -0,0 +1,262 @@
+"""Download ledger — remember what has already been fetched, by post id.
+
+Why this exists
+---------------
+The only deduplication this tool used to have was ``if out.exists()``: the
+*filename* was the key. That is fragile, because the filename is assembled from
+three pieces and only one of them is stable::
+
+    {date}__{post_id}__{num}.{extension}
+     unstable  STABLE     unstable  unstable
+
+* ``{extension}`` is derived from the URL, and a wrong guess silently renamed
+  whole categories of media between releases (a ``.webm`` saved as ``.jpg``, a
+  JPEG poster saved as ``.mp4``). Every such fix re-downloaded the user's entire
+  history.
+* ``{date}`` falls back to *today* when the post date is unknown, so the very
+  same post produced a different filename on the next day.
+
+``post_id`` and ``index`` come from the API and do not change while the post
+exists, so they make a dependable key. The ledger records them and the
+downloader consults it before touching the network.
+
+Storage
+-------
+Append-only JSONL, one object per completed download::
+
+    {"post_id": "DPxyz", "index": 1, "path": "...", "t": 1756000000}
+
+The shape mirrors scrapmf's own archive
+(``~/.config/scrapmf/archive/<site>/<account>.jsonl``), so the two can be
+unified later. The file lives inside the media directory
+(``<dest>/.archive/dedup.jsonl``) so that moving the library moves the record
+along with it.
+
+The ledger is opt-in: pass ``--archive`` (or ``archive=True``) to enable it.
+Without it nothing is written and no directory is created, so a plain download
+leaves no bookkeeping behind. Ledgers written by 1.1.0 and earlier, under
+``.threadstractormf/``, are migrated on first use.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import time
+from pathlib import Path
+from typing import Any
+
+# Hidden directory kept beside the media so it travels with an archive.
+LEDGER_DIRNAME = ".archive"
+LEDGER_FILENAME = "dedup.jsonl"
+
+# The directory used up to 1.1.0. It is renamed, not deleted: an existing
+# library must not re-download itself because the bookkeeping moved.
+LEGACY_LEDGER_DIRNAME = ".threadstractormf"
+
+
+def ledger_path_for(dest: str | Path) -> Path:
+    """Path of the ledger that covers ``dest``.
+
+    ``dest`` must be the *root* destination, not a ``photos/``/``videos/``
+    subdirectory, otherwise one account ends up with several ledgers.
+    """
+    return Path(dest).expanduser().resolve() / LEDGER_DIRNAME / LEDGER_FILENAME
+
+
+def migrate_legacy_ledger(dest: str | Path) -> bool:
+    """Move a pre-1.1.1 ``.threadstractormf/dedup.jsonl`` to ``.archive/``.
+
+    Returns True when a ledger was migrated. Losing it would make the next run
+    treat every already-downloaded file as new, so the rename has to carry the
+    history over rather than quietly starting empty.
+
+    The legacy directory is removed only when it is empty afterwards, so
+    anything else a future version stored there survives.
+    """
+    root = Path(dest).expanduser().resolve()
+    legacy = root / LEGACY_LEDGER_DIRNAME / LEDGER_FILENAME
+    # Rechazar symlinks: un enlace plantado como dedup.jsonl podria redirigir
+    # el rename o la lectura fuera del directorio de destino.
+    if not legacy.is_file() or legacy.is_symlink():
+        return False
+    target = ledger_path_for(root)
+    if target.is_symlink():
+        return False
+    if target.exists():
+        # Both present: keep the new one and merge the old lines into it, so
+        # neither set of downloads is forgotten.
+        try:
+            existing = set(target.read_text(encoding="utf-8").splitlines())
+            extra = [
+                line
+                for line in legacy.read_text(encoding="utf-8").splitlines()
+                if line.strip() and line.strip() not in existing
+            ]
+            if extra:
+                with target.open("a", encoding="utf-8") as fh:
+                    fh.write("\n".join(extra) + "\n")
+        except OSError:
+            return False
+    else:
+        try:
+            target.parent.mkdir(parents=True, exist_ok=True)
+            legacy.replace(target)
+        except OSError:
+            return False
+    try:
+        legacy.parent.rmdir()
+    except OSError:
+        pass  # not empty: leave it alone rather than delete unknown files
+    return True
+
+
+class DownloadLedger:
+    """Append-only set of ``(post_id, index)`` pairs already on disk.
+
+    Loading is eager but cheap: the file holds one short line per media item, so
+    even a few thousand posts parse in milliseconds. Writes are appended
+    immediately so an interrupted run still records what it finished.
+    """
+
+    def __init__(self, path: str | Path | None) -> None:
+        self.path = Path(path) if path is not None else None
+        self._keys: set[tuple[str, int]] = set()
+
+    # ── reading ─────────────────────────────────────────────────────────────
+
+    def load(self) -> None:
+        """Read the ledger into memory. Missing/corrupt lines are skipped."""
+        self._keys = set()
+        if self.path is None or not self.path.exists():
+            return
+        with self.path.open("r", encoding="utf-8") as fh:
+            for line in fh:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    entry = json.loads(line)
+                    key = _entry_key(entry)
+                except Exception:
+                    continue  # a torn or hand-edited line must not break a scrape
+                if key is not None:
+                    self._keys.add(key)
+
+    def has(self, post_id: str, index: int) -> bool:
+        return (post_id, index) in self._keys
+
+    def __len__(self) -> int:
+        return len(self._keys)
+
+    # ── writing ─────────────────────────────────────────────────────────────
+
+    def record(self, post_id: str, index: int, path: str | Path) -> None:
+        """Mark ``(post_id, index)`` as present on disk."""
+        if self.path is None:
+            return
+        key = (post_id, index)
+        if key in self._keys:
+            return
+        self._keys.add(key)
+        entry = {
+            "post_id": post_id,
+            "index": index,
+            "path": str(path),
+            "t": int(time.time()),
+        }
+        try:
+            self.path.parent.mkdir(parents=True, exist_ok=True)
+            with self.path.open("a", encoding="utf-8") as fh:
+                fh.write(json.dumps(entry, ensure_ascii=False) + "\n")
+            os.chmod(self.path, 0o600)
+        except OSError:
+            # An unwritable ledger must never abort a download: worst case the
+            # filename check keeps deduping as it always did.
+            pass
+
+
+def _entry_key(entry: Any) -> tuple[str, int] | None:
+    if not isinstance(entry, dict):
+        return None
+    post_id = entry.get("post_id")
+    if not isinstance(post_id, str) or not post_id:
+        return None
+    index = entry.get("index", 1)
+    if not isinstance(index, int):
+        return None
+    return (post_id, index)
+
+
+# ── legacy-file adoption ───────────────────────────────────────────────────
+#
+# When the extension (or any other part of the name) starts being computed
+# differently, files already on disk keep the *old* name. The ledger starts
+# empty at that point, so the plain existence check misses them. Matching on
+# the post id recovers them instead of downloading everything again.
+
+
+_DIR_LISTING_CACHE: dict[str, tuple[float, list[Path]]] = {}
+
+
+def _cached_dir_listing(directory: Path) -> list[Path]:
+    try:
+        key = str(directory.resolve())
+        mtime = directory.stat().st_mtime
+    except OSError:
+        return []
+    cached = _DIR_LISTING_CACHE.get(key)
+    if cached is not None and cached[0] == mtime:
+        return cached[1]
+    try:
+        entries = [p for p in directory.iterdir() if p.is_file()]
+    except OSError:
+        return []
+    _DIR_LISTING_CACHE[key] = (mtime, entries)
+    return entries
+
+
+def find_existing_for_post(dest: str | Path, post_id: str, index: int = 1) -> Path | None:
+    """Find an already-downloaded file for item ``index`` of ``post_id``.
+
+    Two things must line up, or carousel items overwrite each other's history:
+
+    * a delimiter after the id — end of name, ``.`` or ``_`` — so ``DPxyz123``
+      never adopts ``DPxyz1234``, which is a different post;
+    * the item number that follows, when the filename carries one. Every item of
+      a carousel shares the same ``post_id``; only ``{num}`` distinguishes them.
+      Without this check item 2 of a carousel adopts item 1's file and is never
+      downloaded at all.
+
+    A file with no number can only belong to item 1: that is a post whose media
+    is not part of a carousel, or the ``{post_id}.{ext}`` template.
+    """
+    directory = Path(dest)
+    if not directory.is_dir():
+        return None
+
+    numbered: Path | None = None
+    plain: Path | None = None
+    for candidate in sorted(_cached_dir_listing(directory)):
+        name = candidate.name
+        position = name.find(post_id)
+        if position == -1:
+            continue
+        # El post_id debe ser un identificador completo, no una subcadena de
+        # otro nombre (p. ej. "XP1.jpg" no debe adoptarse para post_id "P1").
+        if position > 0 and name[position - 1].isalnum():
+            continue
+        tail = name[position + len(post_id) :]
+        if tail[:1] not in ("", ".", "_"):
+            continue
+        match = re.match(r"^_(\d+)", tail)
+        if match:
+            if int(match.group(1)) == index and numbered is None:
+                numbered = candidate
+        elif index == 1 and plain is None:
+            plain = candidate
+
+    # An explicitly numbered file is what the current naming scheme produces, so
+    # it wins over a bare "{post_id}.{ext}" that also matches item 1.
+    return numbered or plain

threadstractormf/auth.py

@@ -0,0 +1,319 @@
+"""Auth via Netscape cookies.txt — 100% compatible with gallery-dl / yt-dlp.
+
+gallery-dl and yt-dlp load cookies with http.cookiejar.MozillaCookieJar
+which requires:
+  - Line 1: "# Netscape HTTP Cookie File" or "# HTTP Cookie File"
+  - 7 fields separated by TAB: domain\\tTRUE/FALSE\\tpath\\tTRUE/FALSE\\texpires\\tname\\tvalue
+  - Domain with "." prefix (e.g. .threads.net) = sent to subdomains (TRUE)
+  - Expiry 0 = session cookie
+
+This module replicates that behavior so the same cookies.txt works
+in threadstractormf, gallery-dl and yt-dlp without conversion.
+"""
+
+from __future__ import annotations
+
+import http.cookiejar
+import time
+from pathlib import Path
+from typing import Any
+
+import httpx
+
+
+def _parse_netscape_lines(path: Path) -> list[str]:
+    """Normalise a Netscape cookie file to TAB-separated lines, **in memory**.
+
+    Tolerates the things real exports contain:
+    - a ``scrapmf-source`` line above the MAGIC header
+    - spaces instead of TABs (many generators emit spaces)
+    - ``#HttpOnly_`` prefixes, which are stripped here and re-applied by
+      :func:`load_netscape_cookies`
+    - empty and comment lines, and malformed rows
+
+    Returns the normalised lines; nothing is written to disk. This used to build
+    a temp file purely because ``MozillaCookieJar.load()`` demands a filename,
+    which put live session cookies on disk as ``0o644``.
+    """
+    text = path.read_text(encoding="utf-8", errors="ignore")
+    lines = text.splitlines()
+    # Find MAGIC header
+    magic_idx = -1
+    for i, line in enumerate(lines):
+        if "HTTP Cookie File" in line:
+            magic_idx = i
+            break
+    if magic_idx == -1:
+        raise http.cookiejar.LoadError(f"{path} does not look like a Netscape format cookies file")
+    # Rebuild from MAGIC onwards. The header itself is NOT returned: it carries
+    # no cookie, only MozillaCookieJar needed it to validate the file, and
+    # ensure_netscape_header() already does that.
+    out_lines: list[str] = []
+    for raw_line in lines[magic_idx + 1 :]:
+        stripped = raw_line.strip()
+        if not stripped:
+            continue
+        if stripped.startswith("#") and "HTTP Cookie File" in stripped:
+            continue
+        if stripped.startswith("#") and not stripped.startswith("#HttpOnly_"):
+            # regular comment: keep pure comments?
+            # gallery-dl ignores them, we skip them
+            if stripped.startswith("# This"):
+                continue
+            # other comments: ignore
+            if stripped.startswith("#"):
+                # if it is #HttpOnly_, processed below
+                if not stripped.startswith("#HttpOnly_"):
+                    continue
+        # Keep the #HttpOnly_ prefix: it is a real cookie attribute and the
+        # consumer re-applies it as rest={"HttpOnly": ""}.
+        httponly = raw_line.startswith("#HttpOnly_")
+        line = raw_line[len("#HttpOnly_") :] if httponly else raw_line
+        # If line still starts with #, it is a comment -> skip
+        if line.strip().startswith("#"):
+            continue
+        # Split by TAB or by 2+ spaces
+        # MozillaCookieJar requires TABs, so we convert
+        # First split by TAB if tabs are present
+        if "\t" in line:
+            parts = line.split("\t")
+        else:
+            # split by whitespace 2+ or \s+ but preserving values with spaces?
+            # value may contain spaces / JSON -> use maxsplit=6
+            import re
+
+            parts = re.split(r"\s+", line.strip(), maxsplit=6)
+        if len(parts) != 7:
+            # malformed line, skip silently (gallery-dl does the same)
+            continue
+        domain, flag, cpath, secure, expires, name, value = parts
+        # Rebuild with TABs, re-attaching the prefix so the HttpOnly flag is
+        # not lost.
+        rebuilt = "\t".join([domain, flag, cpath, secure, expires, name, value])
+        out_lines.append(f"#HttpOnly_{rebuilt}" if httponly else rebuilt)
+    return out_lines
+
+
+def load_netscape_cookies(
+    path: str | Path, *, ignore_expires: bool = True, ignore_discard: bool = True
+) -> http.cookiejar.CookieJar:
+    """Load a Netscape cookies.txt and return a CookieJar ready for httpx.
+
+    Returns a CookieJar (compatible with ``httpx.Client(cookies=jar)``).
+    Tolerates scrapmf-source lines, spaces vs TABs, ``#HttpOnly_``, empty lines,
+    and rows whose include-subdomains flag disagrees with the domain's leading
+    dot. A CookieJar rather than ``httpx.Cookies`` avoids CookieConflict when the
+    same name exists on different domains (e.g. sessionid on tiktok +
+    instagram).
+
+    The file is parsed in memory. It used to be rewritten into a
+    ``tempfile.mktemp()`` file because ``MozillaCookieJar.load()`` requires a
+    filename, which left live session cookies on disk as world-readable
+    ``0o644`` and exposed a symlink race before the write.
+    """
+    p = Path(path).expanduser().resolve()
+    if not p.exists():
+        raise FileNotFoundError(f"cookies file not found: {p}")
+
+    # Surface the actionable message first: exporting cookies as JSON is by far
+    # the most common mistake, and the bare LoadError does not hint at it.
+    ensure_netscape_header(p)
+
+    jar = http.cookiejar.CookieJar()
+    now = time.time()
+    for raw in _parse_netscape_lines(p):
+        httponly = raw.startswith("#HttpOnly_")
+        if httponly:
+            raw = raw[len("#HttpOnly_") :]
+        domain, flag, cpath, secure, expires, name, value = raw.split("\t")
+
+        initial_dot = domain.startswith(".")
+        # MozillaCookieJar asserts domain_specified == initial_dot, so a file
+        # whose flag disagrees with its own domain aborted the entire load with
+        # an AssertionError. Derive the flag from the domain instead, which is
+        # what the Netscape format actually means.
+        domain_specified = flag == "TRUE" or initial_dot
+
+        # expires == "" and expires == "0" both mean "session cookie". Reading
+        # "0" as a timestamp would date it to 1970 and drop a live cookie.
+        try:
+            expires_at = int(expires) if expires not in ("", "0") else None
+        except ValueError:
+            expires_at = None
+        discard = expires_at is None
+        if ignore_discard:
+            discard = False
+        if not ignore_expires and expires_at is not None and expires_at < now:
+            continue
+
+        jar.set_cookie(
+            http.cookiejar.Cookie(
+                version=0,
+                name=name,
+                value=value,
+                port=None,
+                port_specified=False,
+                domain=domain,
+                domain_specified=domain_specified,
+                domain_initial_dot=initial_dot,
+                path=cpath,
+                # MozillaCookieJar assumes path_specified is false.
+                path_specified=False,
+                secure=secure == "TRUE",
+                expires=expires_at,
+                discard=discard,
+                comment=None,
+                comment_url=None,
+                rest={"HttpOnly": ""} if httponly else {},
+            )
+        )
+    return jar
+
+
+def load_cookies_dict(
+    cookies: dict[str, str], *, domain: str = ".threads.net"
+) -> http.cookiejar.CookieJar:
+    """Build a CookieJar from a plain ``{name: value}`` mapping.
+
+    Used by ``Threadscraper`` so every accepted ``cookies`` input ends up in the
+    same shape, which the Playwright cookie converter requires.
+    """
+    jar = http.cookiejar.CookieJar()
+    for k, v in cookies.items():
+        c = http.cookiejar.Cookie(
+            version=0,
+            name=k,
+            value=v,
+            port=None,
+            port_specified=False,
+            domain=domain,
+            domain_specified=True,
+            domain_initial_dot=domain.startswith("."),
+            path="/",
+            path_specified=True,
+            secure=True,
+            expires=None,
+            discard=False,
+            comment=None,
+            comment_url=None,
+            rest={},
+        )
+        jar.set_cookie(c)
+    return jar
+
+
+def get_csrf_token(cookies: http.cookiejar.CookieJar | httpx.Cookies) -> str | None:
+    """Extract csrftoken from cookies (required for x-csrftoken header in Threads GraphQL)."""
+    if isinstance(cookies, httpx.Cookies):
+        return cookies.get("csrftoken")
+    for c in cookies:
+        if getattr(c, "name", None) == "csrftoken":
+            value = getattr(c, "value", None)
+            return str(value) if value is not None else None
+    return None
+def _load_brave_profile(profile_dir: str, domain: str, browser_cookie3: Any) -> Any:
+    """Read cookies from a specific Brave profile (Origin, Beta, Nightly).
+
+    ``browser_cookie3.brave()`` only resolves the default ``Brave-Browser``
+    profile, so a live session kept in ``Brave-Origin`` is invisible to it. The
+    ``Brave`` class takes explicit paths, which is what this uses; the
+    ``Local State`` file next to it holds the encryption key.
+    """
+    import browser_cookie3 as _bc3  # noqa: F401  (resolved by caller)
+
+    root = Path.home() / ".config" / "BraveSoftware" / profile_dir
+    cookie_file = root / "Default" / "Cookies"
+    key_file = root / "Local State"
+    if not cookie_file.exists():
+        raise FileNotFoundError(
+            f"no Brave profile at {cookie_file}. "
+            f"Is '{profile_dir}' installed, or is this a non-Linux layout?"
+        )
+    return browser_cookie3.Brave(
+        str(cookie_file), domain_name=domain, key_file=str(key_file)
+    ).load()
+
+
+def _brave_variant(profile_dir: str, browser_cookie3: Any) -> Any:
+    """Build a loader bound to one Brave profile directory."""
+    return lambda domain: _load_brave_profile(profile_dir, domain, browser_cookie3)
+
+
+def load_from_browser(browser: str, *, domain: str = "threads.com") -> http.cookiejar.CookieJar:
+    """Load cookies directly from the browser (brave/chrome/firefox)
+    like gallery-dl --cookies-from-browser.
+
+    Requires: pip install browser-cookie3
+    On Linux copies the DB to /tmp to avoid database locked while the browser is open.
+    Note: browser-cookie3 may leave a temporary copy of the cookie DB in /tmp;
+    ensure /tmp is cleared on reboot or use a private temp dir.
+    """
+    try:
+        import browser_cookie3  # type: ignore
+    except ImportError as e:
+        raise ImportError(
+            "browser-cookie3 not installed. "
+            "pip install 'threadstractormf[browser]' or 'browser-cookie3'"
+        ) from e
+
+    b = browser.lower()
+
+    # Brave ships several independent profiles under the same config root, and
+    # browser_cookie3 only has a shortcut for the default one:
+    #
+    #   browser_cookie3.brave()  ->  BraveSoftware/Brave-Browser   (the *stale* one)
+    #
+    # A user whose live session lives in Brave-Origin therefore gets an old jar
+    # without any sign of it. The class accepts explicit paths, so each variant
+    # is pointed at its own profile directory.
+    loaders = {
+        "brave": lambda d: browser_cookie3.brave(domain_name=d),
+        "brave-browser": lambda d: browser_cookie3.brave(domain_name=d),
+        "brave-origin": _brave_variant("Brave-Origin", browser_cookie3),
+        "brave-nightly": _brave_variant("Brave-Browser-Nightly", browser_cookie3),
+        "brave-beta": _brave_variant("Brave-Beta", browser_cookie3),
+        "chrome": lambda d: browser_cookie3.chrome(domain_name=d),
+        "chromium": lambda d: browser_cookie3.chromium(domain_name=d),
+        "firefox": lambda d: browser_cookie3.firefox(domain_name=d),
+        "edge": lambda d: browser_cookie3.edge(domain_name=d),
+        "opera": lambda d: browser_cookie3.opera(domain_name=d),
+        "vivaldi": lambda d: browser_cookie3.vivaldi(domain_name=d),
+    }
+    loader = loaders.get(b)
+    if loader is None:
+        raise ValueError(
+            f"unsupported browser: {browser} (use {'|'.join(sorted(loaders))})"
+        )
+    jar: http.cookiejar.CookieJar = loader(domain)
+
+    # browser_cookie3 filters by the requested domain only, but Threads keeps
+    # the session split between threads.com and threads.net. Merge both for
+    # every browser — previously only Brave got the threads.net half, silently
+    # leaving Chrome/Firefox/Edge with an incomplete session.
+    if domain == "threads.com":
+        try:
+            for c in loader("threads.net"):
+                jar.set_cookie(c)
+        except Exception:
+            pass
+    return jar
+
+
+def ensure_netscape_header(path: str | Path) -> None:
+    """Validate that the file carries the Netscape MAGIC header.
+
+    The header is searched for anywhere in the file, not just on the first line,
+    matching :func:`_parse_netscape_lines`: some exporters (scrapmf among them)
+    prepend a source comment before the real header. Checking only line 1 would
+    reject files the parser is happy to read.
+    """
+    p = Path(path)
+    if not p.exists():
+        return
+    lines = p.read_text(encoding="utf-8", errors="ignore").splitlines()
+    if not any("HTTP Cookie File" in line for line in lines):
+        raise http.cookiejar.LoadError(
+            f"{p} does not look like a Netscape format cookies file. "
+            "Export with 'Get cookies.txt LOCALLY' (Netscape), not JSON. "
+            "The file must contain a line '# Netscape HTTP Cookie File'."
+        )