threadstractormf 1.4.0__py3-none-any.whl

threadstractormf/template.py

@@ -0,0 +1,123 @@
+"""Template rendering for filenames — compatible with gallery-dl / scrapmf.
+
+Supports variables: {post_id}, {media_id}, {username}, {category}, {subcategory},
+{extension}, {date:%Y-%m-%d}, {num}, {num:02d}
+and configurable ordering: "{date:%Y-%m-%d}_{post_id}_{num:02d}.{extension}" (chronological)
+or "{post_id}_{num:02d}.{extension}" (post_id only).
+"""
+
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+
+from threadstractormf.models import sanitize_filename
+
+
+def render_filename(
+    template: str,
+    *,
+    post_id: str,
+    media_id: str,
+    username: str,
+    category: str = "threads",
+    subcategory: str = "posts",
+    extension: str = "jpg",
+    num: int = 1,
+    date_iso: str | None = None,
+) -> str:
+    # parse date
+    dt: datetime | None = None
+    if date_iso:
+        try:
+            dt = datetime.fromisoformat(date_iso.replace("Z", "+00:00"))
+        except Exception:
+            dt = None
+    if dt is None:
+        dt = datetime.now(timezone.utc)
+
+    # handle {date:FORMAT} and {date}
+    def repl_date(m: re.Match) -> str:
+        fmt = m.group(1)
+        if fmt:
+            # fmt like %Y-%m-%d or %Y
+            try:
+                return dt.strftime(fmt)
+            except Exception:
+                return dt.strftime("%Y-%m-%d")
+        return dt.strftime("%Y-%m-%d")
+
+    # replace {date:%Y-%m-%d} or {date}
+    template = re.sub(r"\{date(?::([^}]+))?\}", repl_date, template)
+
+    # handle {num:02d} and {num}
+    def repl_num(m: re.Match) -> str:
+        fmt = m.group(1)
+        if fmt:
+            # support 02d, 2d, etc.
+            try:
+                # extract width
+                width_match = re.search(r"0?(\d+)d", fmt)
+                if width_match:
+                    width = int(width_match.group(1))
+                    return str(num).zfill(width)
+            except Exception:
+                pass
+            return str(num)
+        return str(num)
+
+    template = re.sub(r"\{num(?::([^}]+))?\}", repl_num, template)
+
+    # simple replacements
+    replacements = {
+        "{post_id}": post_id,
+        "{media_id}": media_id,
+        "{username}": username,
+        "{category}": category,
+        "{subcategory}": subcategory,
+        "{extension}": extension.lstrip("."),
+        "{ext}": extension.lstrip("."),
+    }
+    for k, v in replacements.items():
+        template = template.replace(k, v)
+
+    # handle {scrapmf_root} / {username} etc. already done
+    return template
+
+
+def render_directory(
+    template_parts: list[str] | str,
+    *,
+    username: str,
+    category: str = "threads",
+    subcategory: str = "posts",
+    scrapmf_root: str = "default",
+) -> str:
+    if isinstance(template_parts, str):
+        parts = [template_parts]
+    else:
+        parts = template_parts
+    # {username} comes from the CLI target, which the user types. It must be
+    # reduced to a single harmless path component: otherwise
+    # `--directory-template "{username}"` with the target `@..` resolves the
+    # destination to the parent of --dest, and files land outside it silently.
+    #
+    # sanitize_filename is reused rather than a second filter being written, so
+    # "untrusted text" has one definition in this package. It maps ".." and "."
+    # to "", turns slashes into "_", and collapses runs of underscores.
+    safe_user = sanitize_filename(username)
+    out: list[str] = []
+    for p in parts:
+        # simple
+        p = p.replace("{username}", safe_user)
+        p = p.replace("{user}", safe_user)
+        # {category}, {subcategory} and {scrapmf_root} are internal fixed values
+        # or orchestrator-supplied, so they are left alone: sanitising them would
+        # change the documented scrapmf layout for no gain.
+        p = p.replace("{category}", category)
+        p = p.replace("{subcategory}", subcategory)
+        p = p.replace("{scrapmf_root}", scrapmf_root)
+        # handle {scrapmf_root} legacy aliases
+        p = p.replace("{scarpmf_root}", scrapmf_root)
+        out.append(p)
+    return "/".join(out)

threadstractormf-1.4.0.dist-info/METADATA

@@ -0,0 +1,163 @@
+Metadata-Version: 2.5
+Name: threadstractormf
+Version: 1.4.0
+Summary: Python library for Threads - media download by post_id with Netscape cookies auth (gallery-dl compatible)
+Project-URL: Homepage, https://github.com/ExtractorsMF/ThreadstractorMF
+Project-URL: Repository, https://github.com/ExtractorsMF/ThreadstractorMF
+Author: ExtractorsMF
+License: GPL-3.0-only
+License-File: LICENSE
+Keywords: downloader,gallery-dl,scraper,threads,yt-dlp
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Requires-Python: >=3.10
+Requires-Dist: httpx[http2]>=0.27
+Requires-Dist: pydantic>=2.6
+Requires-Dist: rich>=13
+Requires-Dist: typer>=0.12
+Provides-Extra: antibot
+Requires-Dist: curl-cffi>=0.7; extra == 'antibot'
+Provides-Extra: browser
+Requires-Dist: browser-cookie3>=0.20; extra == 'browser'
+Requires-Dist: playwright>=1.43; extra == 'browser'
+Provides-Extra: dev
+Requires-Dist: gitpython<3.1.60; extra == 'dev'
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest>=8; extra == 'dev'
+Requires-Dist: python-semantic-release<10,>=9; extra == 'dev'
+Requires-Dist: respx>=0.22; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# threadstractormf
+
+Threads media downloader — Python library + CLI, gallery-dl/yt-dlp compatible cookies.
+
+## Project memory
+
+- Index: this README + `docs/` + `docs/memory/`.
+- Compact memory: `docs/memory/architecture.md`, `docs/memory/decisions.md`, `docs/memory/state.md`, `docs/memory/conventions.md`, `docs/memory/testing.md`.
+- Detailed docs: `docs/ARCHITECTURE.md`, `docs/DEVELOPMENT.md`, `docs/SECURITY.md`, `docs/PRIVACY.md`, `docs/GIT_WORKFLOW.md`, `docs/DECISIONS.md`.
+- LLM indexes: `llms.txt`, `llms-full.txt`.
+- Agent instructions: `AGENTS.md`.
+- Last documentation review: 2026-11-03.
+
+## Stack
+
+Python ≥ 3.10, `httpx[http2]`, `pydantic`, `typer`, `rich`. Optional: `curl_cffi` (antibot), `playwright` + `browser-cookie3` (browser).
+
+## Install
+
+```bash
+uv pip install -e ".[dev]"          # dev
+uv pip install -e ".[antibot]"      # TLS impersonation
+uv pip install -e ".[browser]"      # Playwright + cookies
+```
+
+## Quickstart
+
+Library:
+
+```python
+from threadstractormf import Threadscraper
+scraper = Threadscraper(cookies="cookies.txt")
+posts = scraper.get_posts("user", limit=20)
+for post in posts:
+    for media in post.media:
+        scraper.download(media, dest="./dl", username=post.username, date_iso=post.datetime_iso)
+scraper.download_profile_pic("user", dest="./dl")
+```
+
+CLI:
+
+```bash
+threadstractormf --cookies cookies.txt --dest ./dl @user
+threadstractormf --cookies-from-browser brave --dest ./dl @user
+threadstractormf --cookies cookies.txt --limit 50 --dest ./dl @user
+threadstractormf --cookies cookies.txt --get-urls --dest ./dl @user
+threadstractormf --version
+```
+
+## Modules
+
+| Module | Responsibility |
+|---|---|
+| client.py | Facade, wiring, download orchestration |
+| api.py | GraphQL walk + Playwright DOM fallback |
+| _discovery.py | Capture/cache GraphQL template |
+| _backend.py | httpx/curl_cffi backends |
+| auth.py | Netscape/browser cookies |
+| models.py | Post/Media/Profile |
+| template.py | Filename/directory rendering |
+| rate_limit.py | Cooldowns + backoff |
+| archive.py | Download ledger |
+| downloader.py | CDN download + adoption |
+| cli.py | Typer CLI |
+
+## Docker
+
+Published to GitHub Container Registry (GHCR) on every push to `main`:
+
+```bash
+# slim (~150 MB): core only, fast GraphQL path
+docker pull ghcr.io/extractorsmf/threadstractormf:slim
+
+# full (~1 GB): slim + antibot (curl_cffi) + browser (Playwright)
+docker pull ghcr.io/extractorsmf/threadstractormf:full   # also tagged :latest
+docker pull ghcr.io/extractorsmf/threadstractormf:1.2.0
+```
+
+Run (mount cookies and destination as volumes; never bake secrets into the image):
+
+```bash
+docker run --rm \\
+  -v $(pwd)/cookies.txt:/cookies.txt:ro \\
+  -v $(pwd)/dl:/dl \\
+  ghcr.io/extractorsmf/threadstractormf:full \\
+  --cookies /cookies.txt --dest /dl @user
+```
+
+- `THREADS_USE_SANDBOX=1` is set in the `full` image so Playwright works unprivileged.
+- Multi-arch: `linux/amd64`, `linux/arm64`.
+
+## Install from PyPI
+
+```bash
+pip install threadstractormf
+pip install 'threadstractormf[browser]'
+pip install 'threadstractormf[antibot]'
+```
+
+## Essential commands
+
+```bash
+uv run pytest -q
+uv run ruff check .
+uv run mypy threadstractormf scripts
+```
+
+## Key constraints
+
+- Ledger opt-in (`--archive`); keyed by `(post_id, index)`.
+- Browser fallback runs anonymously.
+- GraphQL doc_id is discovered, not hardcoded.
+- stdout = one line per downloaded file; diagnostics on stderr.
+- Do not commit secrets or personal identities.
+
+## More details
+
+- Usage/CLI details: `docs/DEVELOPMENT.md` (commands), `docs/ARCHITECTURE.md` (flow).
+- Security/privacy: `docs/SECURITY.md`, `docs/PRIVACY.md`.
+- Git/commits: `docs/GIT_WORKFLOW.md`.
+- Decisions: `docs/DECISIONS.md`.
+- Known issues/state: `docs/memory/state.md`.
+
+## Status
+
+408 tests pass; ruff and mypy clean. Known gaps: duplicate summary line, library `archive` default, relay flag override, profile-pic ext derivation. See `docs/memory/state.md`.

threadstractormf-1.4.0.dist-info/RECORD

@@ -0,0 +1,18 @@
+threadstractormf/__init__.py,sha256=jylkYQvBAftktLeKDF94k2fdsWdLy_SVwjSyBbUj7ao,269
+threadstractormf/_backend.py,sha256=G4bQzoR9gBJfMgDJN14rxJf9eFKxlcE7Ut1rlSEVtZU,11185
+threadstractormf/_discovery.py,sha256=_p0_rUEg6MFaQNgerN1mgOHC77wjTylPzzjy59JRS2A,16982
+threadstractormf/_parsing.py,sha256=2ALeN9WkHfakQgBOrGAWjTDHJsfJkE_Vmqg4Z1-_8mU,9501
+threadstractormf/api.py,sha256=Oy58BK-X2Juv3Mp-uhcfeodGivLPw5UZbOPINz3v0ug,58683
+threadstractormf/archive.py,sha256=8sQVZpZjjOt5lFbTsvvEQOlyhcc6KzMOp5QgvHVgMZU,10187
+threadstractormf/auth.py,sha256=1oiI72O_KQItk6-Pw7Xp9SWhVc21GnS_iGOiCv8Mvnc,13310
+threadstractormf/cli.py,sha256=EtbKGtv7sXPVlAffkYD5cXS4rh2iCewq3JEuSiRBl5o,17361
+threadstractormf/client.py,sha256=2J40HZchUNFOhezymqJRkONPdxkYVFPv9y2hJdT2qyE,7690
+threadstractormf/downloader.py,sha256=nlHsYTNQ1fsA1d4nQ2g7RLRHdR1Rv_h6PiRoqI_uT2M,13931
+threadstractormf/models.py,sha256=uxGOhInfYoOA0vFBGuzYKWY6eHkGVRczuXIZkblngko,3958
+threadstractormf/rate_limit.py,sha256=HhEpjAXB7X9HBBfYMidKFw0DAAY3gOyU__3H0Zs4Kj0,6961
+threadstractormf/template.py,sha256=VXu4kOfDyuhf617m13klKfFtkpeJSmFlLWnzteehwWo,4040
+threadstractormf-1.4.0.dist-info/METADATA,sha256=7V46N1XIN1uJgkKnygsNmBZNidDZSnItCnq72C_Rv10,5464
+threadstractormf-1.4.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
+threadstractormf-1.4.0.dist-info/entry_points.txt,sha256=4E6c_W81kApVVt_TEqY96GE02w9NAFoGQ2xREBvoiLs,62
+threadstractormf-1.4.0.dist-info/licenses/LICENSE,sha256=OXLcl0T2SZ8Pmy2_dmlvKuetivmyPd5m1q-Gyd-zaYY,35149
+threadstractormf-1.4.0.dist-info/RECORD,,

threadstractormf-1.4.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.33.0
+Root-Is-Purelib: true
+Tag: py3-none-any

threadstractormf-1.4.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+threadstractormf = threadstractormf.cli:app