pyXcom 0.6.0__py3-none-any.whl

pyxcom/storage.py

@@ -0,0 +1,152 @@
+"""Resumable public-post exports with auditable state and hashes."""
+
+import csv
+import gzip
+import hashlib
+import json
+import os
+from dataclasses import fields
+from pathlib import Path
+
+from .layout import internal_dir, migrate_collection
+from .models import CollectionResult, Post
+from .transport import now_utc
+from .tables import export_tables
+
+
+def _atomic_json(path: Path, value: dict) -> None:
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(
+        json.dumps(value, ensure_ascii=False, indent=2), encoding="utf-8"
+    )
+    os.replace(temporary, path)
+
+
+class PostStore:
+    def __init__(self, output_dir: str | Path, *, query: dict) -> None:
+        self.output_dir = Path(output_dir).expanduser()
+        migrate_collection(self.output_dir)
+        self._internal = internal_dir(self.output_dir)
+        self._jsonl = self._internal / "posts.jsonl"
+        self._state_path = self._internal / "state.json"
+        if self._state_path.exists():
+            self.state = json.loads(self._state_path.read_text(encoding="utf-8"))
+            if self.state.get("query") != query:
+                raise ValueError("Output directory belongs to a different query")
+        else:
+            self.state = {
+                "query": query,
+                "cursor": None,
+                "pages_fetched": 0,
+                "complete": False,
+                "reason": "not_started",
+                "missing_ids": [],
+            }
+        self._posts: dict[str, Post] = {}
+        if self._jsonl.exists():
+            for line in self._jsonl.read_text(encoding="utf-8").split("\n"):
+                if line.strip():
+                    post = Post(**json.loads(line))
+                    self._posts[post.id] = post
+
+    @property
+    def count(self) -> int:
+        return len(self._posts)
+
+    @property
+    def cursor(self) -> str | None:
+        return self.state["cursor"]
+
+    @property
+    def complete(self) -> bool:
+        return self.state["complete"]
+
+    def append_page(
+        self, posts: list[Post], next_cursor: str | None, *, count_page: bool = True
+    ) -> int:
+        added = 0
+        with self._jsonl.open("a", encoding="utf-8") as file:
+            for post in posts:
+                if post.id in self._posts:
+                    continue
+                file.write(json.dumps(post.to_dict(), ensure_ascii=True) + "\n")
+                self._posts[post.id] = post
+                added += 1
+            file.flush()
+            os.fsync(file.fileno())
+        self.state.update(
+            cursor=next_cursor,
+            pages_fetched=self.state["pages_fetched"] + int(count_page),
+            reason="in_progress",
+        )
+        _atomic_json(self._state_path, self.state)
+        return added
+
+    def retain_role(self, role: str) -> None:
+        """Normalize a source timeline while preserving its original observations."""
+        kept = {
+            key: post for key, post in self._posts.items() if post.post_role == role
+        }
+        if len(kept) == len(self._posts):
+            return
+        backup = self._internal / "posts.before-timeline-filter.jsonl.gz"
+        if not backup.exists():
+            with gzip.open(backup, "wb") as stream:
+                stream.write(self._jsonl.read_bytes())
+        temporary = self._jsonl.with_suffix(".jsonl.tmp")
+        with temporary.open("w", encoding="utf-8") as stream:
+            for post in kept.values():
+                stream.write(json.dumps(post.to_dict(), ensure_ascii=True) + "\n")
+            stream.flush()
+            os.fsync(stream.fileno())
+        os.replace(temporary, self._jsonl)
+        self._posts = kept
+
+    def add_missing_ids(self, ids: set[str]) -> None:
+        self.state["missing_ids"] = sorted(
+            set(self.state.get("missing_ids", [])) | ids,
+            key=int,
+        )
+        _atomic_json(self._state_path, self.state)
+
+    def finish(self, *, complete: bool, reason: str) -> CollectionResult:
+        self.state.update(complete=complete, reason=reason, updated_at_utc=now_utc())
+        _atomic_json(self._state_path, self.state)
+        self._jsonl.touch(exist_ok=True)
+        csv_path = self._internal / "posts.csv"
+        rows = sorted(
+            self._posts.values(), key=lambda post: (post.created_at_utc, post.id)
+        )
+        names = [item.name for item in fields(Post)]
+        with csv_path.open("w", encoding="utf-8-sig", newline="") as file:
+            writer = csv.DictWriter(file, fieldnames=names)
+            writer.writeheader()
+            for post in rows:
+                row = post.to_dict()
+                for key in ("media_urls", "outbound_urls", "hashtags", "mentions"):
+                    row[key] = json.dumps(row[key], ensure_ascii=False)
+                writer.writerow(row)
+        manifest = {
+            "query": self.state["query"],
+            "post_count": len(rows),
+            "pages_fetched": self.state["pages_fetched"],
+            "complete": complete,
+            "reason": reason,
+            "missing_ids": self.state.get("missing_ids", []),
+            "oldest_post_utc": rows[0].created_at_utc if rows else None,
+            "newest_post_utc": rows[-1].created_at_utc if rows else None,
+            "captured_at_utc": now_utc(),
+            "sha256": {
+                "posts.jsonl": hashlib.sha256(self._jsonl.read_bytes()).hexdigest(),
+                "posts.csv": hashlib.sha256(csv_path.read_bytes()).hexdigest(),
+            },
+        }
+        _atomic_json(self._internal / "manifest.json", manifest)
+        export_tables(self.output_dir)
+        return CollectionResult(
+            output_dir=self.output_dir,
+            post_count=len(rows),
+            pages_fetched=self.state["pages_fetched"],
+            complete=complete,
+            reason=reason,
+        )

pyxcom/tables.py

@@ -0,0 +1,308 @@
+"""Relational views of saved observations; never infer missing conversations."""
+
+import csv
+import hashlib
+import json
+import os
+from dataclasses import fields
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+from .layout import migrate_collection, source_path
+from .models import Post, Profile
+
+_CONTENT = [
+    item.name
+    for item in fields(Post)
+    if item.name
+    not in {
+        "id",
+        "author_id",
+        "author_handle",
+        "post_role",
+        "post_type",
+        "conversation_id",
+        "in_reply_to_id",
+        "reposted_post_id",
+    }
+]
+_USER = (
+    ["user_id", "username", "display_name"]
+    + [
+        item.name
+        for item in fields(Profile)
+        if item.name not in {"id", "handle", "name"}
+    ]
+    + ["profile_available"]
+)
+_POST = ["post_id", "author_id", "post_type"] + _CONTENT
+_COMMENT = [
+    "comment_id",
+    "author_id",
+    "root_post_id",
+    "parent_post_id",
+    "depth",
+    "depth_status",
+    "parent_in_dataset",
+    "root_in_dataset",
+] + _CONTENT
+_INTERACTION = [
+    "interaction_id",
+    "author_id",
+    "interaction_type",
+    "target_post_id",
+] + _CONTENT
+
+
+def _hash(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def _write_csv(path: Path, columns: list[str], rows: list[dict]) -> None:
+    with NamedTemporaryFile(
+        "w", encoding="utf-8-sig", newline="", dir=path.parent, delete=False
+    ) as stream:
+        temporary = Path(stream.name)
+        try:
+            writer = csv.DictWriter(stream, fieldnames=columns)
+            writer.writeheader()
+            for row in rows:
+                writer.writerow(
+                    {
+                        key: json.dumps(value, ensure_ascii=False)
+                        if isinstance(value, (list, dict))
+                        else value
+                        for key, value in row.items()
+                    }
+                )
+            stream.flush()
+            os.fsync(stream.fileno())
+        except BaseException:
+            temporary.unlink(missing_ok=True)
+            raise
+    os.replace(temporary, path)
+
+
+def _depths(records: dict[str, Post]) -> dict[str, tuple[int | None, str]]:
+    """Iterative traversal avoids stack limits, including very long reply threads."""
+    resolved: dict[str, tuple[int | None, str]] = {}
+    for origin in records.values():
+        if origin.post_role != "comment" or origin.id in resolved:
+            continue
+        trail: list[str] = []
+        seen: set[str] = set()
+        current = origin
+        root = origin.conversation_id
+        while True:
+            if current.id in seen:
+                raise ValueError(f"Reply ancestry cycle at post {current.id}")
+            if current.id in resolved:
+                base, status = resolved[current.id]
+                break
+            seen.add(current.id)
+            trail.append(current.id)
+            if not root:
+                base, status = None, "missing_root_id"
+                break
+            if current.conversation_id != root:
+                base, status = None, "inconsistent_root"
+                break
+            parent = current.in_reply_to_id
+            if parent == current.id:
+                raise ValueError(f"Reply ancestry cycle at post {current.id}")
+            if parent == root:
+                if parent in records and records[parent].post_role != "main":
+                    raise ValueError(f"Conversation root {root} is not a main post")
+                base, status = 0, "known"
+                break
+            if parent not in records:
+                base, status = None, "missing_parent"
+                break
+            ancestor = records[parent]
+            if ancestor.post_role != "comment":
+                base, status = None, "inconsistent_root"
+                break
+            if ancestor.conversation_id != root:
+                base, status = None, "inconsistent_root"
+                break
+            current = ancestor
+        for post_id in reversed(trail):
+            base = base + 1 if base is not None else None
+            resolved[post_id] = (base, status)
+    return resolved
+
+
+def export_tables(output_dir: str | Path) -> dict:
+    """Write public relational CSV tables at the collection root."""
+    directory = Path(output_dir).expanduser()
+    migrate_collection(directory)
+    source = source_path(directory, "posts.jsonl")
+    records: dict[str, Post] = {}
+    for line in source.read_text(encoding="utf-8").split("\n"):
+        if not line.strip():
+            continue
+        data = json.loads(line)
+        for key in (
+            "id",
+            "author_id",
+            "conversation_id",
+            "in_reply_to_id",
+            "quoted_post_id",
+            "reposted_post_id",
+        ):
+            if data.get(key) is not None:
+                data[key] = str(data[key])
+        post = Post(**data)
+        if not post.id or not post.author_id:
+            raise ValueError("Post and author IDs must be nonempty")
+        if post.id in records and records[post.id] != post:
+            raise ValueError(f"Conflicting duplicate post ID: {post.id}")
+        records[post.id] = post
+    depths = _depths(records)
+    profiles_path = source_path(directory, "profiles.json")
+    profiles: dict[str, dict] = {}
+    if profiles_path.exists():
+        raw = json.loads(profiles_path.read_text(encoding="utf-8"))
+        entries = (
+            raw if isinstance(raw, list) else ([raw] if "id" in raw else raw.values())
+        )
+        for entry in entries:
+            user_id = str(entry["id"])
+            if user_id in profiles and profiles[user_id] != entry:
+                raise ValueError(f"Conflicting duplicate profile ID: {user_id}")
+            profiles[user_id] = entry
+    users: dict[str, dict] = {
+        user_id: {
+            "user_id": user_id,
+            "username": profile.get("handle"),
+            "display_name": profile.get("name"),
+            **{key: profile.get(key) for key in _USER[3:-1]},
+            "profile_available": True,
+        }
+        for user_id, profile in profiles.items()
+    }
+    posts, comments, interactions = [], [], []
+    main_ids = {p.id for p in records.values() if p.post_role == "main"}
+    content_ids = {p.id for p in records.values() if p.post_role != "repost"}
+    for post in sorted(records.values(), key=lambda p: (p.created_at_utc, p.id)):
+        profile = profiles.get(post.author_id)
+        users[post.author_id] = {
+            "user_id": post.author_id,
+            "username": profile.get("handle") if profile else post.author_handle,
+            "display_name": profile.get("name") if profile else None,
+            **{key: profile.get(key) if profile else None for key in _USER[3:-1]},
+            "profile_available": profile is not None,
+        }
+        data = post.to_dict()
+        payload = {key: data[key] for key in _CONTENT}
+        if post.post_role == "comment":
+            depth, status = depths[post.id]
+            comments.append(
+                {
+                    "comment_id": post.id,
+                    "author_id": post.author_id,
+                    "root_post_id": post.conversation_id,
+                    "parent_post_id": post.in_reply_to_id,
+                    "depth": depth,
+                    "depth_status": status,
+                    "parent_in_dataset": post.in_reply_to_id in content_ids,
+                    "root_in_dataset": post.conversation_id in main_ids,
+                    **payload,
+                }
+            )
+        elif post.post_role == "repost":
+            interactions.append(
+                {
+                    "interaction_id": post.id,
+                    "author_id": post.author_id,
+                    "interaction_type": "repost",
+                    "target_post_id": post.reposted_post_id,
+                    **payload,
+                }
+            )
+        else:
+            posts.append(
+                {
+                    "post_id": post.id,
+                    "author_id": post.author_id,
+                    "post_type": post.post_type,
+                    **payload,
+                }
+            )
+    target = directory
+    target.mkdir(exist_ok=True)
+    tables = {
+        "users.csv": (_USER, list(users.values())),
+        "posts.csv": (_POST, posts),
+        "comments.csv": (_COMMENT, comments),
+    }
+    if interactions:
+        tables["interactions.csv"] = (_INTERACTION, interactions)
+    for name, (columns, rows) in tables.items():
+        _write_csv(target / name, columns, rows)
+    if not interactions:
+        (target / "interactions.csv").unlink(missing_ok=True)
+    manifest: dict = {
+        "schema_version": "1.1",
+        "layout_version": "2.0",
+        "source_kind": "saved_post_observations",
+        "source_sha256": {
+            str(source.relative_to(directory)): _hash(source),
+            **(
+                {str(profiles_path.relative_to(directory)): _hash(profiles_path)}
+                if profiles_path.exists()
+                else {}
+            ),
+        },
+        "counts": {
+            "users": len(users),
+            "posts": len(posts),
+            "comments": len(comments),
+            "interactions": len(interactions),
+            "source_records": len(records),
+        },
+        "missing_root_count": sum(not row["root_in_dataset"] for row in comments),
+        "missing_parent_count": sum(not row["parent_in_dataset"] for row in comments),
+        "unknown_depth_count": sum(row["depth"] is None for row in comments),
+        "conversation_coverage": "observed_records_only",
+        "sha256": {name: _hash(target / name) for name in tables},
+    }
+    collection_path = source_path(directory, "manifest.json")
+    if collection_path.exists() and collection_path != target / "manifest.json":
+        collection = json.loads(collection_path.read_text(encoding="utf-8"))
+        for key in (
+            "query",
+            "pages_fetched",
+            "complete",
+            "reason",
+            "captured_at_utc",
+            "oldest_post_utc",
+            "newest_post_utc",
+            "missing_ids",
+        ):
+            if key in collection:
+                manifest[key] = collection[key]
+    state_path = source_path(directory, "state.json")
+    if manifest.get("query", {}).get("kind") == "post_comments" and state_path.exists():
+        state = json.loads(state_path.read_text(encoding="utf-8")).get(
+            "conversation", {}
+        )
+        manifest["comment_collection"] = {
+            "max_depth": manifest["query"]["max_depth"],
+            "pending_requests": len(state.get("queue", [])),
+            "unresolved_or_deferred_records": len(state.get("pending", {})),
+            "pagination_warnings": sorted(set(state.get("pagination_warnings", []))),
+            "depth_counts": {
+                str(depth): sum(row["depth"] == depth for row in comments)
+                for depth in sorted(
+                    {row["depth"] for row in comments if row["depth"] is not None}
+                )
+            },
+        }
+    with NamedTemporaryFile("w", encoding="utf-8", dir=target, delete=False) as stream:
+        temporary = Path(stream.name)
+        json.dump(manifest, stream, ensure_ascii=False, indent=2)
+        stream.flush()
+        os.fsync(stream.fileno())
+    os.replace(temporary, target / "manifest.json")
+    return manifest

pyxcom/transport.py

@@ -0,0 +1,238 @@
+"""Cookie-authenticated, read-only requests to X's current web endpoints."""
+
+import json
+import re
+import time
+from datetime import datetime, timezone
+from urllib.parse import urlsplit
+import httpx
+
+from .errors import APIError, RateLimitError
+from .models import Post, Profile
+from .parse import parse_post, parse_profile, single_post
+
+_MAIN_SCRIPT = re.compile(
+    r"https://abs\.twimg\.com/responsive-web/client-web/main\.[^\"\s<>]+\.js"
+)
+_OPERATION = re.compile(r'queryId:"([^\"]+)",operationName:"([^\"]+)"')
+_BEARER = re.compile(r"Bearer (A[A-Za-z0-9%]+)")
+TWEET_FEATURES = {
+    "view_counts_everywhere_api_enabled": True,
+    "longform_notetweets_consumption_enabled": True,
+    "longform_notetweets_rich_text_read_enabled": True,
+}
+
+
+def now_utc() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+class XTransport:
+    """Discover query IDs from X's web bundle instead of pinning twikit routes."""
+
+    def __init__(
+        self, cookies: dict[str, str], *, proxy: str | None = None, timeout: float = 30
+    ) -> None:
+        cookie_jar = httpx.Cookies()
+        for name, value in cookies.items():
+            cookie_jar.set(name, value, domain=".x.com", path="/")
+        self._csrf = cookies["ct0"]
+        self._x = httpx.Client(
+            proxy=proxy, cookies=cookie_jar, timeout=timeout, follow_redirects=True
+        )
+        self._assets = httpx.Client(proxy=proxy, timeout=timeout, follow_redirects=True)
+        self._operations: dict[str, str] = {}
+        self._bearer: str | None = None
+
+    def close(self) -> None:
+        self._x.close()
+        self._assets.close()
+
+    def _get(self, client: httpx.Client, url: str, **kwargs) -> httpx.Response:
+        """Retry transient transport/5xx errors, then surface a safe message."""
+        for attempt in range(3):
+            try:
+                response = client.get(url, **kwargs)
+            except httpx.RequestError as exc:
+                if attempt == 2:
+                    raise APIError(
+                        f"Network request to {urlsplit(url).netloc} failed after 3 attempts"
+                    ) from exc
+            else:
+                if response.status_code not in {502, 503, 504} or attempt == 2:
+                    return response
+            time.sleep(2**attempt)
+        raise APIError("Network retry loop ended unexpectedly")
+
+    def discover(self, *, refresh: bool = False) -> None:
+        if self._operations and not refresh:
+            return
+        response = self._get(self._x, "https://x.com/")
+        if response.status_code != 200:
+            raise APIError(f"X homepage returned HTTP {response.status_code}")
+        match = _MAIN_SCRIPT.search(response.text)
+        if not match:
+            raise APIError(
+                "X web bundle was not found; the site layout may have changed"
+            )
+        js_response = self._get(self._assets, match.group(0))
+        if js_response.status_code != 200:
+            raise APIError(f"X web bundle returned HTTP {js_response.status_code}")
+        operations = {
+            name: query_id for query_id, name in _OPERATION.findall(js_response.text)
+        }
+        bearer = _BEARER.search(js_response.text)
+        if not operations or not bearer:
+            raise APIError("X web bundle no longer exposes the expected read routes")
+        self._operations = operations
+        self._bearer = bearer.group(1)
+
+    def _headers(self) -> dict[str, str]:
+        return {
+            "authorization": f"Bearer {self._bearer}",
+            "x-csrf-token": self._csrf,
+            "x-twitter-auth-type": "OAuth2Session",
+            "x-twitter-active-user": "yes",
+            "x-twitter-client-language": "en",
+            "referer": "https://x.com/",
+            "user-agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) "
+                "Chrome/122.0.0.0 Safari/537.36"
+            ),
+        }
+
+    def graphql(
+        self,
+        operation: str,
+        variables: dict,
+        *,
+        features: dict | None = None,
+        field_toggles: dict | None = None,
+    ) -> dict:
+        self.discover()
+        if operation not in self._operations:
+            raise APIError(f"X web bundle has no {operation} operation")
+        params = {
+            "variables": json.dumps(variables, separators=(",", ":")),
+            "features": json.dumps(features or {}, separators=(",", ":")),
+        }
+        if field_toggles is not None:
+            params["fieldToggles"] = json.dumps(field_toggles, separators=(",", ":"))
+        query_id = self._operations[operation]
+        url = f"https://x.com/i/api/graphql/{query_id}/{operation}"
+        response = self._get(self._x, url, params=params, headers=self._headers())
+        if response.status_code == 429:
+            reset = response.headers.get("x-rate-limit-reset")
+            raise RateLimitError(
+                f"X rate-limited {operation}",
+                int(reset) if reset and reset.isdigit() else None,
+            )
+        if response.status_code == 404 and not response.content:
+            # X also uses an empty 404 when a read route is temporarily limited.
+            reset = response.headers.get("x-rate-limit-reset")
+            raise RateLimitError(
+                f"X returned an empty 404 for {operation}; retry after the rate window",
+                int(reset) if reset and reset.isdigit() else None,
+            )
+        if response.status_code != 200:
+            raise APIError(f"X returned HTTP {response.status_code} for {operation}")
+        try:
+            payload = response.json()
+        except ValueError as exc:
+            raise APIError(f"X returned non-JSON data for {operation}") from exc
+        if not isinstance(payload, dict) or payload.get("errors"):
+            raise APIError(f"X returned an error for {operation}")
+        return payload
+
+    def get_profile(self, handle: str) -> Profile:
+        payload = self.graphql(
+            "UserByScreenName",
+            {"screen_name": handle, "withSafetyModeUserFields": True},
+        )
+        return parse_profile(payload, captured_at_utc=now_utc())
+
+    def get_post(self, post_id: str) -> Post:
+        payload = self.graphql(
+            "TweetResultByRestId",
+            {
+                "tweetId": post_id,
+                "withCommunity": False,
+                "includePromotedContent": False,
+                "withVoice": False,
+            },
+            features=TWEET_FEATURES,
+            field_toggles={},
+        )
+        return single_post(payload, captured_at_utc=now_utc())
+
+    def get_posts(self, post_ids: list[str]) -> list[Post]:
+        if len(post_ids) > 50:
+            raise ValueError("get_posts accepts at most 50 IDs per request")
+        if not post_ids:
+            return []
+        payload = self.graphql(
+            "TweetResultsByRestIds",
+            {
+                "tweetIds": post_ids,
+                "includePromotedContent": True,
+                "withBirdwatchNotes": True,
+                "withVoice": True,
+                "withCommunity": True,
+            },
+            features=TWEET_FEATURES,
+            field_toggles={},
+        )
+        found: list[Post] = []
+        for item in payload.get("data", {}).get("tweetResult", []):
+            result = item.get("result")
+            if isinstance(result, dict):
+                post = parse_post(result, captured_at_utc=now_utc())
+                if post is not None:
+                    found.append(post)
+        return found
+
+    def conversation_page(self, post_id: str, cursor: str | None = None) -> dict:
+        """Read a post's conversation, including replies by other authors.
+
+        A focal post may itself be a reply; callers can expand such branches to
+        discover nested comments that the root conversation omits.
+        """
+        variables = {
+            "focalTweetId": post_id,
+            "with_rux_injections": False,
+            "includePromotedContent": False,
+            "withCommunity": True,
+            "withQuickPromoteEligibilityTweetFields": True,
+            "withBirdwatchNotes": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+        }
+        if cursor:
+            variables["cursor"] = cursor
+        return self.graphql(
+            "TweetDetail",
+            variables,
+            features=TWEET_FEATURES,
+            field_toggles={"withArticleRichContentState": False},
+        )
+
+    def user_timeline_page(
+        self, user_id: str, *, timeline: str, cursor: str | None = None
+    ) -> dict:
+        if timeline not in {"posts", "replies"}:
+            raise ValueError("timeline must be posts or replies")
+        operation = (
+            "UserOriginalsTimeline" if timeline == "posts" else "UserRepliesTimeline"
+        )
+        variables = {
+            "userId": user_id,
+            "count": 20,
+            "includePromotedContent": True,
+            "withQuickPromoteEligibilityTweetFields": True,
+            "withVoice": True,
+            "withV2Timeline": True,
+        }
+        if cursor:
+            variables["cursor"] = cursor
+        return self.graphql(operation, variables, features=TWEET_FEATURES)