pyXcom 0.6.0__py3-none-any.whl

pyxcom/comments.py

@@ -0,0 +1,235 @@
+"""Bounded, resumable traversal of public replies to a root X post."""
+
+import time
+from dataclasses import replace
+from typing import TYPE_CHECKING, Iterator
+
+from .errors import ParseError, RateLimitError
+from .models import Post
+from .parse import _walk, parse_post, parse_profile
+from .transport import now_utc
+
+if TYPE_CHECKING:
+    from .client import XClient
+
+
+def parse_conversation(data: dict) -> tuple[list[Post], list[str], dict]:
+    """Read timeline content only, excluding embedded quotes and recommendations."""
+    instructions = next(
+        (
+            n["instructions"]
+            for n in _walk(data.get("data", {}))
+            if isinstance(n.get("instructions"), list)
+        ),
+        None,
+    )
+    if instructions is None:
+        raise ParseError("X conversation response contains no timeline instructions")
+    posts: dict[str, Post] = {}
+    cursors: list[str] = []
+    profiles: dict[str, dict] = {}
+    captured = now_utc()
+
+    def visit(node):
+        if isinstance(node, list):
+            for item in node:
+                visit(item)
+        elif isinstance(node, dict):
+            if node.get("promotedMetadata"):
+                return
+            if "tweet_results" in node:
+                result = (node.get("tweet_results") or {}).get("result")
+                if isinstance(result, dict):
+                    post = parse_post(result, captured_at_utc=captured)
+                    if post:
+                        posts[post.id] = post
+                        tweet = result.get("tweet", result)
+                        user = (
+                            tweet.get("core", {})
+                            .get("user_results", {})
+                            .get("result", {})
+                        )
+                        try:
+                            profile = parse_profile(user, captured_at_utc=captured)
+                            profiles[profile.handle] = profile.to_dict()
+                        except ParseError:
+                            pass
+                return  # Never recurse into a quoted tweet or an author's other data.
+            kind = node.get("cursorType")
+            if kind in (
+                "Bottom",
+                "ShowMore",
+                "ShowMoreThreads",
+                "ShowMoreReplies",
+            ) and node.get("value"):
+                cursors.append(node["value"])
+            for key in (
+                "entries",
+                "entry",
+                "content",
+                "items",
+                "item",
+                "itemContent",
+                "moduleItems",
+            ):
+                if key in node:
+                    visit(node[key])
+
+    visit(instructions)
+    return list(posts.values()), list(dict.fromkeys(cursors)), profiles
+
+
+class CommentTraversal:
+    def __init__(
+        self,
+        client: "XClient",
+        root_id: str,
+        *,
+        max_depth: int,
+        max_comments: int | None,
+        max_pages: int | None,
+        records: dict[str, Post] | None = None,
+        state: dict | None = None,
+    ):
+        if max_depth is None:
+            raise ValueError("max_depth must be a positive integer")
+        for name, value in (
+            ("max_depth", max_depth),
+            ("max_comments", max_comments),
+            ("max_pages", max_pages),
+        ):
+            if value is not None and (
+                isinstance(value, bool) or not isinstance(value, int) or value < 1
+            ):
+                raise ValueError(f"{name} must be a positive integer")
+        self.client = client
+        self.root_id = root_id
+        self.max_depth, self.max_comments, self.max_pages = (
+            max_depth,
+            max_comments,
+            max_pages,
+        )
+        self.records = dict(records or {})
+        self.state = state if state is not None else {}
+        self.state.setdefault("queue", [[root_id, None]])
+        self.state.setdefault("visited", [])
+        self.state.setdefault("expanded", [root_id])
+        self.state.setdefault("pending", {})
+        self.state.setdefault("depths", {root_id: 0})
+        self.state.setdefault("profiles", {})
+        self.state.setdefault("pagination_warnings", [])
+        self.state.setdefault("empty_pages", {})
+        self.complete = False
+        self.reason = "not_started"
+        self.pages_fetched = 0
+        self.page_was_fetched = False
+
+    def _accept_pending(self) -> list[Post]:
+        added: list[Post] = []
+        pending = self.state["pending"]
+        while True:
+            progressed = False
+            for key, value in list(pending.items()):
+                post = Post(**value)
+                parent_depth = self.state["depths"].get(post.in_reply_to_id)
+                if parent_depth is None:
+                    continue
+                depth = parent_depth + 1
+                if depth > self.max_depth or key in self.records:
+                    self.state["depths"][key] = depth
+                    del pending[key]
+                    progressed = True
+                    continue
+                if (
+                    self.max_comments is not None
+                    and len(self.records) - 1 >= self.max_comments
+                ):
+                    return added
+                self.records[key] = post
+                self.state["depths"][key] = depth
+                del pending[key]
+                added.append(post)
+                progressed = True
+                if (
+                    depth < self.max_depth
+                    and post.reply_count != 0
+                    and key not in self.state["expanded"]
+                ):
+                    self.state["queue"].append([key, None])
+                    self.state["expanded"].append(key)
+            if not progressed:
+                return added
+
+    def pages(self) -> Iterator[list[Post]]:
+        if self.root_id not in self.records:
+            root = self.client.get_post(self.root_id)
+            if root.in_reply_to_id or root.reposted_post_id:
+                raise ValueError(
+                    "post_id_or_url must identify a main post, not a reply or repost"
+                )
+            self.records[root.id] = root
+            self.page_was_fetched = False
+            yield [root]
+        while True:
+            self.page_was_fetched = False
+            ready = self._accept_pending()
+            if ready:
+                yield ready
+            if (
+                self.max_comments is not None
+                and len(self.records) - 1 >= self.max_comments
+            ):
+                self.reason = "comment_limit"
+                return
+            if not self.state["queue"]:
+                unresolved = len(self.state["pending"])
+                self.state["unresolved_count"] = unresolved
+                self.complete = not unresolved and not self.state["pagination_warnings"]
+                self.reason = (
+                    "visible_source_end" if self.complete else "partial_conversation"
+                )
+                return
+            if self.max_pages is not None and self.pages_fetched >= self.max_pages:
+                self.reason = "page_limit"
+                return
+            task = self.state["queue"][0]
+            focal, cursor = task
+            try:
+                payload = self.client._x.conversation_page(focal, cursor=cursor)
+            except RateLimitError as exc:
+                self.state["rate_reset_at"] = exc.reset_at
+                self.reason = "rate_limited"
+                return
+            posts, cursors, profiles = parse_conversation(payload)
+            self.state.pop("rate_reset_at", None)
+            self.state["queue"].pop(0)
+            self.state["visited"].append(task)
+            self.state["profiles"].update(profiles)
+            relevant = [
+                p
+                for p in posts
+                if p.conversation_id == self.root_id
+                and p.in_reply_to_id
+                and p.id not in self.records
+            ]
+            for post in relevant:
+                self.state["pending"][post.id] = replace(
+                    post,
+                    discovery_source="x_conversation",
+                    discovery_url=f"https://x.com/i/status/{focal}",
+                ).to_dict()
+            empty = self.state["empty_pages"].get(focal, 0)
+            self.state["empty_pages"][focal] = empty + 1 if not relevant else 0
+            for next_cursor in cursors:
+                follow = [focal, next_cursor]
+                if follow in self.state["visited"]:
+                    self.state["pagination_warnings"].append("repeated_cursor")
+                elif self.state["empty_pages"][focal] >= 2:
+                    self.state["pagination_warnings"].append("empty_page_limit")
+                elif follow not in self.state["queue"]:
+                    self.state["queue"].append(follow)
+            self.pages_fetched += 1
+            self.page_was_fetched = True
+            yield self._accept_pending()
+            if self.client.delay:
+                time.sleep(self.client.delay)

pyxcom/errors.py

@@ -0,0 +1,29 @@
+"""Typed failures surfaced by pyXcom."""
+
+
+class PyXcomError(Exception):
+    """Base package exception."""
+
+
+class AuthenticationError(PyXcomError):
+    """No usable browser login was found."""
+
+
+class CaptureError(PyXcomError):
+    """The browser could not capture the requested X read endpoint."""
+
+
+class APIError(PyXcomError):
+    """X returned an unusable response."""
+
+
+class RateLimitError(APIError):
+    """X limited the requested endpoint."""
+
+    def __init__(self, message: str, reset_at: int | None = None) -> None:
+        super().__init__(message)
+        self.reset_at = reset_at
+
+
+class ParseError(PyXcomError):
+    """The response shape changed or omitted expected data."""

pyxcom/layout.py

@@ -0,0 +1,124 @@
+"""Stable public tables and private, resumable collection artifacts."""
+
+import csv
+import json
+import shutil
+from pathlib import Path
+
+_INTERNAL_FILES = (
+    "posts.jsonl",
+    "state.json",
+    "profiles.json",
+    "account_manifest.json",
+    "report.md",
+    "schema.json",
+    "schema_report.md",
+    "posts.jsonl.before-role-schema.gz",
+)
+
+
+def internal_dir(output: str | Path) -> Path:
+    return Path(output).expanduser() / ".pyxcom"
+
+
+def source_path(output: str | Path, name: str) -> Path:
+    private = internal_dir(output) / name
+    legacy = Path(output).expanduser() / name
+    return private if private.exists() or not legacy.exists() else legacy
+
+
+def child_dir(output: str | Path, name: str) -> Path:
+    return internal_dir(output) / name
+
+
+def _same_contents(left: Path, right: Path) -> bool:
+    """Compare every directory entry and file byte before reusing a backup."""
+    if left.is_symlink() or right.is_symlink():
+        return False
+    if left.is_dir() and right.is_dir():
+        left_entries = {entry.name: entry for entry in left.iterdir()}
+        right_entries = {entry.name: entry for entry in right.iterdir()}
+        return left_entries.keys() == right_entries.keys() and all(
+            _same_contents(entry, right_entries[name])
+            for name, entry in left_entries.items()
+        )
+    return (
+        left.is_file() and right.is_file() and left.read_bytes() == right.read_bytes()
+    )
+
+
+def _move_preserving(source: Path, target: Path, backup: Path) -> None:
+    if not source.exists():
+        return
+    if target.exists():
+        if (
+            source.is_dir()
+            or target.is_dir()
+            or source.read_bytes() != target.read_bytes()
+        ):
+            raise ValueError(f"Conflicting migration paths: {source} and {target}")
+    if backup.exists():
+        if not _same_contents(source, backup):
+            raise ValueError(f"Conflicting legacy backup: {backup}")
+    else:
+        backup.parent.mkdir(parents=True, exist_ok=True)
+        if source.is_dir():
+            shutil.copytree(source, backup)
+        else:
+            shutil.copy2(source, backup)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if target.exists():
+        source.unlink()
+    else:
+        source.rename(target)
+
+
+def migrate_collection(output: str | Path) -> Path:
+    """Move recognized legacy artifacts, preserving backups and unrelated files.
+
+    Repeated calls are safe. Conflicting destinations raise rather than overwrite.
+    Normalized root tables are never mistaken for legacy mixed observations.
+    """
+    directory = Path(output).expanduser()
+    private = internal_dir(directory)
+    private.mkdir(parents=True, exist_ok=True)
+    backup = private / "legacy"
+    for name in _INTERNAL_FILES:
+        _move_preserving(directory / name, private / name, backup / name)
+    root_manifest = directory / "manifest.json"
+    if root_manifest.exists():
+        data = json.loads(root_manifest.read_text(encoding="utf-8"))
+        if "post_count" in data and "counts" not in data:
+            _move_preserving(
+                root_manifest, private / "manifest.json", backup / "manifest.json"
+            )
+    root_csv = directory / "posts.csv"
+    if root_csv.exists():
+        with root_csv.open(encoding="utf-8-sig", newline="") as stream:
+            header = next(csv.reader(stream), [])
+        if "id" in header and ("post_role" in header or "author_handle" in header):
+            _move_preserving(root_csv, private / "posts.csv", backup / "posts.csv")
+    # Only folders with a collection checkpoint are owned collection scopes.
+    for folder in list(directory.iterdir()):
+        if (
+            folder.is_dir()
+            and folder.name not in {".pyxcom", "tables"}
+            and (folder / "state.json").exists()
+        ):
+            _move_preserving(folder, private / folder.name, backup / folder.name)
+    old_tables = directory / "tables"
+    if (old_tables / "manifest.json").exists():
+        data = json.loads((old_tables / "manifest.json").read_text(encoding="utf-8"))
+        if data.get("source_kind") == "saved_post_observations":
+            _move_preserving(old_tables, private / "previous_tables", backup / "tables")
+    for folder in list(private.iterdir()):
+        if (
+            folder.is_dir()
+            and folder.name not in {"legacy", "previous_tables"}
+            and (
+                (folder / "state.json").exists()
+                or (internal_dir(folder) / "state.json").exists()
+            )
+        ):
+            migrate_collection(folder)
+    return directory

pyxcom/models.py

@@ -0,0 +1,96 @@
+"""Portable public-data records returned by pyXcom."""
+
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+
+
+def classify_post(
+    in_reply_to_id: str | None,
+    quoted_post_id: str | None,
+    reposted_post_id: str | None,
+) -> tuple[str, str]:
+    """Return (role, type) from X's explicit post relationships."""
+    if reposted_post_id:
+        return "repost", "repost"
+    if in_reply_to_id:
+        return "comment", "reply"
+    if quoted_post_id:
+        return "main", "quote"
+    return "main", "original"
+
+
+@dataclass(frozen=True)
+class Profile:
+    id: str
+    handle: str
+    name: str
+    description: str | None = None
+    created_at_utc: str | None = None
+    followers_count: int | None = None
+    following_count: int | None = None
+    posts_count: int | None = None
+    media_count: int | None = None
+    verified: bool | None = None
+    url: str | None = None
+    captured_at_utc: str | None = None
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class Post:
+    id: str
+    author_id: str
+    author_handle: str
+    created_at_utc: str
+    text: str
+    url: str
+    post_role: str = ""
+    post_type: str = ""
+    language: str | None = None
+    conversation_id: str | None = None
+    in_reply_to_id: str | None = None
+    quoted_post_id: str | None = None
+    reposted_post_id: str | None = None
+    reply_count: int | None = None
+    repost_count: int | None = None
+    like_count: int | None = None
+    view_count: int | None = None
+    quote_count: int | None = None
+    bookmark_count: int | None = None
+    media_urls: list[str] = field(default_factory=list)
+    outbound_urls: list[str] = field(default_factory=list)
+    hashtags: list[str] = field(default_factory=list)
+    mentions: list[str] = field(default_factory=list)
+    discovery_source: str | None = None
+    discovery_url: str | None = None
+    captured_at_utc: str | None = None
+
+    def __post_init__(self) -> None:
+        role, kind = classify_post(
+            self.in_reply_to_id, self.quoted_post_id, self.reposted_post_id
+        )
+        object.__setattr__(self, "post_role", role)
+        object.__setattr__(self, "post_type", kind)
+
+    def to_dict(self) -> dict:
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class CollectionResult:
+    output_dir: Path
+    post_count: int
+    pages_fetched: int
+    complete: bool
+    reason: str
+
+    def to_dict(self) -> dict:
+        return {
+            "output_dir": str(self.output_dir),
+            "post_count": self.post_count,
+            "pages_fetched": self.pages_fetched,
+            "complete": self.complete,
+            "reason": self.reason,
+        }