pyXcom 0.6.0__py3-none-any.whl

pyxcom/parse.py

@@ -0,0 +1,217 @@
+"""Parse X's public GraphQL response objects without depending on twikit."""
+
+from datetime import datetime, timezone
+from typing import Any, Iterator
+
+from .errors import ParseError
+from .models import Post, Profile
+
+
+def _walk(value: Any) -> Iterator[dict]:
+    if isinstance(value, dict):
+        yield value
+        for child in value.values():
+            yield from _walk(child)
+    elif isinstance(value, list):
+        for child in value:
+            yield from _walk(child)
+
+
+def _count(value: Any) -> int | None:
+    if value is None or value == "":
+        return None
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def _utc(value: str | None) -> str | None:
+    if not value:
+        return None
+    try:
+        return (
+            datetime.strptime(value, "%a %b %d %H:%M:%S %z %Y")
+            .astimezone(timezone.utc)
+            .isoformat()
+        )
+    except ValueError:
+        return None
+
+
+def parse_profile(data: dict, *, captured_at_utc: str | None = None) -> Profile:
+    """Parse a UserByScreenName result or its inner user result."""
+    result = data.get("data", {}).get("user", {}).get("result", data)
+    legacy = result.get("legacy") or {}
+    core = result.get("core") or {}
+    relationships = result.get("relationship_counts") or {}
+    tweet_counts = result.get("tweet_counts") or {}
+    user_id = result.get("rest_id")
+    handle = legacy.get("screen_name") or core.get("screen_name")
+    if not user_id or not handle:
+        raise ParseError("X user response has no user ID or screen name")
+    return Profile(
+        id=str(user_id),
+        handle=handle,
+        name=legacy.get("name") or core.get("name") or handle,
+        description=legacy.get("description")
+        or (result.get("profile_bio") or {}).get("description"),
+        created_at_utc=_utc(legacy.get("created_at") or core.get("created_at")),
+        followers_count=_count(
+            legacy.get("followers_count", relationships.get("followers"))
+        ),
+        following_count=_count(
+            legacy.get("friends_count", relationships.get("following"))
+        ),
+        posts_count=_count(legacy.get("statuses_count", tweet_counts.get("tweets"))),
+        media_count=_count(legacy.get("media_count", tweet_counts.get("media_tweets"))),
+        verified=bool(result.get("is_blue_verified") or legacy.get("verified")),
+        url=f"https://x.com/{handle}",
+        captured_at_utc=captured_at_utc,
+    )
+
+
+def parse_post(data: dict, *, captured_at_utc: str | None = None) -> Post | None:
+    """Parse a Tweet or TweetWithVisibilityResults result."""
+    if data.get("__typename") == "TweetTombstone":
+        return None
+    if "tweet" in data:
+        data = data["tweet"]
+    legacy = data.get("legacy") or {}
+    post_id = data.get("rest_id") or legacy.get("id_str")
+    user_result = data.get("core", {}).get("user_results", {}).get("result") or {}
+    author = user_result.get("legacy") or user_result.get("core") or {}
+    author_id = legacy.get("user_id_str") or user_result.get("rest_id")
+    created = _utc(legacy.get("created_at"))
+    if not post_id or not author_id or not created:
+        return None
+    handle = author.get("screen_name") or ""
+    note = (
+        (data.get("note_tweet") or {}).get("note_tweet_results", {}).get("result", {})
+    )
+    text = note.get("text") or legacy.get("full_text") or ""
+    entities = legacy.get("entities") or {}
+    extended = legacy.get("extended_entities") or {}
+    media_urls: list[str] = []
+    for media in extended.get("media", entities.get("media", [])):
+        if media.get("media_url_https"):
+            media_urls.append(media["media_url_https"])
+        variants = (media.get("video_info") or {}).get("variants") or []
+        mp4 = [
+            variant
+            for variant in variants
+            if variant.get("content_type") == "video/mp4"
+        ]
+        if mp4:
+            best = max(mp4, key=lambda variant: variant.get("bitrate") or 0)
+            if best.get("url"):
+                media_urls.append(best["url"])
+    outbound = [
+        item["expanded_url"]
+        for item in entities.get("urls", [])
+        if item.get("expanded_url")
+    ]
+    hashtags = [
+        item["text"] for item in entities.get("hashtags", []) if item.get("text")
+    ]
+    mentions = [
+        item["screen_name"]
+        for item in entities.get("user_mentions", [])
+        if item.get("screen_name")
+    ]
+    repost_result = legacy.get("retweeted_status_result") or {}
+    return Post(
+        id=str(post_id),
+        author_id=str(author_id),
+        author_handle=handle,
+        created_at_utc=created,
+        text=text,
+        url=f"https://x.com/{handle}/status/{post_id}"
+        if handle
+        else f"https://x.com/i/status/{post_id}",
+        language=legacy.get("lang"),
+        conversation_id=legacy.get("conversation_id_str"),
+        in_reply_to_id=legacy.get("in_reply_to_status_id_str"),
+        quoted_post_id=legacy.get("quoted_status_id_str"),
+        reposted_post_id=(repost_result.get("result") or {}).get("rest_id"),
+        reply_count=_count(legacy.get("reply_count")),
+        repost_count=_count(legacy.get("retweet_count")),
+        like_count=_count(legacy.get("favorite_count")),
+        view_count=_count((data.get("views") or {}).get("count")),
+        quote_count=_count(legacy.get("quote_count")),
+        bookmark_count=_count(legacy.get("bookmark_count")),
+        media_urls=list(dict.fromkeys(media_urls)),
+        outbound_urls=list(dict.fromkeys(outbound)),
+        hashtags=list(dict.fromkeys(hashtags)),
+        mentions=list(dict.fromkeys(mentions)),
+        captured_at_utc=captured_at_utc,
+    )
+
+
+def timeline_posts(data: dict, *, captured_at_utc: str | None = None) -> list[Post]:
+    """Read tweet_results in user, search, or conversation timelines."""
+    posts: dict[str, Post] = {}
+    for node in _walk(data.get("data", data)):
+        tweet_results = node.get("tweet_results")
+        if not isinstance(tweet_results, dict):
+            continue
+        result = tweet_results.get("result")
+        if not isinstance(result, dict):
+            continue
+        post = parse_post(result, captured_at_utc=captured_at_utc)
+        if post is not None:
+            posts[post.id] = post
+    return list(posts.values())
+
+
+def timeline_primary_posts(
+    data: dict, *, captured_at_utc: str | None = None
+) -> list[Post]:
+    """Read timeline entries without recursively counting quoted post content."""
+    instructions = None
+    for node in _walk(data.get("data", data)):
+        if isinstance(node.get("instructions"), list):
+            instructions = node["instructions"]
+            break
+    if instructions is None:
+        return []
+    posts: dict[str, Post] = {}
+    for instruction in instructions:
+        for entry in instruction.get("entries", []):
+            content = entry.get("content") or {}
+            items = [content]
+            for module_item in content.get("items", []):
+                items.append(module_item.get("item") or module_item)
+            for item in items:
+                result = (
+                    (item.get("itemContent") or {})
+                    .get("tweet_results", {})
+                    .get("result")
+                )
+                if isinstance(result, dict):
+                    post = parse_post(result, captured_at_utc=captured_at_utc)
+                    if post is not None:
+                        posts[post.id] = post
+    return list(posts.values())
+
+
+def bottom_cursor(data: dict) -> str | None:
+    """Find the bottom pagination cursor without relying on one timeline shape."""
+    for node in _walk(data.get("data", data)):
+        entry_id = node.get("entryId", "")
+        content = node.get("content") or {}
+        if isinstance(entry_id, str) and entry_id.startswith("cursor-bottom"):
+            return content.get("value")
+        if content.get("cursorType") == "Bottom":
+            return content.get("value")
+    return None
+
+
+def single_post(data: dict, *, captured_at_utc: str | None = None) -> Post:
+    result = data.get("data", {}).get("tweetResult", {}).get("result")
+    if not isinstance(result, dict):
+        raise ParseError("X single-post response has no tweet result")
+    post = parse_post(result, captured_at_utc=captured_at_utc)
+    if post is None:
+        raise ParseError("X post is unavailable or its response changed")
+    return post

pyxcom/schema.py

@@ -0,0 +1,276 @@
+"""Post-role classification and field coverage reports for saved datasets."""
+
+import gzip
+import json
+import os
+from collections import Counter
+from dataclasses import fields
+from pathlib import Path
+
+from .layout import internal_dir, migrate_collection, source_path
+from .models import Post, Profile
+from .storage import PostStore, _atomic_json
+from .validate import finalize_collection, validate_collection
+
+FIELD_GROUPS = {
+    "id": ("身份", "X 帖子 ID；以字符串保存"),
+    "author_id": ("身份", "作者 X ID"),
+    "author_handle": ("身份", "作者账号名"),
+    "url": ("身份", "帖子永久链接"),
+    "created_at_utc": ("时间与正文", "发布时间，UTC ISO 8601"),
+    "text": ("时间与正文", "完整可取得的正文"),
+    "language": ("时间与正文", "X 返回的语言代码"),
+    "post_role": ("分类", "main 主帖 / comment 评论回复 / repost 转发"),
+    "post_type": ("分类", "original / quote / reply / repost"),
+    "conversation_id": ("关系", "会话根帖 ID"),
+    "in_reply_to_id": ("关系", "直接回复的父帖 ID"),
+    "quoted_post_id": ("关系", "被引用帖子 ID"),
+    "reposted_post_id": ("关系", "被转发帖子 ID"),
+    "reply_count": ("互动", "收到的回复数"),
+    "repost_count": ("互动", "转发数"),
+    "like_count": ("互动", "点赞数"),
+    "view_count": ("互动", "公开浏览量"),
+    "quote_count": ("互动", "被引用次数"),
+    "bookmark_count": ("互动", "书签数"),
+    "media_urls": ("内容元素", "图片、视频预览或视频链接数组"),
+    "outbound_urls": ("内容元素", "正文外链数组"),
+    "hashtags": ("内容元素", "话题标签数组"),
+    "mentions": ("内容元素", "提及账号数组"),
+    "discovery_source": ("来源", "发现帖子 ID 的方式"),
+    "discovery_url": ("来源", "发现页面链接"),
+    "captured_at_utc": ("来源", "该条响应的抓取时间；旧数据可能为空"),
+}
+
+RAW_OPTIONAL = [
+    (
+        "in_reply_to_user_id_str / in_reply_to_screen_name",
+        "回复目标账号；当前总表尚未标准化",
+    ),
+    (
+        "extended_entities.media[].type / sizes / video_info",
+        "媒体类型、尺寸、视频信息；当前只导出链接",
+    ),
+    ("possibly_sensitive", "X 的敏感内容标记；并非研究者自定义编码"),
+    ("source / display_text_range", "客户端来源与显示文本范围"),
+    (
+        "quoted_status_result / note_tweet",
+        "引用帖嵌套对象及长文结构；正文已优先取长文文本",
+    ),
+]
+
+
+def _read_posts(path: Path) -> list[Post]:
+    return [
+        Post(**json.loads(line))
+        for line in path.read_text(encoding="utf-8").split("\n")
+        if line.strip()
+    ]
+
+
+def schema_summary(output_dir: str | Path) -> dict:
+    output = Path(output_dir).expanduser()
+    posts = _read_posts(source_path(output, "posts.jsonl"))
+    post_fields = []
+    for item in fields(Post):
+        values = [getattr(post, item.name) for post in posts]
+        available = sum(value is not None and value != "" for value in values)
+        nonempty = sum(bool(value) for value in values if isinstance(value, list))
+        group, description = FIELD_GROUPS[item.name]
+        post_fields.append(
+            {
+                "name": item.name,
+                "group": group,
+                "description": description,
+                "available": available,
+                "missing": len(posts) - available,
+                "nonempty_array": nonempty
+                if any(isinstance(v, list) for v in values)
+                else None,
+                "example_type": next(
+                    (type(v).__name__ for v in values if v is not None), "None"
+                ),
+            }
+        )
+    profiles_path = source_path(output, "profiles.json")
+    profiles = (
+        [Profile(**value) for value in json.loads(profiles_path.read_text()).values()]
+        if profiles_path.exists()
+        else []
+    )
+    profile_fields = [
+        {
+            "name": item.name,
+            "available": sum(
+                getattr(profile, item.name) is not None for profile in profiles
+            ),
+            "missing": len(profiles)
+            - sum(getattr(profile, item.name) is not None for profile in profiles),
+        }
+        for item in fields(Profile)
+    ]
+    roles = Counter(post.post_role for post in posts)
+    kinds = Counter(post.post_type for post in posts)
+    by_account = {}
+    for handle in sorted({post.author_handle for post in posts}):
+        own = [post for post in posts if post.author_handle == handle]
+        by_account[handle] = {
+            "total": len(own),
+            "roles": dict(Counter(post.post_role for post in own)),
+            "types": dict(Counter(post.post_type for post in own)),
+        }
+    return {
+        "output_dir": str(output),
+        "post_count": len(posts),
+        "post_field_count": len(post_fields),
+        "role_counts": dict(roles),
+        "type_counts": dict(kinds),
+        "by_account": by_account,
+        "post_fields": post_fields,
+        "profile_count": len(profiles),
+        "profile_fields": profile_fields,
+        "raw_optional_fields": RAW_OPTIONAL,
+    }
+
+
+def write_schema_report(output_dir: str | Path) -> dict:
+    output = Path(output_dir).expanduser()
+    summary = schema_summary(output)
+    internal_dir(output).mkdir(parents=True, exist_ok=True)
+    _atomic_json(internal_dir(output) / "schema.json", summary)
+    lines = [
+        "# pyXcom 数据字段与主帖/评论分类",
+        "",
+        f"当前去重帖子：**{summary['post_count']:,}** 条；标准化帖子字段：**{summary['post_field_count']}** 个。",
+        "",
+        "## 分类规则",
+        "",
+        "- `reposted_post_id` 有值 → `post_role=repost`、`post_type=repost`。",
+        "- 否则 `in_reply_to_id` 有值 → `post_role=comment`、`post_type=reply`。回复自己的帖也按评论/回复记录。",
+        "- 否则 `quoted_post_id` 有值 → `post_role=main`、`post_type=quote`。",
+        "- 其余 → `post_role=main`、`post_type=original`。若多种关系同时存在，分类优先级为转发、回复、引用。原始关系 ID 保留在各列。",
+        "- 本次六账号数据只收作者本人生成的帖子；纯转发通常显示原作者 ID，未被纳入。因此此处 `repost=0` 是采集口径，不代表这些账号从未转发。",
+        "",
+        "| 主帖/评论角色 | 条数 |",
+        "| --- | ---: |",
+    ]
+    lines += [
+        f"| {key} | {value:,} |"
+        for key, value in sorted(summary["role_counts"].items())
+    ]
+    lines += ["", "| 帖子类型 | 条数 |", "| --- | ---: |"]
+    lines += [
+        f"| {key} | {value:,} |"
+        for key, value in sorted(summary["type_counts"].items())
+    ]
+    lines += [
+        "",
+        "## 各账号分类",
+        "",
+        "| 账号 | 主帖 | 评论/回复 | 转发 | 合计 |",
+        "| --- | ---: | ---: | ---: | ---: |",
+    ]
+    for handle, value in summary["by_account"].items():
+        roles = value["roles"]
+        lines.append(
+            f"| @{handle} | {roles.get('main', 0):,} | {roles.get('comment', 0):,} "
+            f"| {roles.get('repost', 0):,} | {value['total']:,} |"
+        )
+    lines += [
+        "",
+        "## 当前帖子表的字段覆盖",
+        "",
+        "| 分组 | 字段 | 含义 | 有值条数 | 空值条数 | 非空数组条数 |",
+        "| --- | --- | --- | ---: | ---: | ---: |",
+    ]
+    for item in summary["post_fields"]:
+        lines.append(
+            f"| {item['group']} | `{item['name']}` | {item['description']} "
+            f"| {item['available']:,} | {item['missing']:,} "
+            f"| {item['nonempty_array'] if item['nonempty_array'] is not None else '—'} |"
+        )
+    lines += ["", "## 账号资料字段", ""]
+    lines.append(", ".join(f"`{item['name']}`" for item in summary["profile_fields"]))
+    lines += [
+        "",
+        "## X 原始返回中还可能取得、目前尚未标准化的字段",
+        "",
+        "| 原始字段 | 说明 |",
+        "| --- | --- |",
+    ]
+    lines += [f"| `{name}` | {description} |" for name, description in RAW_OPTIONAL]
+    lines += [
+        "",
+        "这些可选字段依帖子类型和 X 当前返回格式而异；不能把不存在的字段当作零。登录账号自己的 `favorited`、`bookmarked` 等状态不是公开互动总数，不建议混入公共帖子表。",
+        "",
+        "CSV 中数组字段是 JSON 字符串；JSONL 中保留数组、整数及 null。`captured_at_utc` 的旧记录为空时，不能用批次导出时间冒充每条帖子的抓取时间。",
+        "",
+    ]
+    (internal_dir(output) / "schema_report.md").write_text(
+        "\n".join(lines), encoding="utf-8"
+    )
+    return summary
+
+
+def apply_role_schema(output_dir: str | Path) -> dict:
+    """Backfill role/type in saved JSONL; preserve gzipped originals once."""
+    output = Path(output_dir).expanduser()
+    migrate_collection(output)
+    changed: list[str] = []
+    paths = sorted(
+        (
+            p
+            for p in output.rglob("posts.jsonl")
+            if "legacy" not in p.relative_to(output).parts
+        ),
+        key=lambda path: len(path.parts),
+        reverse=True,
+    )
+    for path in paths:
+        folder = path.parent.parent if path.parent.name == ".pyxcom" else path.parent
+        state_path = source_path(folder, "state.json")
+        if not state_path.exists():
+            continue
+        raw = [
+            json.loads(line)
+            for line in path.read_text(encoding="utf-8").split("\n")
+            if line.strip()
+        ]
+        posts = [Post(**record) for record in raw]
+        if all(
+            record.get("post_role") == post.post_role
+            and record.get("post_type") == post.post_type
+            for record, post in zip(raw, posts)
+        ):
+            continue
+        backup = path.parent / "posts.jsonl.before-role-schema.gz"
+        if not backup.exists():
+            with gzip.open(backup, "wb") as file:
+                file.write(path.read_bytes())
+        temporary = path.with_suffix(".jsonl.tmp")
+        with temporary.open("w", encoding="utf-8") as file:
+            for post in posts:
+                file.write(json.dumps(post.to_dict(), ensure_ascii=True) + "\n")
+        os.replace(temporary, path)
+        state = json.loads(state_path.read_text(encoding="utf-8"))
+        store = PostStore(folder, query=state["query"])
+        store.finish(
+            complete=state.get("complete", False),
+            reason=state.get("reason", "migrated"),
+        )
+        changed.append(str(folder))
+    if source_path(output, "profiles.json").exists():
+        finalize_collection(output)
+    summary = write_schema_report(output)
+    validations = [
+        validate_collection(
+            path.parent.parent if path.parent.name == ".pyxcom" else path.parent
+        )
+        for path in paths
+    ]
+    if not all(result["valid"] for result in validations):
+        raise ValueError("One or more migrated collections failed validation")
+    return {
+        "changed_directories": changed,
+        "validated_directories": len(validations),
+        **summary,
+    }

pyxcom/search.py

@@ -0,0 +1,94 @@
+"""Discover public X post IDs through a configurable Nitter search mirror."""
+
+import re
+from dataclasses import dataclass
+from urllib.parse import urlencode, urljoin, urlsplit
+
+import httpx
+from bs4 import BeautifulSoup
+
+from .errors import APIError, RateLimitError
+
+_POST_ID = re.compile(r"/status/(\d+)")
+
+
+@dataclass(frozen=True)
+class SearchPage:
+    post_ids: list[str]
+    next_url: str | None
+    source_url: str
+
+
+class MirrorSearch:
+    """Searches only public mirror pages; X login cookies are never sent here."""
+
+    def __init__(
+        self,
+        *,
+        base_url: str = "https://x.noodl3.net",
+        proxy: str | None = None,
+        timeout: float = 30,
+    ) -> None:
+        self.base_url = base_url.rstrip("/")
+        parts = urlsplit(self.base_url)
+        if parts.scheme != "https" or not parts.netloc:
+            raise ValueError("mirror base_url must be HTTPS")
+        self._host = parts.netloc
+        self._http = httpx.Client(proxy=proxy, timeout=timeout, follow_redirects=True)
+
+    def close(self) -> None:
+        self._http.close()
+
+    def initial_url(
+        self,
+        keyword: str,
+        *,
+        since: str | None = None,
+        until: str | None = None,
+        user: str | None = None,
+    ) -> str:
+        terms = [keyword.strip()]
+        if not terms[0] and not user:
+            raise ValueError("keyword or user is required")
+        if user:
+            terms.append(f"from:{user}")
+        if since:
+            terms.append(f"since:{since}")
+        if until:
+            terms.append(f"until:{until}")
+        query = " ".join(part for part in terms if part)
+        return f"{self.base_url}/search?{urlencode({'f': 'tweets', 'q': query})}"
+
+    def page(self, url: str) -> SearchPage:
+        if urlsplit(url).scheme != "https" or urlsplit(url).netloc != self._host:
+            raise APIError("Search cursor changed mirror domain")
+        response = self._http.get(url)
+        if response.status_code == 429:
+            raise RateLimitError("Search mirror is rate-limited")
+        if response.status_code != 200:
+            raise APIError(f"Search mirror returned HTTP {response.status_code}")
+        soup = BeautifulSoup(response.text, "html.parser")
+        title = soup.title.get_text(" ", strip=True) if soup.title else ""
+        if "Error" in title or "bot" in title.lower() or "captcha" in title.lower():
+            raise APIError("Search mirror returned an error or verification page")
+        post_ids: list[str] = []
+        for item in soup.select(".timeline-item"):
+            date_link = item.select_one(".tweet-date a[href]")
+            if date_link is None:
+                continue
+            href = date_link.get("href")
+            if not isinstance(href, str):
+                continue
+            match = _POST_ID.search(href)
+            if match:
+                post_ids.append(match.group(1))
+        more = soup.select_one(".show-more a[href*='cursor=']")
+        more_href = more.get("href") if more else None
+        next_url = (
+            urljoin(str(response.url), more_href)
+            if isinstance(more_href, str)
+            else None
+        )
+        if next_url and urlsplit(next_url).netloc != self._host:
+            raise APIError("Search mirror returned a cross-domain cursor")
+        return SearchPage(list(dict.fromkeys(post_ids)), next_url, str(response.url))