pyXcom 0.6.0__py3-none-any.whl

pyxcom/__init__.py

@@ -0,0 +1,29 @@
+"""pyXcom: public X collection with an existing Chrome or Edge login."""
+
+from .client import XClient
+from .errors import APIError, AuthenticationError, PyXcomError, RateLimitError
+from .models import CollectionResult, Post, Profile, classify_post
+from .schema import apply_role_schema, schema_summary, write_schema_report
+from .tables import export_tables
+from .validate_tables import validate_tables
+from .validate import finalize_collection, validate_collection
+
+__all__ = [
+    "XClient",
+    "PyXcomError",
+    "AuthenticationError",
+    "APIError",
+    "RateLimitError",
+    "export_tables",
+    "validate_tables",
+    "Profile",
+    "Post",
+    "CollectionResult",
+    "classify_post",
+    "schema_summary",
+    "write_schema_report",
+    "apply_role_schema",
+    "finalize_collection",
+    "validate_collection",
+]
+__version__ = "0.6.0"

pyxcom/auth.py

@@ -0,0 +1,66 @@
+"""Load only X authentication cookies from an existing local browser profile."""
+
+import os
+import sys
+from pathlib import Path
+
+import browser_cookie3
+
+from .errors import AuthenticationError
+
+
+def profile_cookie_db(browser: str, profile: str) -> Path:
+    """Resolve a Chrome/Edge profile cookie DB on supported desktop systems."""
+    if browser not in {"chrome", "edge"}:
+        raise AuthenticationError("Supported browsers are chrome and edge")
+    if sys.platform == "darwin":
+        folder = "Google/Chrome" if browser == "chrome" else "Microsoft Edge"
+        return (
+            Path.home() / "Library/Application Support" / folder / profile / "Cookies"
+        )
+    if sys.platform.startswith("linux"):
+        folder = "google-chrome" if browser == "chrome" else "microsoft-edge"
+        return Path.home() / ".config" / folder / profile / "Cookies"
+    if sys.platform == "win32":
+        local = Path(os.environ.get("LOCALAPPDATA", ""))
+        folder = "Google/Chrome" if browser == "chrome" else "Microsoft/Edge"
+        return local / folder / "User Data" / profile / "Network" / "Cookies"
+    raise AuthenticationError("Unsupported operating system; pass cookie_db explicitly")
+
+
+def load_x_cookies(
+    browser: str = "chrome",
+    *,
+    profile: str | None = None,
+    cookie_db: str | Path | None = None,
+) -> dict[str, str]:
+    """Return auth_token and ct0 in memory; never save or log their values."""
+    if browser not in {"chrome", "edge"}:
+        raise AuthenticationError("Supported browsers are chrome and edge")
+    path = Path(cookie_db).expanduser() if cookie_db else None
+    if path is None and profile:
+        path = profile_cookie_db(browser, profile)
+    if path is not None and not path.exists():
+        raise AuthenticationError(
+            "Browser cookie database not found at the selected path"
+        )
+    try:
+        reader = browser_cookie3.chrome if browser == "chrome" else browser_cookie3.edge
+        kwargs = {"domain_name": ".x.com"}
+        if path is not None:
+            kwargs["cookie_file"] = str(path)
+        jar = reader(**kwargs)
+    except Exception as exc:
+        raise AuthenticationError(
+            "Could not read the selected browser profile"
+        ) from exc
+    cookies = {
+        cookie.name: cookie.value
+        for cookie in jar
+        if cookie.name in {"auth_token", "ct0"} and cookie.domain.lstrip(".") == "x.com"
+    }
+    if set(cookies) != {"auth_token", "ct0"}:
+        raise AuthenticationError(
+            "No active X login in this browser profile; sign in to x.com or select another profile"
+        )
+    return cookies

pyxcom/batch.py

@@ -0,0 +1,260 @@
+"""Resumable multi-account collection, audit, and summary reporting."""
+
+import json
+import time
+from collections import Counter
+from pathlib import Path
+from statistics import median
+from typing import TYPE_CHECKING, Callable
+
+from .errors import PyXcomError
+from .layout import child_dir, internal_dir, migrate_collection, source_path
+from .models import CollectionResult, Post, Profile
+from .storage import PostStore, _atomic_json
+from .transport import now_utc
+
+if TYPE_CHECKING:
+    from .client import XClient
+
+
+def _state(path: Path) -> dict:
+    return json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+
+
+def _posts(path: Path) -> list[Post]:
+    if not path.exists():
+        return []
+    return [
+        Post(**json.loads(line))
+        for line in path.read_text(encoding="utf-8").split("\n")
+        if line.strip()
+    ]
+
+
+def _account_status(output: Path, handle: str, profile: dict) -> dict:
+    account_dir = child_dir(output, handle)
+    originals = _state(source_path(child_dir(account_dir, "originals"), "state.json"))
+    replies = _state(source_path(child_dir(account_dir, "replies"), "state.json"))
+    original_count = len(
+        _posts(source_path(child_dir(account_dir, "originals"), "posts.jsonl"))
+    )
+    reply_count = len(
+        _posts(source_path(child_dir(account_dir, "replies"), "posts.jsonl"))
+    )
+    total = len({post.id for post in _posts(source_path(account_dir, "posts.jsonl"))})
+    return {
+        "handle": handle,
+        "user_id": profile["id"],
+        "originals": {
+            "count": original_count,
+            "pages": originals.get("pages_fetched", 0),
+            "complete": originals.get("complete", False),
+            "reason": originals.get("reason", "not_started"),
+            "rate_reset_at": originals.get("rate_reset_at"),
+        },
+        "replies": {
+            "count": reply_count,
+            "pages": replies.get("pages_fetched", 0),
+            "complete": replies.get("complete", False),
+            "reason": replies.get("reason", "not_started"),
+            "rate_reset_at": replies.get("rate_reset_at"),
+        },
+        "unique_posts": total,
+        "complete": originals.get("complete", False) and replies.get("complete", False),
+    }
+
+
+def _write_batch_report(
+    output: Path, accounts: list[dict], result: CollectionResult
+) -> None:
+    posts = _posts(source_path(output, "posts.jsonl"))
+    by_author = Counter(post.author_id for post in posts)
+    role_counts = Counter((post.author_id, post.post_role) for post in posts)
+    month_counts = Counter((post.author_id, post.created_at_utc[:7]) for post in posts)
+    months = sorted({post.created_at_utc[:7] for post in posts})
+    lines = [
+        "# X 多账号采集报告",
+        "",
+        f"生成时间（UTC）：{now_utc()}",
+        f"去重后 **{result.post_count:,}** 条；所有账号分页结束：**{'是' if result.complete else '否'}**（{result.reason}）。",
+        "",
+        "| 账号 | 主帖 | 评论/回复 | 转发 | 去重条数 | 浏览量中位数 | Posts状态 | Replies状态 |",
+        "| --- | ---: | ---: | ---: | ---: | ---: | --- | --- |",
+    ]
+    for account in accounts:
+        views = [
+            post.view_count
+            for post in posts
+            if post.author_id == account["user_id"] and post.view_count is not None
+        ]
+        median_views = f"{median(views):,.0f}" if views else "—"
+        lines.append(
+            f"| [@{account['handle']}](https://x.com/{account['handle']}) "
+            f"| {role_counts[(account['user_id'], 'main')]:,} "
+            f"| {role_counts[(account['user_id'], 'comment')]:,} "
+            f"| {role_counts[(account['user_id'], 'repost')]:,} "
+            f"| {by_author[account['user_id']]:,} | {median_views} "
+            f"| {account['originals']['reason']} | {account['replies']['reason']} |"
+        )
+    lines += [
+        "",
+        "分类按每条帖子自身的回复/引用/转发关系，不按 Posts 或 Replies 标签页来源；两种标签页可能显示同一帖。`main` 包含原创和引用帖，`comment` 是有父帖 ID 的回复。",
+        "本次采集按目标账号的作者 ID 筛选，纯转发的原作者不是目标账号，因此未纳入表内；`repost=0` 不代表账号从未转发。",
+        "",
+        "## 每月可见帖子数（UTC）",
+        "",
+        "| 月份 | "
+        + " | ".join("@" + account["handle"] for account in accounts)
+        + " |",
+        "| --- | " + " | ".join("---:" for _ in accounts) + " |",
+    ]
+    for month in months:
+        lines.append(
+            "| "
+            + month
+            + " | "
+            + " | ".join(
+                str(month_counts[(account["user_id"], month)]) for account in accounts
+            )
+            + " |"
+        )
+    lines += [
+        "",
+        "## 覆盖边界",
+        "",
+        "`passed_since` 表示时间线已越过指定起始日；`source_end` 表示没有下一页；`empty_timeline_end` 表示连续两个只有游标的空页。最后一种是平台分页耗尽信号，不能单独证明历史全量。`rate_limited` 和 `page_limit` 表示尚未完成，可用同一命令续抓。",
+        "",
+        "删除、受保护、地区限制或未索引的帖子无法据此恢复。互动数是抓取时快照。原始 JSONL、CSV、账号资料、各账号游标及 SHA-256 哈希与本报告一同保存；登录 cookie 不写入输出。",
+        "",
+    ]
+    (internal_dir(output) / "report.md").write_text("\n".join(lines), encoding="utf-8")
+
+
+def collect_accounts(
+    client: "XClient",
+    handles: list[str],
+    output_dir: str | Path,
+    *,
+    since: str,
+    until: str,
+    pages_per_round: int = 5,
+    rounds: int = 1,
+    wait_on_rate_limit: bool = False,
+    progress: Callable[[dict], None] | None = None,
+) -> CollectionResult:
+    """Round-robin collection; rounds=0 continues until complete or stalled."""
+    if pages_per_round < 1 or rounds < 0:
+        raise ValueError("pages_per_round must be positive and rounds nonnegative")
+    if not handles:
+        raise ValueError("At least one account is required")
+    output = Path(output_dir).expanduser()
+    migrate_collection(output)
+    saved_profiles = source_path(output, "profiles.json")
+    if saved_profiles.exists():
+        for handle, record in json.loads(
+            saved_profiles.read_text(encoding="utf-8")
+        ).items():
+            client._profile_cache[handle.lower()] = Profile(**record)
+    profiles = {
+        client.get_user(handle).handle: client.get_user(handle).to_dict()
+        for handle in handles
+    }
+    canonical = list(dict.fromkeys(profiles))
+    _atomic_json(internal_dir(output) / "profiles.json", profiles)
+    query = {
+        "kind": "account_batch",
+        "handles": canonical,
+        "since": since,
+        "until": until,
+    }
+    store = PostStore(output, query=query)
+    if store.complete:
+        result = store.finish(complete=True, reason="all_accounts_complete")
+        _write_batch_report(
+            output, [_account_status(output, h, profiles[h]) for h in canonical], result
+        )
+        return result
+    completed_rounds = 0
+    reason = "round_limit"
+    while rounds == 0 or completed_rounds < rounds:
+        before = sum(
+            _state(
+                source_path(child_dir(child_dir(output, h), kind), "state.json")
+            ).get("pages_fetched", 0)
+            for h in canonical
+            for kind in ("originals", "replies")
+        )
+        for handle in canonical:
+            try:
+                client.save_user_activity(
+                    handle,
+                    child_dir(output, handle),
+                    since=since,
+                    until=until,
+                    max_pages=pages_per_round,
+                )
+            except PyXcomError as exc:
+                if progress:
+                    progress(
+                        {
+                            "account": handle,
+                            "error": type(exc).__name__,
+                            "detail": str(exc),
+                        }
+                    )
+            if progress:
+                progress(
+                    {
+                        "account": handle,
+                        **_account_status(output, handle, profiles[handle]),
+                    }
+                )
+        accounts = [_account_status(output, h, profiles[h]) for h in canonical]
+        collected: dict[str, Post] = {}
+        for handle in canonical:
+            for post in _posts(source_path(child_dir(output, handle), "posts.jsonl")):
+                collected[post.id] = post
+        store.append_page(list(collected.values()), None)
+        store.state["pages_fetched"] = sum(
+            account[kind]["pages"]
+            for account in accounts
+            for kind in ("originals", "replies")
+        )
+        completed_rounds += 1
+        _atomic_json(
+            internal_dir(output) / "account_manifest.json",
+            {"query": query, "accounts": accounts, "updated_at_utc": now_utc()},
+        )
+        checkpoint = store.finish(complete=False, reason="in_progress")
+        _write_batch_report(output, accounts, checkpoint)
+        if all(account["complete"] for account in accounts):
+            reason = "all_accounts_complete"
+            break
+        after = store.state["pages_fetched"]
+        if after <= before:
+            rate_states = [
+                account[kind]
+                for account in accounts
+                for kind in ("originals", "replies")
+                if account[kind]["reason"] == "rate_limited"
+            ]
+            if not rate_states:
+                reason = "stalled"
+                break
+            if not wait_on_rate_limit or rounds != 0:
+                reason = "rate_limited"
+                break
+            resets = [
+                state["rate_reset_at"]
+                for state in rate_states
+                if state.get("rate_reset_at") and state["rate_reset_at"] > time.time()
+            ]
+            wait = max(5, min(resets) - time.time() + 5) if resets else 900
+            if progress:
+                progress({"status": "rate_limited", "wait_seconds": round(wait)})
+            time.sleep(wait)
+    complete = reason == "all_accounts_complete"
+    result = store.finish(complete=complete, reason=reason)
+    accounts = [_account_status(output, h, profiles[h]) for h in canonical]
+    _write_batch_report(output, accounts, result)
+    return result

pyxcom/cli.py

@@ -0,0 +1,300 @@
+"""Command-line entry point for pyXcom."""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+from .client import XClient
+from .layout import internal_dir
+from .errors import PyXcomError
+from .schema import apply_role_schema, write_schema_report
+from .tables import export_tables
+from .validate import finalize_collection, validate_collection
+
+
+def _common_parser() -> argparse.ArgumentParser:
+    common = argparse.ArgumentParser(add_help=False)
+    common.add_argument("--browser", choices=("chrome", "edge"), default="chrome")
+    common.add_argument("--profile", help="Browser profile folder, e.g. 'Profile 3'")
+    common.add_argument(
+        "--cookie-db", type=Path, help="Explicit Chrome/Edge Cookies DB path"
+    )
+    common.add_argument(
+        "--proxy", help="HTTP or SOCKS proxy URL for X and mirror requests"
+    )
+    common.add_argument(
+        "--mirror-base",
+        help="Explicit HTTPS mirror for cross-user search; sends search terms to that mirror",
+    )
+    common.add_argument(
+        "--delay", type=float, default=1.0, help="Seconds between pages"
+    )
+    return common
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="pyxcom", description="Collect public X data")
+    commands = parser.add_subparsers(dest="command", required=True)
+    common = _common_parser()
+    profile = commands.add_parser(
+        "user", aliases=["profile"], parents=[common], help="Get public user profile"
+    )
+    profile.set_defaults(command="profile")
+    profile.add_argument("handle")
+    profile.add_argument("--output", type=Path)
+    post = commands.add_parser("post", parents=[common], help="Get one public post")
+    post.add_argument("post_id_or_url")
+    post.add_argument("--output", type=Path)
+    raw = commands.add_parser(
+        "raw-post", parents=[common], help="Get raw public X JSON"
+    )
+    raw.add_argument("post_id_or_url")
+    raw.add_argument("--output", type=Path)
+    posts = commands.add_parser(
+        "user-posts",
+        aliases=["posts"],
+        parents=[common],
+        help="Save authored main posts",
+    )
+    posts.set_defaults(command="posts")
+    posts.add_argument("handle")
+    posts.add_argument("--timeline", choices=("posts", "replies"), default="posts")
+    posts.add_argument("--since", help="Inclusive UTC date YYYY-MM-DD")
+    posts.add_argument("--until", help="Exclusive UTC date YYYY-MM-DD")
+    posts.add_argument("--max-pages", type=int)
+    posts.add_argument("--limit", type=int)
+    posts.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    activity = commands.add_parser(
+        "user-activity",
+        aliases=["activity"],
+        parents=[common],
+        help="Save posts and authored replies",
+    )
+    activity.set_defaults(command="activity")
+    activity.add_argument("handle")
+    activity.add_argument("--since")
+    activity.add_argument("--until")
+    activity.add_argument("--max-pages", type=int)
+    activity.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    search = commands.add_parser(
+        "search-posts",
+        aliases=["search"],
+        parents=[common],
+        help="Search keyword, dates, and user",
+    )
+    search.set_defaults(command="search")
+    search.add_argument("keyword", help="Literal word or phrase for direct user search")
+    search.add_argument(
+        "--handle", "--user", dest="user", help="Author handle for direct X search"
+    )
+    search.add_argument("--since", help="Inclusive UTC date YYYY-MM-DD")
+    search.add_argument("--until", help="Exclusive UTC date YYYY-MM-DD")
+    search.add_argument("--max-pages", type=int)
+    search.add_argument("--limit", type=int)
+    search.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    batch = commands.add_parser(
+        "users-activity",
+        aliases=["batch"],
+        parents=[common],
+        help="Collect several accounts and write a combined report",
+    )
+    batch.set_defaults(command="batch")
+    batch.add_argument("--handles", nargs="+", required=True)
+    batch.add_argument("--since", required=True)
+    batch.add_argument("--until", required=True)
+    batch.add_argument("--pages-per-round", type=int, default=5)
+    batch.add_argument(
+        "--rounds", type=int, default=1, help="0 keeps resuming until complete"
+    )
+    batch.add_argument("--wait-on-rate-limit", action="store_true")
+    batch.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    validate = commands.add_parser(
+        "validate", help="Check saved rows, scope, and SHA-256 hashes"
+    )
+    validate.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    finalize = commands.add_parser(
+        "finalize", help="Rebuild CSV and report after an interrupted run"
+    )
+    finalize.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    schema = commands.add_parser(
+        "schema", help="Report field coverage and optionally add main/comment columns"
+    )
+    schema.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    schema.add_argument(
+        "--apply", action="store_true", help="Backfill role/type in saved CSV and JSONL"
+    )
+    tables = commands.add_parser(
+        "export", help="Export saved records as users/posts/comments relational tables"
+    )
+    tables.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    replies = commands.add_parser(
+        "user-replies", parents=[common], help="Save replies authored by an account"
+    )
+    replies.set_defaults(command="posts", timeline="replies", limit=None)
+    replies.add_argument("handle")
+    replies.add_argument("--since")
+    replies.add_argument("--until")
+    replies.add_argument("--max-pages", type=int)
+    replies.add_argument("--limit", type=int)
+    replies.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    comments = commands.add_parser(
+        "post-comments",
+        parents=[common],
+        help="Collect replies below one main post, including nested replies",
+    )
+    comments.add_argument("post_id_or_url")
+    comments.add_argument("--max-depth", type=int, default=2)
+    comments.add_argument("--max-comments", type=int, default=100)
+    comments.add_argument("--max-pages", type=int, default=20)
+    comments.add_argument(
+        "--output-dir", "--output", dest="output", type=Path, required=True
+    )
+    return parser
+
+
+def _write_or_print(value: dict, output: Path | None) -> None:
+    text = json.dumps(value, ensure_ascii=False, indent=2)
+    if output is None:
+        print(text)
+    else:
+        output.parent.mkdir(parents=True, exist_ok=True)
+        output.write_text(text + "\n", encoding="utf-8")
+        print(str(output))
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.command == "export":
+            print(json.dumps(export_tables(args.output), ensure_ascii=False))
+            return 0
+        if args.command == "validate":
+            validation = validate_collection(args.output)
+            print(json.dumps(validation, ensure_ascii=False))
+            return 0 if validation["valid"] else 3
+        if args.command == "finalize":
+            result = finalize_collection(args.output)
+            print(json.dumps(result.to_dict(), ensure_ascii=False))
+            return 0
+        if args.command == "schema":
+            summary = (
+                apply_role_schema(args.output)
+                if args.apply
+                else write_schema_report(args.output)
+            )
+            print(
+                json.dumps(
+                    {
+                        "post_count": summary["post_count"],
+                        "post_field_count": summary["post_field_count"],
+                        "role_counts": summary["role_counts"],
+                        "type_counts": summary["type_counts"],
+                        "schema_report": str(
+                            internal_dir(args.output) / "schema_report.md"
+                        ),
+                        "changed_directories": len(
+                            summary.get("changed_directories", [])
+                        ),
+                    },
+                    ensure_ascii=False,
+                )
+            )
+            return 0
+        with XClient(
+            browser=args.browser,
+            profile=args.profile,
+            cookie_db=args.cookie_db,
+            proxy=args.proxy,
+            mirror_base=args.mirror_base,
+            delay=args.delay,
+        ) as client:
+            if args.command == "post-comments":
+                result = client.save_post_comments(
+                    args.post_id_or_url,
+                    args.output,
+                    max_depth=args.max_depth,
+                    max_comments=args.max_comments,
+                    max_pages=args.max_pages,
+                )
+                print(json.dumps(result.to_dict(), ensure_ascii=False))
+            elif args.command == "profile":
+                _write_or_print(client.get_user(args.handle).to_dict(), args.output)
+            elif args.command == "post":
+                _write_or_print(
+                    client.get_post(args.post_id_or_url).to_dict(), args.output
+                )
+            elif args.command == "raw-post":
+                _write_or_print(client.get_raw_post(args.post_id_or_url), args.output)
+            elif args.command == "posts":
+                result = client.save_user_posts(
+                    args.handle,
+                    args.output,
+                    timeline=args.timeline,
+                    since=args.since,
+                    until=args.until,
+                    max_pages=args.max_pages,
+                    limit=args.limit,
+                )
+                print(json.dumps(result.to_dict(), ensure_ascii=False))
+            elif args.command == "activity":
+                result = client.save_user_activity(
+                    args.handle,
+                    args.output,
+                    since=args.since,
+                    until=args.until,
+                    max_pages=args.max_pages,
+                )
+                print(json.dumps(result.to_dict(), ensure_ascii=False))
+            elif args.command == "search":
+                result = client.save_search_posts(
+                    args.keyword,
+                    args.output,
+                    handle=args.user,
+                    since=args.since,
+                    until=args.until,
+                    max_pages=args.max_pages,
+                    limit=args.limit,
+                )
+                print(json.dumps(result.to_dict(), ensure_ascii=False))
+            elif args.command == "batch":
+                result = client.save_users_activity(
+                    args.handles,
+                    args.output,
+                    since=args.since,
+                    until=args.until,
+                    pages_per_round=args.pages_per_round,
+                    rounds=args.rounds,
+                    wait_on_rate_limit=args.wait_on_rate_limit,
+                    progress=lambda item: print(
+                        json.dumps(item, ensure_ascii=False), flush=True
+                    ),
+                )
+                print(json.dumps(result.to_dict(), ensure_ascii=False))
+    except (PyXcomError, ValueError, OSError) as exc:
+        print(f"pyxcom: {exc}", file=sys.stderr)
+        return 2
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())