scraper-for-x 0.1.0__tar.gz

scraper_for_x-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,87 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  lint-and-test:
+    # macOS is the first-class, tested target (v1); the ubuntu leg exists
+    # because the fixture/parse/CLI layer is browser-agnostic, and running it
+    # on Linux too is a cheap way to make that cross-platform claim real
+    # (plan §14). Neither leg installs an actual browser binary.
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install pinned dev dependencies
+        run: python -m pip install -r requirements-dev.lock
+
+      - name: Install package (dependencies already pinned above)
+        run: python -m pip install -e . --no-deps
+
+      - name: Ruff lint
+        run: ruff check .
+
+      - name: Ruff format check
+        run: ruff format --check .
+
+      - name: Fixture PII/secret scan
+        run: python scripts/check_fixtures_pii.py
+
+      - name: Run tests
+        run: pytest
+
+  build-and-smoke:
+    # This installs the BASE wheel only — no [browser] extra, so scrapling is
+    # never installed in this job. That is deliberate: scrapling is an extra
+    # for this package (unlike scrape-fb, where it's a base dependency and
+    # that package's smoke test only guards against a missing browser
+    # *binary*). Here, the load-bearing regression this job catches is an
+    # eager top-level `import scrapling` anywhere in the base import path
+    # (session.py/cli.py/__init__.py) — if that ever creeps in, `scrape-x
+    # --version` fails at import time with scrapling not even installed,
+    # and this job is the only thing that would catch it (plan §13/§14).
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-latest, ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install build tool
+        run: python -m pip install build
+
+      - name: Build wheel
+        run: python -m build --wheel
+
+      - name: Install built (base) wheel into a clean venv
+        # POSIX venv layout (bin/, not Scripts\) and the bash default shell
+        # both assume macos-latest/ubuntu-latest only — adding windows-latest
+        # to the matrix above would need this step rewritten, not just the
+        # matrix line (Windows unsupported in v1; see README/pyproject).
+        run: |
+          python -m venv smoke-venv
+          smoke-venv/bin/pip install dist/*.whl
+
+      - name: Smoke test the entry point (scrapling absent)
+        run: |
+          smoke-venv/bin/scrape-x --version
+          smoke-venv/bin/scrape-x --help

scraper_for_x-0.1.0/.github/workflows/publish.yml

@@ -0,0 +1,52 @@
+name: Publish to PyPI
+
+# Fires when a GitHub Release is published (Releases page -> "Draft a new
+# release" -> pick/create tag vX.Y.Z -> Publish release) — NOT on a bare
+# `git push --tags`, which creates the tag but no visible Release entry.
+# github.ref_name still resolves to the release's tag name under this event.
+on:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Verify tag matches pyproject.toml version
+        run: python3 scripts/check_tag_version.py "${{ github.ref_name }}"
+
+      - name: Install build tool
+        run: python -m pip install build
+
+      - name: Build sdist and wheel
+        run: python -m build
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    permissions:
+      # Required to mint the OIDC token for PyPI Trusted Publishing — there is
+      # no stored API token anywhere in this repo (plan §15).
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      # Pinned to the commit for release v1.14.0 — a floating tag on an
+      # action that runs with publish credentials is a hijack vector
+      # (plan §15). Bump deliberately, not automatically.
+      - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0

scraper_for_x-0.1.0/.gitignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+build/
+dist/
+
+# Dev environment
+.venv/
+.ruff_cache/
+.pytest_cache/
+uv.lock
+
+# Raw captures and runtime outputs are a PII trap — ignore by an unambiguous,
+# distinct name/location so a safe fixture and an unsafe raw capture can never
+# collide under one glob (see plan §4).
+scratch/
+*.raw.json
+output/
+profiles/
+
+# NOTE: fixtures under tests/fixtures/*.json are committed, synthetic-by-
+# construction (see scripts/check_fixtures_pii.py) and need NO un-ignore
+# lines here, because the raw-capture ignore above is scoped to `*.raw.json`
+# specifically, not a blanket `*.json` — a real capture and a fixture can
+# never collide under one glob. If that ever changes to a blanket `*.json`
+# ignore, every fixture would then need un-ignoring BY EXACT NAME, never a
+# wildcard (`!tests/fixtures/*.json` would silently re-include any new raw
+# capture dropped into the same directory).
+
+.DS_Store

scraper_for_x-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.6
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: local
+    hooks:
+      - id: fixture-pii-scan
+        name: scan committed fixtures for PII/secrets
+        entry: python3 scripts/check_fixtures_pii.py
+        language: system
+        files: ^tests/fixtures/.*\.json$
+        pass_filenames: false

scraper_for_x-0.1.0/CHANGELOG.md

@@ -0,0 +1,23 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+## [0.1.0] - 2026-07-05
+
+### Added
+- Initial release: `scrape-x login` / `status` / `setup` / `doctor` / `fetch` / `search` / `tweet`.
+- Logged-in X/Twitter reads via a harvest-then-replay hybrid (stealth-browser login, `httpx` GraphQL replay). Confirmed live: `UserTweets` and `TweetDetail` need no `x-client-transaction-id`.
+- `--limit`, `--since`/`--until` retrieval with stop-reason reporting.
+- JSON and NDJSON output formats.
+- Python API: `XScraper`, `Tweet`, `User`, `Media`.
+
+### Known limitations
+- `scrape-x search` and `scrape-x fetch --replies` (and the Python API's `search()`/`fetch_user_tweets(replies=True)`/`iter_user_tweets(replies=True)`) are **not yet implemented** — live-verified 2026-07-05 that `SearchTimeline`/`UserTweetsAndReplies` require a single-use `x-client-transaction-id` per request, which this package's harvest-then-replay architecture doesn't reproduce. Both fail fast with a clear `FeatureNotImplementedError` (exit code 1). See [wiki/FAQ-and-Troubleshooting.md](wiki/FAQ-and-Troubleshooting.md).
+
+[Unreleased]: https://github.com/tjdwls101010/Scraper-for-X/compare/v0.1.0...HEAD
+[0.1.0]: https://github.com/tjdwls101010/Scraper-for-X/releases/tag/v0.1.0

scraper_for_x-0.1.0/CLAUDE.md

@@ -0,0 +1,65 @@
+# CLAUDE.md
+
+Behavioral guidelines to reduce common LLM coding mistakes. Merge with project-specific instructions as needed.
+
+**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment.
+
+## 1. Think Before Coding
+
+**Don't assume. Don't hide confusion. Surface tradeoffs.**
+
+Before implementing:
+- State your assumptions explicitly. If uncertain, ask.
+- If multiple interpretations exist, present them - don't pick silently.
+- If a simpler approach exists, say so. Push back when warranted.
+- If something is unclear, stop. Name what's confusing. Ask.
+
+## 2. Simplicity First
+
+**Minimum code that solves the problem. Nothing speculative.**
+
+- No features beyond what was asked.
+- No abstractions for single-use code.
+- No "flexibility" or "configurability" that wasn't requested.
+- No error handling for impossible scenarios.
+- If you write 200 lines and it could be 50, rewrite it.
+
+Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify.
+
+## 3. Surgical Changes
+
+**Touch only what you must. Clean up only your own mess.**
+
+When editing existing code:
+- Don't "improve" adjacent code, comments, or formatting.
+- Don't refactor things that aren't broken.
+- Match existing style, even if you'd do it differently.
+- If you notice unrelated dead code, mention it - don't delete it.
+
+When your changes create orphans:
+- Remove imports/variables/functions that YOUR changes made unused.
+- Don't remove pre-existing dead code unless asked.
+
+The test: Every changed line should trace directly to the user's request.
+
+## 4. Goal-Driven Execution
+
+**Define success criteria. Loop until verified.**
+
+Transform tasks into verifiable goals:
+- "Add validation" → "Write tests for invalid inputs, then make them pass"
+- "Fix the bug" → "Write a test that reproduces it, then make it pass"
+- "Refactor X" → "Ensure tests pass before and after"
+
+For multi-step tasks, state a brief plan:
+```
+1. [Step] → verify: [check]
+2. [Step] → verify: [check]
+3. [Step] → verify: [check]
+```
+
+Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification.
+
+---
+
+**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions come before implementation rather than after mistakes.

scraper_for_x-0.1.0/DISCLAIMER.md

@@ -0,0 +1,54 @@
+# Disclaimer — read before you use this
+
+This is not legal advice. It is a plain-language summary of risks you take on by using this tool. If any of this matters to your situation, talk to a lawyer.
+
+## 1. This violates X's Terms of Service
+
+X's Terms of Service prohibit scraping and automated access without written permission. X enforces this via account suspension/termination and, historically, litigation. Using this tool is entirely at your own risk. Use a **dedicated or throwaway account**, not your primary one, and keep volume low (§9 of the design doc; the CLI's non-bypassable inter-request floor and per-run scope exist because of this).
+
+## 2. X is notably litigious about scraping — but read the outcomes honestly
+
+X Corp has pursued scraping-related litigation (e.g. *X Corp v. Bright Data*), but the documented outcomes and enforcement have skewed toward **commercial mass-scrapers and data brokers**, not solo personal-scale read use. The probability of that landing on any one individual doing low-volume personal reads is uncertain; the exposure is nonetheless real, and this is named here so the choice stays informed rather than assumed away.
+
+## 3. Publishing this tool exposes its maintainer, not just its users
+
+This package is named `scraper-for-x`, published under a real GitHub identity, and distributed via PyPI Trusted Publishing, which binds each release to a named GitHub repository and account (`tjdwls101010/Scraper-for-X`). That is a deliberate, informed choice by the maintainer — but it means the maintainer is identifiable in a way an anonymous or unpublished tool would not be. This exposure exists regardless of how the tool is actually used by others, and is recorded here so the choice stays informed.
+
+## 4. You may become a "data controller" for other people's data
+
+Tweets you scrape belong to other people — authors, repliers, anyone quoted or mentioned. Collecting identifiable personal data about other people can make *you* a **data controller under GDPR**, with real obligations: a lawful basis for processing, honoring data-subject access/deletion requests, and limiting retention. "I did this for personal use" is not automatically a lawful basis.
+
+The **CCPA/CPRA** is different in kind: it regulates for-profit "businesses" that meet statutory thresholds (e.g. revenue or data-volume tests) and generally exempts purely personal/household activity — so it typically does *not* attach to a solo, personal-scale scraper. Even so, jurisdiction-specific privacy-tort and publication-of-private-facts risk can still apply outside of CCPA/CPRA's scope. Minimize what you keep, and delete captured output once you're done with it. The MIT license on this code says nothing about, and does not excuse, privacy-law obligations around the *data* you collect with it. *(Non-legal-advice.)*
+
+## 5. Output files are not scrubbed — treat them as sensitive
+
+Captured tweets contain third-party names, tweet text, and media URLs. This tool:
+- never writes output to a location you'd casually commit to git (default `--output` is outside any repo; see README),
+- never redacts the *output* files themselves (only diagnostic/verbose logs go through redaction — see below),
+- relies on you to delete output you no longer need.
+
+Don't commit scraped output to a public (or even private) git repository, and don't share it beyond what you'd be comfortable being responsible for under §4.
+
+## 6. Diagnostics are redacted — but redaction is not a certification
+
+`-v`/`--verbose` output, error dumps, and anything printed to your terminal are passed through a single redaction path that strips session-token-shaped fields, `pbs.twimg.com`/`video.twimg.com` query strings, and truncates tweet text and names. This includes **cookie-import parse errors** — a malformed cookie export file is reported by line/field position ("line 3 malformed", "ct0 failed shape check"), never by echoing the raw cookie value. This reduces accidental leakage into terminal scrollback, bug reports, or screenshots — it is **not** a guarantee that every sensitive value is caught, and it does not apply to the actual `--output` file, which is the full, unredacted capture by design (that's the point of the tool). `--raw --no-redact` disables this path entirely for debugging; only use it locally.
+
+## 7. Account-ban risk, and how to reduce it
+
+Automating an X account — even just reading, via a real logged-in session — violates X's Terms of Service and X is aggressive about flagging automation. To reduce (not eliminate) the risk of a suspension or challenge:
+- Use a **dedicated or throwaway account**, never your primary one.
+- Keep volume low: shallow, recent fetches are safer than deep archival pulls. Deeper `--since`/`--limit` runs make more requests and raise both rate-limit and account-flag risk.
+- Run `scrape-x` from the **same network/IP** where the session was originally established (`scrape-x login`, or wherever you exported cookies from). An abrupt IP or client change against an existing session — especially pairing a cookie-imported session with a datacenter/VPN IP — is exactly the kind of signal X's abuse systems weight, and can soft-lock the session even without an outright ban.
+- Never run this in a loop or scheduler; there is no built-in scheduler/daemon, and none will be added.
+
+## 8. Your session credential is a live, password-less login — protect it
+
+`scrape-x login` (or cookie import) persists your X session as `{auth_token, ct0, user-agent}` to a directory on disk, permissioned `0700` with the credential file itself `0600`. Anyone who can read that file has authenticated, password-less access to your X account — no password or 2FA required, because the session already satisfied both. Concretely:
+- **Do not** back this up to Time Machine, sync it via iCloud/Dropbox, or commit it anywhere.
+- If you import cookies from an exported file, that source file *also* still contains a live session after import — delete or secure it; this tool never deletes it for you.
+- If the machine or disk is lost or compromised, **revoke the session immediately** by logging out of that session on x.com (Settings → Security and account access → Apps and sessions, or equivalent), not just by deleting the local directory.
+- A printed reminder from this tool (at login, or on cookie import) is exactly that — a reminder. **It is not a technical control.** Nothing in this tool encrypts the credential at rest or prevents you from copying it elsewhere; the `0700`/`0600` permissions and the reminder are the entire enforcement mechanism.
+
+## 9. No warranty
+
+This software is provided "as is" under the MIT License, without warranty of any kind. See [LICENSE](LICENSE). X's internal API can and does change without notice — query-ids rotate, response shapes drift — and this tool may stop working, or silently return incomplete data, at any time (see the design doc's durability section).

scraper_for_x-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 tjdwls101010
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

scraper_for_x-0.1.0/PKG-INFO

@@ -0,0 +1,118 @@
+Metadata-Version: 2.4
+Name: scraper-for-x
+Version: 0.1.0
+Summary: Read-only scraping of logged-in X/Twitter data via a harvest-then-replay hybrid: a stealth-browser login harvests the session once, then reads are replayed over httpx. No client-transaction-id replay.
+Project-URL: Homepage, https://github.com/tjdwls101010/Scraper-for-X
+Project-URL: Issues, https://github.com/tjdwls101010/Scraper-for-X/issues
+Project-URL: Changelog, https://github.com/tjdwls101010/Scraper-for-X/blob/main/CHANGELOG.md
+Author: tjdwls101010
+License-Expression: MIT
+License-File: LICENSE
+Keywords: graphql,scraper,scraping,twitter,x
+Classifier: Development Status :: 3 - Alpha
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: MacOS
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.11
+Requires-Dist: httpx>=0.27
+Requires-Dist: platformdirs>=4.0
+Provides-Extra: browser
+Requires-Dist: scrapling[fetchers]<0.5,>=0.4.10; extra == 'browser'
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: pre-commit>=3.8; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.8; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# scraper-for-x
+
+Read-only scraping of logged-in **X/Twitter** data via a **harvest-then-replay hybrid**: a stealth browser (or a cookie import) logs you in once and harvests the session, then every read afterward is a plain `httpx` GraphQL request — no browser in the loop, no `x-client-transaction-id` replay.
+
+> **Read [DISCLAIMER.md](DISCLAIMER.md) before using this.** Using this tool violates X's Terms of Service, publishing it exposes its maintainer, and scraping other people's tweets can make *you* a data controller over their personal data under GDPR. Use a dedicated/throwaway account, not your primary one.
+
+## Installation
+
+Base install — cookie-import login only, **no browser dependency**:
+
+```bash
+pip install scraper-for-x
+```
+
+The `[browser]` extra — adds a stealth browser for `scrape-x login`:
+
+```bash
+pip install "scraper-for-x[browser]"
+```
+
+If you only ever import cookies from a session you already have (e.g. exported from your own logged-in browser), the base install is all you need.
+
+## Quick Start
+
+```bash
+# 1. One-time interactive login — opens a real browser window, you log in by hand.
+scrape-x login
+
+# 2. Fetch a profile's tweets.
+scrape-x fetch nasa --limit 50
+```
+
+## CLI overview
+
+| Command | Purpose |
+|---|---|
+| `scrape-x login` | One-time login: headed stealth browser by default, or `--cookies FILE` to import an existing session |
+| `scrape-x status` | Check whether the persisted session is logged in, expired, or rate-limited |
+| `scrape-x setup` | Provision the login browser into an isolated cache (requires `[browser]`) |
+| `scrape-x doctor` | Authenticated round-trip + query-id freshness check (`--refresh` re-anchors query-ids from x.com's `main.js`, browser-free) |
+| `scrape-x fetch <identifier>` | A profile's tweets/media (`--limit`, `--since`, `--until`, `--by screen_name\|id`) — **`--replies` not yet implemented, see below** |
+| `scrape-x search <query>` | **Not yet implemented, see below** |
+| `scrape-x tweet <identifier>` | A single tweet plus its reply/conversation thread (`--replies`) |
+
+`fetch`/`search`/`tweet` all share `--format json|ndjson`, `--output PATH`, `--profile NAME`, `--profile-dir PATH`, `--wait-on-limit`, `--max-wait`, `--raw` (+ `--no-redact`), and `-v/--verbose`. See the [CLI Reference](wiki/CLI-Reference.md) for every flag and exit code.
+
+### Known limitation: `search` and `fetch --replies` (v0.1.0)
+
+Live-verified 2026-07-05: X's `SearchTimeline` and `UserTweetsAndReplies` GraphQL operations require a fresh, **single-use** `x-client-transaction-id` header on every request — unlike plain profile fetches and single-tweet lookups, which this package proved work over plain `httpx` replay with no such header. A captured transaction-id cannot be harvested once and reused like a session cookie or query-id; reproducing X's generator for it is exactly the fragility this package's harvest-then-replay architecture was built to avoid (see [twikit#408](https://github.com/d60/twikit/issues/408)).
+
+Both `scrape-x search` and `scrape-x fetch --replies` fail fast with a clear `FeatureNotImplementedError` (exit code 1) rather than a confusing network error. A browser-observe fallback for these two operations specifically (the same approach `scrape-fb` uses) is on the roadmap — see [wiki/FAQ-and-Troubleshooting.md](wiki/FAQ-and-Troubleshooting.md).
+
+## Python API
+
+```python
+from scraper_for_x import XScraper
+
+XScraper(profile="default").login()  # one-time, opens a headed browser
+
+with XScraper(profile="default") as x:
+    tweets = x.fetch_user_tweets("nasa", limit=50)
+    for tweet in x.iter_user_tweets("nasa", limit=50):
+        ...  # must be consumed inside the `with` block
+
+    x.fetch_tweet("https://x.com/nasa/status/1234567890")
+    # x.search(...) raises FeatureNotImplementedError -- see "Known limitation" above.
+```
+
+## Documentation
+
+This README covers the essentials. For everything else, see the **[wiki](wiki/README.md)**:
+
+- [Installation](wiki/Installation.md)
+- [Quick Start](wiki/Quick-Start.md)
+- [CLI Reference](wiki/CLI-Reference.md) — every flag, every exit code
+- [Python API Reference](wiki/Python-API-Reference.md)
+- [Configuration](wiki/Configuration.md) — profiles, environment variables
+- [Output Schema](wiki/Output-Schema.md) — every `Tweet`/`User`/`Media` field
+- [Security and Privacy](wiki/Security-and-Privacy.md) — the full threat model behind [DISCLAIMER.md](DISCLAIMER.md)
+- [FAQ and Troubleshooting](wiki/FAQ-and-Troubleshooting.md)
+- [Contributing](wiki/Contributing.md)
+
+## License
+
+MIT — see [LICENSE](LICENSE). The license covers the code; it does not cover what you do with the data you collect (see [DISCLAIMER.md](DISCLAIMER.md)).

scraper_for_x-0.1.0/README.md

@@ -0,0 +1,85 @@
+# scraper-for-x
+
+Read-only scraping of logged-in **X/Twitter** data via a **harvest-then-replay hybrid**: a stealth browser (or a cookie import) logs you in once and harvests the session, then every read afterward is a plain `httpx` GraphQL request — no browser in the loop, no `x-client-transaction-id` replay.
+
+> **Read [DISCLAIMER.md](DISCLAIMER.md) before using this.** Using this tool violates X's Terms of Service, publishing it exposes its maintainer, and scraping other people's tweets can make *you* a data controller over their personal data under GDPR. Use a dedicated/throwaway account, not your primary one.
+
+## Installation
+
+Base install — cookie-import login only, **no browser dependency**:
+
+```bash
+pip install scraper-for-x
+```
+
+The `[browser]` extra — adds a stealth browser for `scrape-x login`:
+
+```bash
+pip install "scraper-for-x[browser]"
+```
+
+If you only ever import cookies from a session you already have (e.g. exported from your own logged-in browser), the base install is all you need.
+
+## Quick Start
+
+```bash
+# 1. One-time interactive login — opens a real browser window, you log in by hand.
+scrape-x login
+
+# 2. Fetch a profile's tweets.
+scrape-x fetch nasa --limit 50
+```
+
+## CLI overview
+
+| Command | Purpose |
+|---|---|
+| `scrape-x login` | One-time login: headed stealth browser by default, or `--cookies FILE` to import an existing session |
+| `scrape-x status` | Check whether the persisted session is logged in, expired, or rate-limited |
+| `scrape-x setup` | Provision the login browser into an isolated cache (requires `[browser]`) |
+| `scrape-x doctor` | Authenticated round-trip + query-id freshness check (`--refresh` re-anchors query-ids from x.com's `main.js`, browser-free) |
+| `scrape-x fetch <identifier>` | A profile's tweets/media (`--limit`, `--since`, `--until`, `--by screen_name\|id`) — **`--replies` not yet implemented, see below** |
+| `scrape-x search <query>` | **Not yet implemented, see below** |
+| `scrape-x tweet <identifier>` | A single tweet plus its reply/conversation thread (`--replies`) |
+
+`fetch`/`search`/`tweet` all share `--format json|ndjson`, `--output PATH`, `--profile NAME`, `--profile-dir PATH`, `--wait-on-limit`, `--max-wait`, `--raw` (+ `--no-redact`), and `-v/--verbose`. See the [CLI Reference](wiki/CLI-Reference.md) for every flag and exit code.
+
+### Known limitation: `search` and `fetch --replies` (v0.1.0)
+
+Live-verified 2026-07-05: X's `SearchTimeline` and `UserTweetsAndReplies` GraphQL operations require a fresh, **single-use** `x-client-transaction-id` header on every request — unlike plain profile fetches and single-tweet lookups, which this package proved work over plain `httpx` replay with no such header. A captured transaction-id cannot be harvested once and reused like a session cookie or query-id; reproducing X's generator for it is exactly the fragility this package's harvest-then-replay architecture was built to avoid (see [twikit#408](https://github.com/d60/twikit/issues/408)).
+
+Both `scrape-x search` and `scrape-x fetch --replies` fail fast with a clear `FeatureNotImplementedError` (exit code 1) rather than a confusing network error. A browser-observe fallback for these two operations specifically (the same approach `scrape-fb` uses) is on the roadmap — see [wiki/FAQ-and-Troubleshooting.md](wiki/FAQ-and-Troubleshooting.md).
+
+## Python API
+
+```python
+from scraper_for_x import XScraper
+
+XScraper(profile="default").login()  # one-time, opens a headed browser
+
+with XScraper(profile="default") as x:
+    tweets = x.fetch_user_tweets("nasa", limit=50)
+    for tweet in x.iter_user_tweets("nasa", limit=50):
+        ...  # must be consumed inside the `with` block
+
+    x.fetch_tweet("https://x.com/nasa/status/1234567890")
+    # x.search(...) raises FeatureNotImplementedError -- see "Known limitation" above.
+```
+
+## Documentation
+
+This README covers the essentials. For everything else, see the **[wiki](wiki/README.md)**:
+
+- [Installation](wiki/Installation.md)
+- [Quick Start](wiki/Quick-Start.md)
+- [CLI Reference](wiki/CLI-Reference.md) — every flag, every exit code
+- [Python API Reference](wiki/Python-API-Reference.md)
+- [Configuration](wiki/Configuration.md) — profiles, environment variables
+- [Output Schema](wiki/Output-Schema.md) — every `Tweet`/`User`/`Media` field
+- [Security and Privacy](wiki/Security-and-Privacy.md) — the full threat model behind [DISCLAIMER.md](DISCLAIMER.md)
+- [FAQ and Troubleshooting](wiki/FAQ-and-Troubleshooting.md)
+- [Contributing](wiki/Contributing.md)
+
+## License
+
+MIT — see [LICENSE](LICENSE). The license covers the code; it does not cover what you do with the data you collect (see [DISCLAIMER.md](DISCLAIMER.md)).

scraper_for_x-0.1.0/pyproject.toml

@@ -0,0 +1,62 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "scraper-for-x"
+version = "0.1.0"
+description = "Read-only scraping of logged-in X/Twitter data via a harvest-then-replay hybrid: a stealth-browser login harvests the session once, then reads are replayed over httpx. No client-transaction-id replay."
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.11"
+authors = [{ name = "tjdwls101010" }]
+keywords = ["twitter", "x", "scraper", "scraping", "graphql"]
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Environment :: Console",
+    "Intended Audience :: Developers",
+    "Operating System :: MacOS",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Internet :: WWW/HTTP :: Indexing/Search",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+dependencies = [
+    "httpx>=0.27",
+    "platformdirs>=4.0",
+]
+
+[project.optional-dependencies]
+browser = [
+    "scrapling[fetchers]>=0.4.10,<0.5",
+]
+dev = [
+    "pytest>=8.0",
+    "ruff>=0.8",
+    "pre-commit>=3.8",
+    "build>=1.2",
+]
+
+[project.urls]
+Homepage = "https://github.com/tjdwls101010/Scraper-for-X"
+Issues = "https://github.com/tjdwls101010/Scraper-for-X/issues"
+Changelog = "https://github.com/tjdwls101010/Scraper-for-X/blob/main/CHANGELOG.md"
+
+[project.scripts]
+scrape-x = "scraper_for_x.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/scraper_for_x"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

scraper_for_x-0.1.0/requirements-dev.lock

@@ -0,0 +1,34 @@
+# Pinned dev/CI dependency snapshot for reproducible CI runs (plan §14).
+# NOT the published package's dependency spec — that lives in pyproject.toml.
+# Regenerate with: uv pip freeze --python .venv/bin/python | grep -v '^scraper\|^-e '
+#
+# Deliberately does NOT include scrapling: the base install (and this repo's
+# CI lint-and-test job) never needs it -- only the wheel-install smoke test's
+# explicit job is to confirm scrapling's ABSENCE from a clean base install
+# (plan §13/§14 G-lazy-import). `scrapling[fetchers]` lives solely under the
+# `[browser]` extra in pyproject.toml.
+anyio==4.14.1
+build==1.5.0
+certifi==2026.6.17
+cfgv==3.5.0
+distlib==0.4.3
+filelock==3.29.5
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+identify==2.6.19
+idna==3.18
+iniconfig==2.3.0
+nodeenv==1.10.0
+packaging==26.2
+platformdirs==4.10.0
+pluggy==1.6.0
+pre-commit==4.6.0
+pygments==2.20.0
+pyproject-hooks==1.2.0
+pytest==9.1.1
+python-discovery==1.4.3
+pyyaml==6.0.3
+ruff==0.15.20
+typing-extensions==4.16.0
+virtualenv==21.5.1