broadcastx 0.1.0__py3-none-any.whl

broadcastx/__init__.py

@@ -0,0 +1,3 @@
+"""BroadcastX - Discover and download X/Twitter broadcast videos."""
+
+__version__ = "0.1.0"

broadcastx/cli.py

@@ -0,0 +1,244 @@
+"""
+BroadcastX CLI — Discover and download X/Twitter broadcast videos.
+
+Usage:
+    broadcastx scan @username
+    broadcastx download https://x.com/i/broadcasts/...
+    broadcastx download --from broadcasts.json
+"""
+
+import asyncio
+from pathlib import Path
+
+import click
+from rich.console import Console
+
+from . import __version__
+from .config import DEFAULT_BROADCASTS_FILE, DEFAULT_BROWSER, DEFAULT_VIDEOS_DIR
+from .downloader import check_ffmpeg, check_yt_dlp, download_all, download_broadcast
+from .monitor import monitor_user
+from .pause_detector import detect_pauses, pause_report, trim_intervals
+from .scanner import scan_user
+from .scrape_broadcasts import scrape_broadcasts
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version=__version__, prog_name="broadcastx")
+def main():
+    """BroadcastX — Discover and download X/Twitter broadcast videos."""
+    pass
+
+
+@main.command()
+@click.argument("username")
+@click.option("--max-scrolls", "-n", default=100, help="Maximum scroll actions (default: 100)")
+@click.option("--scroll-delay", "-d", default=2.0, help="Delay between scrolls in seconds (default: 2.0)")
+@click.option("--idle-timeout", "-t", default=10.0, help="Stop after N seconds with no new data (default: 10)")
+@click.option("--output", "-o", default=None, help="Output JSON file path")
+@click.option("--headless/--no-headless", default=False, help="Run browser headless (default: visible)")
+def scan(username, max_scrolls, scroll_delay, idle_timeout, output, headless):
+    """Scan a user's timeline for broadcast links.
+
+    USERNAME can be with or without @ (e.g., @elonmusk or elonmusk).
+    """
+    asyncio.run(scan_user(
+        username=username,
+        max_scrolls=max_scrolls,
+        scroll_delay=scroll_delay,
+        idle_timeout=idle_timeout,
+        headless=headless,
+        output_file=output,
+    ))
+
+
+@main.command()
+@click.argument("username")
+@click.option("--output", "-o", default=None, help="Output JSON file path")
+@click.option("--delay", default=1.0, help="Delay between API calls in seconds (default: 1.0)")
+@click.option("--headless/--no-headless", default=False, help="Run browser headless (default: visible)")
+@click.option("--verbose", "-v", is_flag=True, help="Show detailed output")
+@click.option("--fresh", is_flag=True, help="Ignore saved state, start from beginning")
+@click.option("--auth-token", default=None, help="Manual auth_token cookie (skips browser)")
+@click.option("--csrf-token", default=None, help="Manual ct0/CSRF token (skips browser)")
+@click.option("--user-id", default=None, help="Manual user ID (skips user ID lookup)")
+def scrape(username, output, delay, headless, verbose, fresh, auth_token, csrf_token, user_id):
+    """Scrape ALL past broadcasts from a user's timeline.
+
+    Uses GraphQL API pagination. Saves cursor state so you can resume
+    after rate limits. Run the same command again to continue.
+
+    USERNAME can be with or without @ (e.g., @SpaceX or SpaceX).
+
+    Examples:
+
+        broadcastx scrape @SpaceX
+
+        broadcastx scrape @SpaceX --fresh    # ignore saved state
+
+        broadcastx scrape @SpaceX --delay 2.0 -v
+    """
+    if fresh:
+        from .scrape_broadcasts import _state_file
+        state_path = _state_file(username.lstrip("@"))
+        if state_path.exists():
+            state_path.unlink()
+            console.print(f"[dim]Cleared saved state: {state_path}[/dim]")
+
+    asyncio.run(scrape_broadcasts(
+        username=username,
+        headless=headless,
+        output_file=output,
+        delay=delay,
+        verbose=verbose,
+        auth_token=auth_token,
+        csrf_token=csrf_token,
+        user_id=user_id,
+    ))
+
+
+@main.command()
+@click.argument("username")
+@click.option("--check-interval", default=30 * 60, help="Seconds between profile checks (default: 1800)")
+@click.option("--live-interval", default=5 * 60, help="Seconds between live-status checks (default: 300)")
+@click.option("--output", "-o", default=None, help="Monitor event JSON file path")
+@click.option("--output-dir", default=None, help="Directory for downloaded videos")
+@click.option("--browser", "-b", default=DEFAULT_BROWSER, help=f"Browser for yt-dlp cookies (default: {DEFAULT_BROWSER})")
+@click.option("--headless/--no-headless", default=False, help="Run browser headless (default: visible)")
+@click.option("--download/--no-download", default=True, help="Download when broadcast ends (default: download)")
+@click.option("--once", is_flag=True, help="Run one detection cycle, useful for testing")
+def monitor(username, check_interval, live_interval, output, output_dir, browser, headless, download, once):
+    """Monitor a profile for current live broadcasts and download ended replays.
+
+    USERNAME can be with or without @ (e.g., @SpaceX or SpaceX).
+    """
+    if download and not check_yt_dlp():
+        console.print("[red]✗ yt-dlp not found.[/red]")
+        console.print("  Install with: [bold]brew install yt-dlp[/bold]")
+        raise SystemExit(1)
+
+    if download and not check_ffmpeg():
+        console.print("[red]✗ ffmpeg not found.[/red]")
+        console.print("  Install with: [bold]brew install ffmpeg[/bold]")
+        raise SystemExit(1)
+
+    asyncio.run(monitor_user(
+        username=username,
+        check_interval=check_interval,
+        live_interval=live_interval,
+        headless=headless,
+        output_file=output,
+        output_dir=output_dir,
+        browser=browser,
+        download=download,
+        once=once,
+    ))
+
+
+@main.command()
+@click.argument("urls", nargs=-1)
+@click.option("--from", "from_file", default=None, type=click.Path(), help="Load URLs from a JSON file")
+@click.option("--output-dir", "-o", default=None, help="Output directory for videos")
+@click.option("--browser", "-b", default=DEFAULT_BROWSER, help=f"Browser for cookies (default: {DEFAULT_BROWSER})")
+@click.option("--verbose", "-v", is_flag=True, help="Show yt-dlp output")
+@click.option("--parallel", "-p", default=1, help="Number of concurrent downloads (default: 1)")
+def download(urls, from_file, output_dir, browser, verbose, parallel):
+    """Download broadcast video(s).
+
+    Pass one or more broadcast URLs directly, or use --from to load from a JSON file.
+
+    Examples:
+
+        broadcastx download https://x.com/i/broadcasts/1vAxRkBbDRzKl
+
+        broadcastx download --from output/broadcasts.json
+
+        broadcastx download --from output/broadcasts.json -o ./my_videos
+
+    Rotation correction is applied automatically: if the broadcast carries
+    phone-orientation metadata, the downloaded video is re-encoded so it
+    displays upright. A `.rotation.jsonl` sidecar is also written alongside
+    the video for inspection.
+    """
+    # Pre-flight checks
+    if not check_yt_dlp():
+        console.print("[red]✗ yt-dlp not found.[/red]")
+        console.print("  Install with: [bold]brew install yt-dlp[/bold]")
+        raise SystemExit(1)
+
+    if not check_ffmpeg():
+        console.print("[red]✗ ffmpeg not found.[/red]")
+        console.print("  Install with: [bold]brew install ffmpeg[/bold]")
+        raise SystemExit(1)
+
+    if not urls and not from_file:
+        console.print("[yellow]Provide URLs or use --from <file>.[/yellow]")
+        raise SystemExit(1)
+
+    out = Path(output_dir) if output_dir else DEFAULT_VIDEOS_DIR
+
+    results = download_all(
+        urls=list(urls),
+        from_file=from_file,
+        output_dir=out,
+        browser=browser,
+        verbose=verbose,
+        parallel=parallel,
+    )
+
+    # Exit with error code if any downloads failed
+    if any(not r.success for r in results):
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()
+@main.command()
+@click.argument("broadcast_url")
+@click.option("--browser", "-b", default=DEFAULT_BROWSER, help=f"Browser for cookies (default: {DEFAULT_BROWSER})")
+@click.option("--trim/--detect-only", default=False, help="Actually trim paused sections (default: detect only)")
+@click.option("--output", "-o", default=None, help="Output video for --trim (default: <video>.trimmed.mp4)")
+@click.option("--size-ratio", default=0.50, help="Size-drop threshold (default 0.50)")
+@click.option("--gap-density", default=0.50, help="PDT-gap density threshold (default 0.50)")
+@click.option("--min-pause", default=10.0, help="Minimum pause duration in seconds (default 10)")
+def trim_pauses(broadcast_url, browser, trim, output, size_ratio, gap_density, min_pause):
+    """Detect (and optionally trim) paused sections in a broadcast.
+
+    Analyses HLS segments via HTTP HEAD requests (no full download) and
+    playlist PDT timestamps to find sections where the video was paused
+    while audio continued.  Default: detect-only.  Pass --trim to cut.
+    """
+    if trim and not check_ffmpeg():
+        console.print("[red]ffmpeg not found - install with: brew install ffmpeg[/red]")
+        raise SystemExit(1)
+
+    console.print("[bold]Analysing HLS segments for pauses...[/bold]")
+
+    pauses = detect_pauses(
+        broadcast_url,
+        browser=browser,
+        size_ratio_threshold=size_ratio,
+        gap_density_threshold=gap_density,
+        min_pause_sec=min_pause,
+    )
+
+    console.print(pause_report(pauses))
+
+    if trim and pauses:
+        video_path = Path("output") / "videos" / f"{broadcast_url.split('/')[-1]}.mp4"
+        if not video_path.exists():
+            console.print(f"[red]Video not found: {video_path}")
+            console.print("  Download first: broadcastx download <url>")
+            raise SystemExit(1)
+
+        out = Path(output) if output else Path(str(video_path).replace(".mp4", ".trimmed.mp4"))
+        console.print(f"\n[bold]Trimming -> {out}...")
+        try:
+            trim_intervals(video_path, pauses, out)
+            console.print(f"  [green]Done -> {out}")
+        except Exception as e:
+            console.print(f"  [red]Failed: {e}")
+            raise SystemExit(1)
+    elif trim and not pauses:
+        console.print("[green]Nothing to trim.")

broadcastx/config.py

@@ -0,0 +1,67 @@
+"""Shared configuration and constants for BroadcastX."""
+
+import re
+from pathlib import Path
+
+# Default output directory (relative to cwd)
+DEFAULT_OUTPUT_DIR = Path("output")
+DEFAULT_VIDEOS_DIR = DEFAULT_OUTPUT_DIR / "videos"
+DEFAULT_BROADCASTS_FILE = DEFAULT_OUTPUT_DIR / "broadcasts.json"
+
+# Browser to extract cookies from (for yt-dlp)
+DEFAULT_BROWSER = "chrome"
+
+# X broadcast IDs observed in real broadcast URLs are opaque alphanumeric
+# tokens, e.g. 1vAxRkBbDRzKl. Reject tiny fragments such as /broadcasts/1.
+BROADCAST_ID_RE = r"[A-Za-z0-9]{8,}"
+
+# Broadcast URL patterns to match
+BROADCAST_PATTERNS = [
+    re.compile(rf"https?://(?:x|twitter)\.com/i/broadcasts/({BROADCAST_ID_RE})(?![A-Za-z0-9_])"),
+    re.compile(rf"https?://(?:www\.)?pscp\.tv/w/({BROADCAST_ID_RE})(?![A-Za-z0-9_])"),
+]
+
+# Twitter GraphQL endpoints to intercept
+GRAPHQL_ENDPOINTS = [
+    "UserTweets",
+    "UserTweetsAndReplies",
+    "TweetDetail",
+    "SearchTimeline",
+]
+
+# Scanner defaults
+DEFAULT_MAX_SCROLLS = 100       # Maximum number of scroll actions
+DEFAULT_SCROLL_DELAY = 2.0      # Seconds between scrolls
+DEFAULT_IDLE_TIMEOUT = 10.0     # Stop after N seconds with no new tweets
+DEFAULT_HEADLESS = False        # Show browser by default (useful for login)
+
+# yt-dlp output template — uses broadcast ID as filename
+YTDLP_OUTPUT_TEMPLATE = "%(id)s [%(timestamp>%Y-%m-%d %H.%M.%S)s] %(title)s.%(ext)s"
+
+
+def ensure_output_dirs():
+    """Create output directories if they don't exist."""
+    DEFAULT_OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
+    DEFAULT_VIDEOS_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def extract_broadcast_id(url: str) -> str | None:
+    """Extract broadcast ID from a broadcast URL."""
+    for pattern in BROADCAST_PATTERNS:
+        match = pattern.search(url)
+        if match:
+            return match.group(1)
+    return None
+
+
+def is_broadcast_url(url: str) -> bool:
+    """Check if a URL is a broadcast URL."""
+    return extract_broadcast_id(url) is not None
+
+
+def normalize_broadcast_url(url: str) -> str | None:
+    """Normalize a broadcast URL to the canonical x.com format."""
+    bid = extract_broadcast_id(url)
+    if bid:
+        return f"https://x.com/i/broadcasts/{bid}"
+    return None

broadcastx/downloader.py

@@ -0,0 +1,337 @@
+"""
+Downloader module — Download broadcast videos using yt-dlp.
+
+Usage:
+    from broadcastx.downloader import download_broadcast, download_all
+
+    # Single download
+    download_broadcast("https://x.com/i/broadcasts/1vAxRkBbDRzKl")
+
+    # Batch download from JSON file
+    download_all("broadcasts.json", output_dir="./videos")
+"""
+
+import json
+import shutil
+import subprocess
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass
+from pathlib import Path
+
+from rich.console import Console
+
+from .config import (
+    DEFAULT_BROWSER,
+    DEFAULT_VIDEOS_DIR,
+    YTDLP_OUTPUT_TEMPLATE,
+    extract_broadcast_id,
+    is_broadcast_url,
+    normalize_broadcast_url,
+)
+from .rotation import extract_rotation_sidecar
+from .rotation import rotate_video as _apply_rotation
+
+console = Console()
+
+
+@dataclass
+class DownloadResult:
+    """Result of a single broadcast download attempt."""
+    url: str
+    success: bool
+    output_file: str | None = None
+    rotation_sidecar_file: str | None = None
+    rotation_applied: bool = False
+    error: str | None = None
+    warning: str | None = None
+
+
+def check_yt_dlp() -> bool:
+    """Check if yt-dlp is installed and accessible."""
+    return shutil.which("yt-dlp") is not None
+
+
+def check_ffmpeg() -> bool:
+    """Check if ffmpeg is installed and accessible."""
+    return shutil.which("ffmpeg") is not None
+
+
+def download_broadcast(
+    url: str,
+    output_dir: Path = DEFAULT_VIDEOS_DIR,
+    browser: str = DEFAULT_BROWSER,
+    verbose: bool = False,
+) -> DownloadResult:
+    """
+    Download a single broadcast video using yt-dlp.
+
+    No timeout — broadcasts can be hours long. yt-dlp runs until completion
+    with live output streamed to the terminal.
+
+    Args:
+        url: Broadcast URL (x.com/i/broadcasts/... or pscp.tv/w/...)
+        output_dir: Directory to save the video
+        browser: Browser to extract cookies from
+        verbose: Show yt-dlp output
+       
+    Returns:
+        DownloadResult with success status and output file path
+    """
+    normalized = normalize_broadcast_url(url)
+    if not normalized:
+        return DownloadResult(
+            url=url,
+            success=False,
+            error=f"Not a valid broadcast URL: {url}",
+        )
+
+    broadcast_id = extract_broadcast_id(url)
+    output_dir = Path(output_dir)
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    output_template = str(output_dir / YTDLP_OUTPUT_TEMPLATE)
+
+    cmd = [
+        "yt-dlp",
+        "--cookies-from-browser", browser,
+        "-f", "bestvideo+bestaudio/best",
+        "--output", output_template,
+        "--merge-output-format", "mp4",
+        "--no-warnings",
+        "--newline",            # Progress on new lines
+        "--no-overwrites",      # Skip already downloaded
+        normalized,
+    ]
+
+    console.print(f"  [dim]Downloading {broadcast_id}...[/dim]")
+    if verbose:
+        console.print(f"  [dim]$ {' '.join(cmd)}[/dim]")
+
+    try:
+        # Stream output live — no timeout, broadcasts can be very long
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+        )
+
+        output_lines = []
+        output_file = None
+
+        # Start rotation extraction in parallel with yt-dlp download
+        # Both read from CDN independently; no need to wait.
+        rotation_result: list = []
+        def _run_sidecar():
+            try:
+                sc = extract_rotation_sidecar(normalized, output_dir, browser=browser)
+                rotation_result.append(("ok", str(sc)))
+            except Exception as e:
+                rotation_result.append(("error", e))
+
+        rotation_thread = threading.Thread(target=_run_sidecar, daemon=True)
+        rotation_thread.start()
+
+        for line in process.stdout:
+            line = line.rstrip()
+            output_lines.append(line)
+
+            # Always show progress/status lines
+            if verbose or "[download]" in line or "[Merger]" in line or "already" in line.lower():
+                console.print(f"    [dim]{line}[/dim]")
+
+            # Try to capture the output filename
+            if "Merging formats into" in line and '"' in line:
+                start = line.index('"') + 1
+                end = line.rindex('"')
+                output_file = line[start:end]
+            elif "Destination:" in line:
+                output_file = line.split("Destination:")[-1].strip()
+
+        process.wait()
+
+        if process.returncode == 0:
+            sidecar_file = None
+            warning = None
+            rotation_thread.join(timeout=300)
+            if rotation_result:
+                r = rotation_result[0]
+                kind, value = r
+                if kind == "error":
+                    warning = f"Rotation sidecar failed: {value}"
+                    console.print(f"  [yellow]{warning}[/yellow]")
+                else:
+                    sidecar_file = value
+                    console.print(f"  [green]Rotation sidecar[/green] → {sidecar_file}")
+            rotation_applied = False
+            if sidecar_file and output_file and Path(output_file).exists():
+                try:
+                    _apply_rotation(output_file, sidecar_file)
+                    # rotate_video returns the same path when it replaces in-place
+                    rotation_applied = True
+                    console.print(f"  [green]Rotation corrected[/green] → {output_file}")
+                except subprocess.CalledProcessError as e:
+                    err = (e.stderr.decode("utf-8", "replace") if isinstance(e.stderr, bytes) else (e.stderr or ""))
+                    w = f"Rotation correction failed (exit {e.returncode}): {err[:400]}"
+                    warning = f"{warning}; {w}" if warning else w
+                    console.print(f"  [yellow]{w}[/yellow]")
+                except Exception as e:
+                    w = f"Rotation correction failed: {e!r}"
+                    warning = f"{warning}; {w}" if warning else w
+                    console.print(f"  [yellow]{w}[/yellow]")
+            return DownloadResult(
+                url=normalized,
+                success=True,
+                output_file=output_file,
+                rotation_sidecar_file=sidecar_file,
+                rotation_applied=rotation_applied,
+                warning=warning,
+            )
+        else:
+            # Extract error from output
+            error_msg = "\n".join(output_lines[-3:]) or f"yt-dlp exited with code {process.returncode}"
+            return DownloadResult(
+                url=normalized,
+                success=False,
+                error=error_msg,
+            )
+    except FileNotFoundError:
+        return DownloadResult(
+            url=normalized,
+            success=False,
+            error="yt-dlp not found. Install it with: brew install yt-dlp",
+        )
+
+
+def download_all(
+    urls: list[str] | None = None,
+    from_file: Path | str | None = None,
+    output_dir: Path = DEFAULT_VIDEOS_DIR,
+    browser: str = DEFAULT_BROWSER,
+    verbose: bool = False,
+    parallel: int = 1,
+) -> list[DownloadResult]:
+    """
+    Download multiple broadcast videos.
+
+    Args:
+        urls: List of broadcast URLs
+        from_file: Path to JSON file containing broadcast data
+        output_dir: Directory to save videos
+        browser: Browser to extract cookies from
+        verbose: Show yt-dlp output
+        parallel: Number of concurrent downloads (default: 1 = sequential)
+       
+    Returns:
+        List of DownloadResult objects
+    """
+    all_urls = list(urls or [])
+
+    # Load URLs from file if provided
+    if from_file:
+        file_urls = _load_urls_from_file(Path(from_file))
+        all_urls.extend(file_urls)
+
+    if not all_urls:
+        console.print("[yellow]No broadcast URLs to download.[/yellow]")
+        return []
+
+    # Deduplicate while preserving order
+    seen = set()
+    unique_urls = []
+    for url in all_urls:
+        normalized = normalize_broadcast_url(url)
+        if normalized and normalized not in seen:
+            seen.add(normalized)
+            unique_urls.append(normalized)
+
+    total = len(unique_urls)
+    console.print(f"\n[bold]Downloading {total} broadcast(s)" + (f" ({parallel} parallel)" if parallel > 1 else "") + f"...[/bold]\n")
+
+    if parallel <= 1:
+        # Sequential download
+        results = []
+        for i, url in enumerate(unique_urls, 1):
+            console.print(f"[bold][{i}/{total}][/bold] {url}")
+            result = download_broadcast(url, output_dir=output_dir, browser=browser, verbose=verbose)
+            results.append(result)
+            if result.success:
+                console.print(f"  [green]✓ Done[/green]" + (f" → {result.output_file}" if result.output_file else ""))
+            else:
+                console.print(f"  [red]✗ Failed: {result.error}[/red]")
+            console.print()
+    else:
+        # Parallel download
+        results = [None] * total
+        lock = threading.Lock()
+        completed_count = 0
+
+        def _download_one(index: int, url: str) -> tuple[int, DownloadResult]:
+            return index, download_broadcast(url, output_dir=output_dir, browser=browser, verbose=verbose)
+
+        with ThreadPoolExecutor(max_workers=parallel) as executor:
+            futures = {
+                executor.submit(_download_one, i, url): (i, url)
+                for i, url in enumerate(unique_urls)
+            }
+
+            for future in as_completed(futures):
+                idx, result = future.result()
+                results[idx] = result
+                url = unique_urls[idx]
+                bid = extract_broadcast_id(url)
+
+                with lock:
+                    completed_count += 1
+                    if result.success:
+                        console.print(f"  [green]✓[/green] [{completed_count}/{total}] {bid}" + (f" → {result.output_file}" if result.output_file else ""))
+                    else:
+                        console.print(f"  [red]✗[/red] [{completed_count}/{total}] {bid}: {result.error}")
+
+        console.print()
+
+    # Summary
+    succeeded = sum(1 for r in results if r and r.success)
+    failed = sum(1 for r in results if r and not r.success)
+    console.print(f"[bold]Done:[/bold] {succeeded} succeeded, {failed} failed")
+
+    return results
+
+
+def _load_urls_from_file(path: Path) -> list[str]:
+    """
+    Load broadcast URLs from a JSON file.
+
+    Supports multiple formats:
+        1. Plain list of URLs:    ["https://x.com/i/broadcasts/abc", ...]
+        2. Scanner output:        {"broadcasts": [{"url": "..."}, ...]}
+        3. List of objects:       [{"url": "..."}, ...]
+    """
+    if not path.exists():
+        console.print(f"[red]File not found: {path}[/red]")
+        return []
+
+    try:
+        data = json.loads(path.read_text())
+    except json.JSONDecodeError as e:
+        console.print(f"[red]Invalid JSON in {path}: {e}[/red]")
+        return []
+
+    urls = []
+    if isinstance(data, list):
+        for item in data:
+            if isinstance(item, str):
+                urls.append(item)
+            elif isinstance(item, dict) and "url" in item:
+                urls.append(item["url"])
+    elif isinstance(data, dict) and "broadcasts" in data:
+        for item in data["broadcasts"]:
+            if isinstance(item, str):
+                urls.append(item)
+            elif isinstance(item, dict) and "url" in item:
+                urls.append(item["url"])
+
+    valid = [u for u in urls if is_broadcast_url(u)]
+    console.print(f"  [dim]Loaded {len(valid)} broadcast URL(s) from {path}[/dim]")
+    return valid