instagram-graphql-scraper 0.1.0__py3-none-any.whl

instagram_graphql_scraper/__init__.py

@@ -0,0 +1,6 @@
+"""Public package exports for the Instagram GraphQL scraper."""
+
+from .scraper import InstagramGraphqlScraper
+from .embed import InstagramEmbedClient, fetch_posts_embed
+
+__all__ = ["InstagramGraphqlScraper", "InstagramEmbedClient", "fetch_posts_embed"]

instagram_graphql_scraper/base/__init__.py

@@ -0,0 +1 @@
+"""Browser bootstrap package for the Instagram scraper."""

instagram_graphql_scraper/base/base.py

@@ -0,0 +1,46 @@
+"""Selenium Wire browser bootstrap helpers."""
+
+# -*- coding:utf-8 -*-
+from seleniumwire import webdriver
+from selenium.webdriver.chrome.service import Service
+from typing import Optional
+
+class BasePage:
+    """Create and configure the Selenium Wire Chrome driver."""
+
+    def __init__(self, driver_path: Optional[str] = None, open_browser: bool = False):
+        """Initialize a Chrome driver.
+
+        Args:
+            driver_path: Optional path to the ChromeDriver executable.
+            open_browser: Whether to show the browser instead of using headless mode.
+        """
+        chrome_options = self._build_options(open_browser)
+        service = Service(driver_path) if driver_path else Service()
+        self.driver = webdriver.Chrome(service=service, options=chrome_options)
+        if open_browser:
+            self.driver.maximize_window()
+        else:
+            self.driver.set_window_size(1920, 1080)
+
+    @staticmethod
+    def _build_options(open_browser: bool) -> webdriver.ChromeOptions:
+        """Build Chrome options used by the scraper browser.
+
+        Args:
+            open_browser: Whether headless mode should be disabled.
+
+        Returns:
+            Configured Selenium Chrome options.
+        """
+        options = webdriver.ChromeOptions()
+        options.add_argument("--disable-blink-features")
+        options.add_argument("--disable-notifications")
+        options.add_argument("--disable-blink-features=AutomationControlled")
+        options.add_argument("--window-size=1920,1080")
+        if not open_browser:
+            options.add_argument("--headless=new")
+        options.add_argument("--blink-settings=imagesEnabled=false")
+        options.add_experimental_option("excludeSwitches", ["enable-automation"])
+        options.add_experimental_option("useAutomationExtension", False)
+        return options

instagram_graphql_scraper/detail.py

@@ -0,0 +1,178 @@
+"""Instagram post HTML detail parsing and fail-soft merge helpers."""
+
+import re
+from datetime import datetime, timezone
+from html.parser import HTMLParser
+from typing import Any
+
+
+class PostDetailParser(HTMLParser):
+    """Collect metadata, timestamps, and exact count anchors from post HTML."""
+
+    def __init__(self):
+        """Initialize parser state for one Instagram detail response."""
+        super().__init__()
+        self.meta: dict[str, str] = {}
+        self.times: list[str] = []
+        self._count_anchor: str | None = None
+        self._count_text: list[str] = []
+        self.exact_counts: dict[str, int] = {}
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        """Capture relevant metadata from an opening HTML tag.
+
+        Args:
+            tag: HTML tag name.
+            attrs: Parsed HTML attributes.
+        """
+        attributes = dict(attrs)
+        if tag == "meta":
+            key = attributes.get("name") or attributes.get("property")
+            content = attributes.get("content")
+            if key and content and key in {"description", "og:description"}:
+                self.meta[key] = content
+        elif tag == "time" and attributes.get("datetime"):
+            datetime_value = attributes.get("datetime")
+            if datetime_value:
+                self.times.append(datetime_value)
+        elif tag == "a":
+            event = attributes.get("data-log-event")
+            if event in {"likeCountClick", "captionCommentsClick"}:
+                self._count_anchor = event
+                self._count_text = []
+
+    def handle_data(self, data: str) -> None:
+        """Collect text belonging to an exact count anchor."""
+        if self._count_anchor:
+            self._count_text.append(data)
+
+    def handle_endtag(self, tag: str) -> None:
+        """Finalize an exact count when its anchor closes."""
+        if tag == "a" and self._count_anchor:
+            text = "".join(self._count_text)
+            match = re.search(r"([\d,]+)\s+(?:likes|comments)", text, re.IGNORECASE)
+            if match:
+                count = int(match.group(1).replace(",", ""))
+                field = "like_count" if self._count_anchor == "likeCountClick" else "comment_count"
+                self.exact_counts[field] = count
+            self._count_anchor = None
+            self._count_text = []
+
+
+def _parse_count(description: str, label: str) -> tuple[int | None, bool]:
+    """Parse a displayed count and report whether it uses a rounded suffix.
+
+    Args:
+        description: Post description metadata text.
+        label: Metric label such as ``likes`` or ``comments``.
+
+    Returns:
+        A tuple containing the parsed count and an approximation flag.
+    """
+    match = re.search(rf"([\d,.]+)\s*([KMB])?\s+{label}\b", description, re.IGNORECASE)
+    if not match:
+        return None, False
+    number = float(match.group(1).replace(",", ""))
+    suffix = (match.group(2) or "").upper()
+    multiplier = {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}[suffix]
+    return int(number * multiplier), bool(suffix)
+
+
+def _parse_embedded_count(html: str, field_name: str) -> int | None:
+    """Extract an exact count from embedded Instagram structured data."""
+    pattern = rf'\\?"{field_name}\\?"\s*:\s*\\?\{{\\?"count\\?"\s*:\s*(\d+)'
+    match = re.search(pattern, html)
+    return int(match.group(1)) if match else None
+
+
+def _parse_datetime(value: str | None) -> tuple[int | None, str | None]:
+    """Convert an ISO timestamp to UTC Unix and ISO-8601 values."""
+    if not value:
+        return None, None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None, None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    parsed = parsed.astimezone(timezone.utc)
+    return int(parsed.timestamp()), parsed.isoformat()
+
+
+def _parse_description_date(description: str) -> str | None:
+    """Extract a date-only fallback from an English description."""
+    match = re.search(r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s+(\d{4})\b", description)
+    if not match:
+        return None
+    try:
+        return datetime.strptime(" ".join(match.groups()), "%B %d %Y").date().isoformat()
+    except ValueError:
+        return None
+
+
+def parse_post_detail_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
+    """Normalize metrics and publication metadata from post HTML.
+
+    Args:
+        html: HTML returned by an Instagram post or Embed page.
+        shortcode: Optional shortcode associated with the response.
+
+    Returns:
+        Normalized detail fields with ``None`` for unavailable values.
+    """
+    parser = PostDetailParser()
+    parser.feed(html)
+    description = parser.meta.get("description") or parser.meta.get("og:description") or ""
+    like_count, likes_approximate = _parse_count(description, "likes")
+    comment_count, comments_approximate = _parse_count(description, "comments")
+    if "like_count" in parser.exact_counts:
+        like_count, likes_approximate = parser.exact_counts["like_count"], False
+    if "comment_count" in parser.exact_counts:
+        comment_count, comments_approximate = parser.exact_counts["comment_count"], False
+    embedded_like_count = _parse_embedded_count(html, "edge_liked_by")
+    embedded_comment_count = _parse_embedded_count(html, "edge_media_to_comment")
+    if embedded_like_count is not None:
+        like_count, likes_approximate = embedded_like_count, False
+    if embedded_comment_count is not None:
+        comment_count, comments_approximate = embedded_comment_count, False
+    video_view_count, _ = _parse_count(description, "views")
+    if video_view_count is None:
+        video_view_count, _ = _parse_count(description, "plays")
+    timestamp, published_at = _parse_datetime(parser.times[0] if parser.times else None)
+    if published_at is None:
+        published_at = _parse_description_date(description)
+    return {
+        "shortcode": shortcode,
+        "like_count": like_count,
+        "comment_count": comment_count,
+        "video_view_count": video_view_count,
+        "taken_at_timestamp": timestamp,
+        "published_at": published_at,
+        "like_count_is_approximate": likes_approximate,
+        "comment_count_is_approximate": comments_approximate,
+        "detail_source": "post_embed_html" if parser.exact_counts or embedded_like_count is not None or embedded_comment_count is not None else ("post_html_metadata" if description or parser.times else None),
+        "detail_shortcode": shortcode,
+    }
+
+
+def merge_post_detail(post: dict[str, Any], detail: dict[str, Any]) -> dict[str, Any]:
+    """Merge missing detail fields without replacing existing post values.
+
+    Args:
+        post: Existing normalized timeline post.
+        detail: Normalized Embed or detail response.
+
+    Returns:
+        The updated post dictionary.
+    """
+    for field in (
+        "like_count", "comment_count", "video_view_count", "video_duration",
+        "video_url", "display_uri", "product_type", "media_type", "is_video",
+        "taken_at_timestamp", "published_at",
+    ):
+        if post.get(field) is None and detail.get(field) is not None:
+            post[field] = detail[field]
+    for field in ("like_count_is_approximate", "comment_count_is_approximate", "detail_source"):
+        if detail.get(field) is not None:
+            post[field] = detail[field]
+    return post

instagram_graphql_scraper/embed.py

@@ -0,0 +1,168 @@
+"""Asynchronous Instagram Embed fetching and normalization utilities."""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+import httpx
+
+try:
+    from .detail import parse_post_detail_html
+except ImportError:
+    from detail import parse_post_detail_html
+
+try:
+    from .instagram_context_json import extract_context_json, get_media, normalize_media
+except ImportError:
+    from instagram_context_json import extract_context_json, get_media, normalize_media
+
+
+class InstagramEmbedClient:
+    """Fetch and normalize Instagram Embed posts with bounded concurrency."""
+
+    def __init__(
+        self,
+        max_concurrency: int = 10,
+        timeout: float = 30,
+        max_retries: int = 2,
+        client: httpx.AsyncClient | None = None,
+    ):
+        """Configure a reusable async Embed client.
+
+        Args:
+            max_concurrency: Maximum number of requests running concurrently.
+            timeout: Per-request timeout in seconds.
+            max_retries: Number of retries for transient failures.
+            client: Optional externally managed ``httpx.AsyncClient``.
+        """
+        if max_concurrency < 1:
+            raise ValueError("max_concurrency must be at least 1")
+        self.max_concurrency = max_concurrency
+        self.timeout = timeout
+        self.max_retries = max_retries
+        self._client = client
+        self._owns_client = client is None
+        self._semaphore = asyncio.Semaphore(max_concurrency)
+        self._tasks: dict[str, asyncio.Task[dict[str, Any]]] = {}
+
+    async def __aenter__(self) -> "InstagramEmbedClient":
+        """Create the shared HTTP client when entering the context."""
+        await self._ensure_client()
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> None:
+        """Close the internally owned HTTP client on context exit."""
+        await self.aclose()
+
+    async def _ensure_client(self) -> httpx.AsyncClient:
+        """Return the shared async HTTP client, creating it if needed."""
+        if self._client is None:
+            self._client = httpx.AsyncClient(
+                timeout=self.timeout,
+                limits=httpx.Limits(
+                    max_connections=self.max_concurrency,
+                    max_keepalive_connections=self.max_concurrency,
+                ),
+                headers={"User-Agent": "Mozilla/5.0"},
+                follow_redirects=True,
+            )
+        return self._client
+
+    @staticmethod
+    def embed_url(shortcode: str, media_type: Any = None) -> str:
+        """Build the public Embed URL for a post or Reel."""
+        path = "reel" if media_type == 2 or media_type == "video" else "p"
+        return f"https://www.instagram.com/{path}/{shortcode}/embed/captioned/"
+
+    async def fetch_post(self, shortcode: str, media_type: Any = None) -> dict[str, Any]:
+        """Fetch one post, reusing an in-flight task for duplicate shortcodes."""
+        if not shortcode:
+            return {"shortcode": shortcode, "error": "missing shortcode"}
+        if shortcode in self._tasks:
+            return await self._tasks[shortcode]
+        task = asyncio.create_task(self._fetch_post_once(shortcode, media_type))
+        self._tasks[shortcode] = task
+        return await task
+
+    async def _fetch_post_once(self, shortcode: str, media_type: Any) -> dict[str, Any]:
+        """Fetch, parse, and normalize one Embed response with retry handling."""
+        client = await self._ensure_client()
+        url = self.embed_url(shortcode, media_type)
+        last_error: Exception | None = None
+        for attempt in range(self.max_retries + 1):
+            try:
+                async with self._semaphore:
+                    response = await client.get(url)
+                if response.status_code == 404:
+                    return {"shortcode": shortcode, "error": "HTTP 404"}
+                if response.status_code in {401, 403}:
+                    return {"shortcode": shortcode, "error": f"HTTP {response.status_code}"}
+                if response.status_code == 429 or response.status_code >= 500:
+                    response.raise_for_status()
+                response.raise_for_status()
+                return normalize_embed_html(response.text, shortcode)
+            except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as error:
+                last_error = error
+                if attempt < self.max_retries:
+                    await asyncio.sleep(2**attempt)
+            except httpx.HTTPStatusError as error:
+                status_code = error.response.status_code
+                if status_code in {429, 500, 502, 503, 504} and attempt < self.max_retries:
+                    await asyncio.sleep(2**attempt)
+                    continue
+                return {"shortcode": shortcode, "error": f"HTTP {status_code}"}
+            except (ValueError, KeyError) as error:
+                return {"shortcode": shortcode, "error": str(error)}
+        return {"shortcode": shortcode, "error": str(last_error or "request failed")}
+
+    async def fetch_many(self, posts: Iterable[Mapping[str, Any] | str]) -> list[dict[str, Any]]:
+        """Fetch many posts while preserving input order and failure results."""
+        items = list(posts)
+        tasks = []
+        for item in items:
+            if isinstance(item, str):
+                tasks.append(self.fetch_post(item))
+            else:
+                tasks.append(self.fetch_post(str(item.get("shortcode") or ""), item.get("media_type")))
+        return await asyncio.gather(*tasks)
+
+    async def aclose(self) -> None:
+        """Close the internally owned HTTP client."""
+        if self._owns_client and self._client is not None:
+            await self._client.aclose()
+            self._client = None
+
+
+async def fetch_posts_embed(
+    posts: Iterable[Mapping[str, Any] | str],
+    max_concurrency: int = 10,
+    timeout: float = 30,
+    max_retries: int = 2,
+) -> list[dict[str, Any]]:
+    """Fetch a batch of Embed posts using a temporary bounded client.
+
+    Args:
+        posts: Shortcodes or timeline post dictionaries.
+        max_concurrency: Maximum number of concurrent requests.
+        timeout: Per-request timeout in seconds.
+        max_retries: Number of transient retries.
+
+    Returns:
+        Normalized detail results in input order.
+    """
+    async with InstagramEmbedClient(max_concurrency, timeout, max_retries) as client:
+        return await client.fetch_many(posts)
+
+
+def normalize_embed_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
+    """Normalize context JSON, falling back to exact HTML metrics when needed."""
+    try:
+        context = extract_context_json(html)
+        return normalize_media(get_media(context))
+    except (ValueError, KeyError) as error:
+        fallback = parse_post_detail_html(html, shortcode)
+        if fallback.get("detail_source"):
+            return fallback
+        raise ValueError(str(error)) from error

instagram_graphql_scraper/example.py

@@ -0,0 +1,31 @@
+"""Basic Instagram scraper usage examples."""
+
+# -*- coding: utf-8 -*-
+if __package__:
+    from . import InstagramGraphqlScraper as ig_graphql_scraper
+else:
+    from scraper import InstagramGraphqlScraper as ig_graphql_scraper
+
+
+## Example.1 - without logging in
+if __name__ == "__main__":
+    instagram_user_name = "love.yuweishao"
+    instagram_user_id = "100044253168423"
+    days_limit = 100 # Number of days within which to scrape posts
+    driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver" 
+    ig_spider = ig_graphql_scraper(driver_path=driver_path, open_browser=False)
+    res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_id, days_limit=days_limit, display_progress=True)
+
+
+## Example.2 - login in your Instagram account to collect data
+# if __name__ == "__main__":
+    # instagram_user_name = "love.yuweishao"
+    # instagram_user_id = "100044253168423"
+    # ig_account = "instagram_account"
+    # ig_pwd = "instagram_password"
+    # days_limit = 30 # Number of days within which to scrape posts
+    # driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver" 
+    # ig_spider = ig_graphql_scraper(ig_account=ig_account, ig_pwd=ig_pwd, driver_path=driver_path, open_browser=False)
+    # res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_name, days_limit=days_limit, display_progress=True)
+    # print(res)
+