instagram-graphql-scraper 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. instagram_graphql_scraper-0.1.0/LICENSE +21 -0
  2. instagram_graphql_scraper-0.1.0/PKG-INFO +93 -0
  3. instagram_graphql_scraper-0.1.0/README.md +70 -0
  4. instagram_graphql_scraper-0.1.0/__init__.py +6 -0
  5. instagram_graphql_scraper-0.1.0/base/__init__.py +1 -0
  6. instagram_graphql_scraper-0.1.0/base/base.py +46 -0
  7. instagram_graphql_scraper-0.1.0/detail.py +178 -0
  8. instagram_graphql_scraper-0.1.0/embed.py +168 -0
  9. instagram_graphql_scraper-0.1.0/example.py +31 -0
  10. instagram_graphql_scraper-0.1.0/graphql.py +324 -0
  11. instagram_graphql_scraper-0.1.0/instagram_context_json.py +149 -0
  12. instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/PKG-INFO +93 -0
  13. instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/SOURCES.txt +23 -0
  14. instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/dependency_links.txt +1 -0
  15. instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/requires.txt +5 -0
  16. instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/top_level.txt +1 -0
  17. instagram_graphql_scraper-0.1.0/manual_integration.py +20 -0
  18. instagram_graphql_scraper-0.1.0/models.py +21 -0
  19. instagram_graphql_scraper-0.1.0/pages/__init__.py +1 -0
  20. instagram_graphql_scraper-0.1.0/pages/page_optional.py +151 -0
  21. instagram_graphql_scraper-0.1.0/scraper.py +339 -0
  22. instagram_graphql_scraper-0.1.0/setup.cfg +4 -0
  23. instagram_graphql_scraper-0.1.0/setup.py +31 -0
  24. instagram_graphql_scraper-0.1.0/tests/test_graphql.py +404 -0
  25. instagram_graphql_scraper-0.1.0/utils/locator.py +40 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 faustren
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,93 @@
1
+ Metadata-Version: 2.4
2
+ Name: instagram-graphql-scraper
3
+ Version: 0.1.0
4
+ Summary: Collects posts from a public Instagram profile via captured GraphQL requests.
5
+ Author: faustren
6
+ License: MIT
7
+ Requires-Python: >=3.10
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Requires-Dist: selenium>=4.20
11
+ Requires-Dist: selenium-wire>=5.1
12
+ Requires-Dist: requests>=2.31
13
+ Requires-Dist: brotli>=1.1
14
+ Requires-Dist: httpx>=0.27
15
+ Dynamic: author
16
+ Dynamic: description
17
+ Dynamic: description-content-type
18
+ Dynamic: license
19
+ Dynamic: license-file
20
+ Dynamic: requires-dist
21
+ Dynamic: requires-python
22
+ Dynamic: summary
23
+
24
+ # instagram-graphql-scraper
25
+
26
+ Collects posts from a public Instagram profile by capturing the browser's profile GraphQL request, then replaying subsequent pages through one `requests.Session`.
27
+
28
+ ## Installation
29
+
30
+ ```bash
31
+ pip install -r requirements.txt
32
+ # or
33
+ pip install -e .
34
+ ```
35
+
36
+ ## Usage
37
+
38
+ ```python
39
+ from instagram_graphql_scraper import InstagramGraphqlScraper
40
+
41
+ scraper = InstagramGraphqlScraper(driver_path="/path/to/chromedriver")
42
+ try:
43
+ posts = scraper.get_user_posts("username", max_pages=2, max_posts=24)
44
+ finally:
45
+ scraper.close()
46
+ ```
47
+
48
+ The browser flow opens the profile, dismisses an optional login dialog, scrolls once, clears Selenium Wire requests, and clicks the dynamic show-more-posts button once. The captured request supplies the URL, headers, cookies, form payload, variables, and `doc_id`; no dynamic GraphQL values are hardcoded.
49
+
50
+ `days_limit` keeps posts newer than N days and stops pagination early once older posts are reached, using each post's `accessibility_caption` date; posts without a parseable date are kept. Combine with `max_pages` or `max_posts` for additional bounds.
51
+
52
+ For bounded asynchronous Embed enrichment (opt-in, not run automatically by `get_user_posts`):
53
+
54
+ ```python
55
+ import asyncio
56
+ from instagram_graphql_scraper import InstagramEmbedClient
57
+
58
+ async def enrich(posts):
59
+ async with InstagramEmbedClient(max_concurrency=10) as client:
60
+ return await client.fetch_many(posts)
61
+
62
+ details = asyncio.run(enrich(posts))
63
+ ```
64
+
65
+ The batch client preserves input order, deduplicates shortcodes per run, and returns a per-post error result instead of failing the whole batch. The single-post normalized CLI remains available:
66
+
67
+ ```bash
68
+ python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/"
69
+ python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/" --full
70
+ ```
71
+
72
+ ## Tests
73
+
74
+ ```bash
75
+ python3 -m pytest -q
76
+ ```
77
+
78
+ The manual browser check is intentionally not part of CI:
79
+
80
+ ```bash
81
+ python3 manual_integration.py 1989ivyshao --driver-path /path/to/chromedriver --max-pages 2
82
+ ```
83
+
84
+ ## Documentation
85
+
86
+ See [docs/](docs/) for the system architecture diagram, class diagram, and a bilingual ([繁體中文](docs/sre_testing_runbook.md) / [English](docs/sre_testing_runbook.en.md)) SRE testing runbook covering environment setup, parameter boundaries, and known issues.
87
+
88
+
89
+ Do not place cookies, CSRF/LSD tokens, or captured payloads in source, fixtures, or logs.
90
+
91
+ ## License
92
+
93
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,70 @@
1
+ # instagram-graphql-scraper
2
+
3
+ Collects posts from a public Instagram profile by capturing the browser's profile GraphQL request, then replaying subsequent pages through one `requests.Session`.
4
+
5
+ ## Installation
6
+
7
+ ```bash
8
+ pip install -r requirements.txt
9
+ # or
10
+ pip install -e .
11
+ ```
12
+
13
+ ## Usage
14
+
15
+ ```python
16
+ from instagram_graphql_scraper import InstagramGraphqlScraper
17
+
18
+ scraper = InstagramGraphqlScraper(driver_path="/path/to/chromedriver")
19
+ try:
20
+ posts = scraper.get_user_posts("username", max_pages=2, max_posts=24)
21
+ finally:
22
+ scraper.close()
23
+ ```
24
+
25
+ The browser flow opens the profile, dismisses an optional login dialog, scrolls once, clears Selenium Wire requests, and clicks the dynamic show-more-posts button once. The captured request supplies the URL, headers, cookies, form payload, variables, and `doc_id`; no dynamic GraphQL values are hardcoded.
26
+
27
+ `days_limit` keeps posts newer than N days and stops pagination early once older posts are reached, using each post's `accessibility_caption` date; posts without a parseable date are kept. Combine with `max_pages` or `max_posts` for additional bounds.
28
+
29
+ For bounded asynchronous Embed enrichment (opt-in, not run automatically by `get_user_posts`):
30
+
31
+ ```python
32
+ import asyncio
33
+ from instagram_graphql_scraper import InstagramEmbedClient
34
+
35
+ async def enrich(posts):
36
+ async with InstagramEmbedClient(max_concurrency=10) as client:
37
+ return await client.fetch_many(posts)
38
+
39
+ details = asyncio.run(enrich(posts))
40
+ ```
41
+
42
+ The batch client preserves input order, deduplicates shortcodes per run, and returns a per-post error result instead of failing the whole batch. The single-post normalized CLI remains available:
43
+
44
+ ```bash
45
+ python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/"
46
+ python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/" --full
47
+ ```
48
+
49
+ ## Tests
50
+
51
+ ```bash
52
+ python3 -m pytest -q
53
+ ```
54
+
55
+ The manual browser check is intentionally not part of CI:
56
+
57
+ ```bash
58
+ python3 manual_integration.py 1989ivyshao --driver-path /path/to/chromedriver --max-pages 2
59
+ ```
60
+
61
+ ## Documentation
62
+
63
+ See [docs/](docs/) for the system architecture diagram, class diagram, and a bilingual ([繁體中文](docs/sre_testing_runbook.md) / [English](docs/sre_testing_runbook.en.md)) SRE testing runbook covering environment setup, parameter boundaries, and known issues.
64
+
65
+
66
+ Do not place cookies, CSRF/LSD tokens, or captured payloads in source, fixtures, or logs.
67
+
68
+ ## License
69
+
70
+ MIT, see [LICENSE](LICENSE).
@@ -0,0 +1,6 @@
1
+ """Public package exports for the Instagram GraphQL scraper."""
2
+
3
+ from .scraper import InstagramGraphqlScraper
4
+ from .embed import InstagramEmbedClient, fetch_posts_embed
5
+
6
+ __all__ = ["InstagramGraphqlScraper", "InstagramEmbedClient", "fetch_posts_embed"]
@@ -0,0 +1 @@
1
+ """Browser bootstrap package for the Instagram scraper."""
@@ -0,0 +1,46 @@
1
+ """Selenium Wire browser bootstrap helpers."""
2
+
3
+ # -*- coding:utf-8 -*-
4
+ from seleniumwire import webdriver
5
+ from selenium.webdriver.chrome.service import Service
6
+ from typing import Optional
7
+
8
+ class BasePage:
9
+ """Create and configure the Selenium Wire Chrome driver."""
10
+
11
+ def __init__(self, driver_path: Optional[str] = None, open_browser: bool = False):
12
+ """Initialize a Chrome driver.
13
+
14
+ Args:
15
+ driver_path: Optional path to the ChromeDriver executable.
16
+ open_browser: Whether to show the browser instead of using headless mode.
17
+ """
18
+ chrome_options = self._build_options(open_browser)
19
+ service = Service(driver_path) if driver_path else Service()
20
+ self.driver = webdriver.Chrome(service=service, options=chrome_options)
21
+ if open_browser:
22
+ self.driver.maximize_window()
23
+ else:
24
+ self.driver.set_window_size(1920, 1080)
25
+
26
+ @staticmethod
27
+ def _build_options(open_browser: bool) -> webdriver.ChromeOptions:
28
+ """Build Chrome options used by the scraper browser.
29
+
30
+ Args:
31
+ open_browser: Whether headless mode should be disabled.
32
+
33
+ Returns:
34
+ Configured Selenium Chrome options.
35
+ """
36
+ options = webdriver.ChromeOptions()
37
+ options.add_argument("--disable-blink-features")
38
+ options.add_argument("--disable-notifications")
39
+ options.add_argument("--disable-blink-features=AutomationControlled")
40
+ options.add_argument("--window-size=1920,1080")
41
+ if not open_browser:
42
+ options.add_argument("--headless=new")
43
+ options.add_argument("--blink-settings=imagesEnabled=false")
44
+ options.add_experimental_option("excludeSwitches", ["enable-automation"])
45
+ options.add_experimental_option("useAutomationExtension", False)
46
+ return options
@@ -0,0 +1,178 @@
1
+ """Instagram post HTML detail parsing and fail-soft merge helpers."""
2
+
3
+ import re
4
+ from datetime import datetime, timezone
5
+ from html.parser import HTMLParser
6
+ from typing import Any
7
+
8
+
9
+ class PostDetailParser(HTMLParser):
10
+ """Collect metadata, timestamps, and exact count anchors from post HTML."""
11
+
12
+ def __init__(self):
13
+ """Initialize parser state for one Instagram detail response."""
14
+ super().__init__()
15
+ self.meta: dict[str, str] = {}
16
+ self.times: list[str] = []
17
+ self._count_anchor: str | None = None
18
+ self._count_text: list[str] = []
19
+ self.exact_counts: dict[str, int] = {}
20
+
21
+ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
22
+ """Capture relevant metadata from an opening HTML tag.
23
+
24
+ Args:
25
+ tag: HTML tag name.
26
+ attrs: Parsed HTML attributes.
27
+ """
28
+ attributes = dict(attrs)
29
+ if tag == "meta":
30
+ key = attributes.get("name") or attributes.get("property")
31
+ content = attributes.get("content")
32
+ if key and content and key in {"description", "og:description"}:
33
+ self.meta[key] = content
34
+ elif tag == "time" and attributes.get("datetime"):
35
+ datetime_value = attributes.get("datetime")
36
+ if datetime_value:
37
+ self.times.append(datetime_value)
38
+ elif tag == "a":
39
+ event = attributes.get("data-log-event")
40
+ if event in {"likeCountClick", "captionCommentsClick"}:
41
+ self._count_anchor = event
42
+ self._count_text = []
43
+
44
+ def handle_data(self, data: str) -> None:
45
+ """Collect text belonging to an exact count anchor."""
46
+ if self._count_anchor:
47
+ self._count_text.append(data)
48
+
49
+ def handle_endtag(self, tag: str) -> None:
50
+ """Finalize an exact count when its anchor closes."""
51
+ if tag == "a" and self._count_anchor:
52
+ text = "".join(self._count_text)
53
+ match = re.search(r"([\d,]+)\s+(?:likes|comments)", text, re.IGNORECASE)
54
+ if match:
55
+ count = int(match.group(1).replace(",", ""))
56
+ field = "like_count" if self._count_anchor == "likeCountClick" else "comment_count"
57
+ self.exact_counts[field] = count
58
+ self._count_anchor = None
59
+ self._count_text = []
60
+
61
+
62
+ def _parse_count(description: str, label: str) -> tuple[int | None, bool]:
63
+ """Parse a displayed count and report whether it uses a rounded suffix.
64
+
65
+ Args:
66
+ description: Post description metadata text.
67
+ label: Metric label such as ``likes`` or ``comments``.
68
+
69
+ Returns:
70
+ A tuple containing the parsed count and an approximation flag.
71
+ """
72
+ match = re.search(rf"([\d,.]+)\s*([KMB])?\s+{label}\b", description, re.IGNORECASE)
73
+ if not match:
74
+ return None, False
75
+ number = float(match.group(1).replace(",", ""))
76
+ suffix = (match.group(2) or "").upper()
77
+ multiplier = {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}[suffix]
78
+ return int(number * multiplier), bool(suffix)
79
+
80
+
81
+ def _parse_embedded_count(html: str, field_name: str) -> int | None:
82
+ """Extract an exact count from embedded Instagram structured data."""
83
+ pattern = rf'\\?"{field_name}\\?"\s*:\s*\\?\{{\\?"count\\?"\s*:\s*(\d+)'
84
+ match = re.search(pattern, html)
85
+ return int(match.group(1)) if match else None
86
+
87
+
88
+ def _parse_datetime(value: str | None) -> tuple[int | None, str | None]:
89
+ """Convert an ISO timestamp to UTC Unix and ISO-8601 values."""
90
+ if not value:
91
+ return None, None
92
+ try:
93
+ parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
94
+ except ValueError:
95
+ return None, None
96
+ if parsed.tzinfo is None:
97
+ parsed = parsed.replace(tzinfo=timezone.utc)
98
+ parsed = parsed.astimezone(timezone.utc)
99
+ return int(parsed.timestamp()), parsed.isoformat()
100
+
101
+
102
+ def _parse_description_date(description: str) -> str | None:
103
+ """Extract a date-only fallback from an English description."""
104
+ match = re.search(r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s+(\d{4})\b", description)
105
+ if not match:
106
+ return None
107
+ try:
108
+ return datetime.strptime(" ".join(match.groups()), "%B %d %Y").date().isoformat()
109
+ except ValueError:
110
+ return None
111
+
112
+
113
+ def parse_post_detail_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
114
+ """Normalize metrics and publication metadata from post HTML.
115
+
116
+ Args:
117
+ html: HTML returned by an Instagram post or Embed page.
118
+ shortcode: Optional shortcode associated with the response.
119
+
120
+ Returns:
121
+ Normalized detail fields with ``None`` for unavailable values.
122
+ """
123
+ parser = PostDetailParser()
124
+ parser.feed(html)
125
+ description = parser.meta.get("description") or parser.meta.get("og:description") or ""
126
+ like_count, likes_approximate = _parse_count(description, "likes")
127
+ comment_count, comments_approximate = _parse_count(description, "comments")
128
+ if "like_count" in parser.exact_counts:
129
+ like_count, likes_approximate = parser.exact_counts["like_count"], False
130
+ if "comment_count" in parser.exact_counts:
131
+ comment_count, comments_approximate = parser.exact_counts["comment_count"], False
132
+ embedded_like_count = _parse_embedded_count(html, "edge_liked_by")
133
+ embedded_comment_count = _parse_embedded_count(html, "edge_media_to_comment")
134
+ if embedded_like_count is not None:
135
+ like_count, likes_approximate = embedded_like_count, False
136
+ if embedded_comment_count is not None:
137
+ comment_count, comments_approximate = embedded_comment_count, False
138
+ video_view_count, _ = _parse_count(description, "views")
139
+ if video_view_count is None:
140
+ video_view_count, _ = _parse_count(description, "plays")
141
+ timestamp, published_at = _parse_datetime(parser.times[0] if parser.times else None)
142
+ if published_at is None:
143
+ published_at = _parse_description_date(description)
144
+ return {
145
+ "shortcode": shortcode,
146
+ "like_count": like_count,
147
+ "comment_count": comment_count,
148
+ "video_view_count": video_view_count,
149
+ "taken_at_timestamp": timestamp,
150
+ "published_at": published_at,
151
+ "like_count_is_approximate": likes_approximate,
152
+ "comment_count_is_approximate": comments_approximate,
153
+ "detail_source": "post_embed_html" if parser.exact_counts or embedded_like_count is not None or embedded_comment_count is not None else ("post_html_metadata" if description or parser.times else None),
154
+ "detail_shortcode": shortcode,
155
+ }
156
+
157
+
158
+ def merge_post_detail(post: dict[str, Any], detail: dict[str, Any]) -> dict[str, Any]:
159
+ """Merge missing detail fields without replacing existing post values.
160
+
161
+ Args:
162
+ post: Existing normalized timeline post.
163
+ detail: Normalized Embed or detail response.
164
+
165
+ Returns:
166
+ The updated post dictionary.
167
+ """
168
+ for field in (
169
+ "like_count", "comment_count", "video_view_count", "video_duration",
170
+ "video_url", "display_uri", "product_type", "media_type", "is_video",
171
+ "taken_at_timestamp", "published_at",
172
+ ):
173
+ if post.get(field) is None and detail.get(field) is not None:
174
+ post[field] = detail[field]
175
+ for field in ("like_count_is_approximate", "comment_count_is_approximate", "detail_source"):
176
+ if detail.get(field) is not None:
177
+ post[field] = detail[field]
178
+ return post
@@ -0,0 +1,168 @@
1
+ """Asynchronous Instagram Embed fetching and normalization utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from collections.abc import Iterable, Mapping
7
+ from typing import Any
8
+
9
+ import httpx
10
+
11
+ try:
12
+ from .detail import parse_post_detail_html
13
+ except ImportError:
14
+ from detail import parse_post_detail_html
15
+
16
+ try:
17
+ from .instagram_context_json import extract_context_json, get_media, normalize_media
18
+ except ImportError:
19
+ from instagram_context_json import extract_context_json, get_media, normalize_media
20
+
21
+
22
+ class InstagramEmbedClient:
23
+ """Fetch and normalize Instagram Embed posts with bounded concurrency."""
24
+
25
+ def __init__(
26
+ self,
27
+ max_concurrency: int = 10,
28
+ timeout: float = 30,
29
+ max_retries: int = 2,
30
+ client: httpx.AsyncClient | None = None,
31
+ ):
32
+ """Configure a reusable async Embed client.
33
+
34
+ Args:
35
+ max_concurrency: Maximum number of requests running concurrently.
36
+ timeout: Per-request timeout in seconds.
37
+ max_retries: Number of retries for transient failures.
38
+ client: Optional externally managed ``httpx.AsyncClient``.
39
+ """
40
+ if max_concurrency < 1:
41
+ raise ValueError("max_concurrency must be at least 1")
42
+ self.max_concurrency = max_concurrency
43
+ self.timeout = timeout
44
+ self.max_retries = max_retries
45
+ self._client = client
46
+ self._owns_client = client is None
47
+ self._semaphore = asyncio.Semaphore(max_concurrency)
48
+ self._tasks: dict[str, asyncio.Task[dict[str, Any]]] = {}
49
+
50
+ async def __aenter__(self) -> "InstagramEmbedClient":
51
+ """Create the shared HTTP client when entering the context."""
52
+ await self._ensure_client()
53
+ return self
54
+
55
+ async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> None:
56
+ """Close the internally owned HTTP client on context exit."""
57
+ await self.aclose()
58
+
59
+ async def _ensure_client(self) -> httpx.AsyncClient:
60
+ """Return the shared async HTTP client, creating it if needed."""
61
+ if self._client is None:
62
+ self._client = httpx.AsyncClient(
63
+ timeout=self.timeout,
64
+ limits=httpx.Limits(
65
+ max_connections=self.max_concurrency,
66
+ max_keepalive_connections=self.max_concurrency,
67
+ ),
68
+ headers={"User-Agent": "Mozilla/5.0"},
69
+ follow_redirects=True,
70
+ )
71
+ return self._client
72
+
73
+ @staticmethod
74
+ def embed_url(shortcode: str, media_type: Any = None) -> str:
75
+ """Build the public Embed URL for a post or Reel."""
76
+ path = "reel" if media_type == 2 or media_type == "video" else "p"
77
+ return f"https://www.instagram.com/{path}/{shortcode}/embed/captioned/"
78
+
79
+ async def fetch_post(self, shortcode: str, media_type: Any = None) -> dict[str, Any]:
80
+ """Fetch one post, reusing an in-flight task for duplicate shortcodes."""
81
+ if not shortcode:
82
+ return {"shortcode": shortcode, "error": "missing shortcode"}
83
+ if shortcode in self._tasks:
84
+ return await self._tasks[shortcode]
85
+ task = asyncio.create_task(self._fetch_post_once(shortcode, media_type))
86
+ self._tasks[shortcode] = task
87
+ return await task
88
+
89
+ async def _fetch_post_once(self, shortcode: str, media_type: Any) -> dict[str, Any]:
90
+ """Fetch, parse, and normalize one Embed response with retry handling."""
91
+ client = await self._ensure_client()
92
+ url = self.embed_url(shortcode, media_type)
93
+ last_error: Exception | None = None
94
+ for attempt in range(self.max_retries + 1):
95
+ try:
96
+ async with self._semaphore:
97
+ response = await client.get(url)
98
+ if response.status_code == 404:
99
+ return {"shortcode": shortcode, "error": "HTTP 404"}
100
+ if response.status_code in {401, 403}:
101
+ return {"shortcode": shortcode, "error": f"HTTP {response.status_code}"}
102
+ if response.status_code == 429 or response.status_code >= 500:
103
+ response.raise_for_status()
104
+ response.raise_for_status()
105
+ return normalize_embed_html(response.text, shortcode)
106
+ except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as error:
107
+ last_error = error
108
+ if attempt < self.max_retries:
109
+ await asyncio.sleep(2**attempt)
110
+ except httpx.HTTPStatusError as error:
111
+ status_code = error.response.status_code
112
+ if status_code in {429, 500, 502, 503, 504} and attempt < self.max_retries:
113
+ await asyncio.sleep(2**attempt)
114
+ continue
115
+ return {"shortcode": shortcode, "error": f"HTTP {status_code}"}
116
+ except (ValueError, KeyError) as error:
117
+ return {"shortcode": shortcode, "error": str(error)}
118
+ return {"shortcode": shortcode, "error": str(last_error or "request failed")}
119
+
120
+ async def fetch_many(self, posts: Iterable[Mapping[str, Any] | str]) -> list[dict[str, Any]]:
121
+ """Fetch many posts while preserving input order and failure results."""
122
+ items = list(posts)
123
+ tasks = []
124
+ for item in items:
125
+ if isinstance(item, str):
126
+ tasks.append(self.fetch_post(item))
127
+ else:
128
+ tasks.append(self.fetch_post(str(item.get("shortcode") or ""), item.get("media_type")))
129
+ return await asyncio.gather(*tasks)
130
+
131
+ async def aclose(self) -> None:
132
+ """Close the internally owned HTTP client."""
133
+ if self._owns_client and self._client is not None:
134
+ await self._client.aclose()
135
+ self._client = None
136
+
137
+
138
+ async def fetch_posts_embed(
139
+ posts: Iterable[Mapping[str, Any] | str],
140
+ max_concurrency: int = 10,
141
+ timeout: float = 30,
142
+ max_retries: int = 2,
143
+ ) -> list[dict[str, Any]]:
144
+ """Fetch a batch of Embed posts using a temporary bounded client.
145
+
146
+ Args:
147
+ posts: Shortcodes or timeline post dictionaries.
148
+ max_concurrency: Maximum number of concurrent requests.
149
+ timeout: Per-request timeout in seconds.
150
+ max_retries: Number of transient retries.
151
+
152
+ Returns:
153
+ Normalized detail results in input order.
154
+ """
155
+ async with InstagramEmbedClient(max_concurrency, timeout, max_retries) as client:
156
+ return await client.fetch_many(posts)
157
+
158
+
159
+ def normalize_embed_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
160
+ """Normalize context JSON, falling back to exact HTML metrics when needed."""
161
+ try:
162
+ context = extract_context_json(html)
163
+ return normalize_media(get_media(context))
164
+ except (ValueError, KeyError) as error:
165
+ fallback = parse_post_detail_html(html, shortcode)
166
+ if fallback.get("detail_source"):
167
+ return fallback
168
+ raise ValueError(str(error)) from error
@@ -0,0 +1,31 @@
1
+ """Basic Instagram scraper usage examples."""
2
+
3
+ # -*- coding: utf-8 -*-
4
+ if __package__:
5
+ from . import InstagramGraphqlScraper as ig_graphql_scraper
6
+ else:
7
+ from scraper import InstagramGraphqlScraper as ig_graphql_scraper
8
+
9
+
10
+ ## Example.1 - without logging in
11
+ if __name__ == "__main__":
12
+ instagram_user_name = "love.yuweishao"
13
+ instagram_user_id = "100044253168423"
14
+ days_limit = 100 # Number of days within which to scrape posts
15
+ driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver"
16
+ ig_spider = ig_graphql_scraper(driver_path=driver_path, open_browser=False)
17
+ res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_id, days_limit=days_limit, display_progress=True)
18
+
19
+
20
+ ## Example.2 - login in your Instagram account to collect data
21
+ # if __name__ == "__main__":
22
+ # instagram_user_name = "love.yuweishao"
23
+ # instagram_user_id = "100044253168423"
24
+ # ig_account = "instagram_account"
25
+ # ig_pwd = "instagram_password"
26
+ # days_limit = 30 # Number of days within which to scrape posts
27
+ # driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver"
28
+ # ig_spider = ig_graphql_scraper(ig_account=ig_account, ig_pwd=ig_pwd, driver_path=driver_path, open_browser=False)
29
+ # res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_name, days_limit=days_limit, display_progress=True)
30
+ # print(res)
31
+