instagram-graphql-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- instagram_graphql_scraper-0.1.0/LICENSE +21 -0
- instagram_graphql_scraper-0.1.0/PKG-INFO +93 -0
- instagram_graphql_scraper-0.1.0/README.md +70 -0
- instagram_graphql_scraper-0.1.0/__init__.py +6 -0
- instagram_graphql_scraper-0.1.0/base/__init__.py +1 -0
- instagram_graphql_scraper-0.1.0/base/base.py +46 -0
- instagram_graphql_scraper-0.1.0/detail.py +178 -0
- instagram_graphql_scraper-0.1.0/embed.py +168 -0
- instagram_graphql_scraper-0.1.0/example.py +31 -0
- instagram_graphql_scraper-0.1.0/graphql.py +324 -0
- instagram_graphql_scraper-0.1.0/instagram_context_json.py +149 -0
- instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/PKG-INFO +93 -0
- instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/SOURCES.txt +23 -0
- instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/dependency_links.txt +1 -0
- instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/requires.txt +5 -0
- instagram_graphql_scraper-0.1.0/instagram_graphql_scraper.egg-info/top_level.txt +1 -0
- instagram_graphql_scraper-0.1.0/manual_integration.py +20 -0
- instagram_graphql_scraper-0.1.0/models.py +21 -0
- instagram_graphql_scraper-0.1.0/pages/__init__.py +1 -0
- instagram_graphql_scraper-0.1.0/pages/page_optional.py +151 -0
- instagram_graphql_scraper-0.1.0/scraper.py +339 -0
- instagram_graphql_scraper-0.1.0/setup.cfg +4 -0
- instagram_graphql_scraper-0.1.0/setup.py +31 -0
- instagram_graphql_scraper-0.1.0/tests/test_graphql.py +404 -0
- instagram_graphql_scraper-0.1.0/utils/locator.py +40 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 faustren
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: instagram-graphql-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Collects posts from a public Instagram profile via captured GraphQL requests.
|
|
5
|
+
Author: faustren
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Requires-Dist: selenium>=4.20
|
|
11
|
+
Requires-Dist: selenium-wire>=5.1
|
|
12
|
+
Requires-Dist: requests>=2.31
|
|
13
|
+
Requires-Dist: brotli>=1.1
|
|
14
|
+
Requires-Dist: httpx>=0.27
|
|
15
|
+
Dynamic: author
|
|
16
|
+
Dynamic: description
|
|
17
|
+
Dynamic: description-content-type
|
|
18
|
+
Dynamic: license
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
Dynamic: requires-dist
|
|
21
|
+
Dynamic: requires-python
|
|
22
|
+
Dynamic: summary
|
|
23
|
+
|
|
24
|
+
# instagram-graphql-scraper
|
|
25
|
+
|
|
26
|
+
Collects posts from a public Instagram profile by capturing the browser's profile GraphQL request, then replaying subsequent pages through one `requests.Session`.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
|
|
30
|
+
```bash
|
|
31
|
+
pip install -r requirements.txt
|
|
32
|
+
# or
|
|
33
|
+
pip install -e .
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Usage
|
|
37
|
+
|
|
38
|
+
```python
|
|
39
|
+
from instagram_graphql_scraper import InstagramGraphqlScraper
|
|
40
|
+
|
|
41
|
+
scraper = InstagramGraphqlScraper(driver_path="/path/to/chromedriver")
|
|
42
|
+
try:
|
|
43
|
+
posts = scraper.get_user_posts("username", max_pages=2, max_posts=24)
|
|
44
|
+
finally:
|
|
45
|
+
scraper.close()
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
The browser flow opens the profile, dismisses an optional login dialog, scrolls once, clears Selenium Wire requests, and clicks the dynamic show-more-posts button once. The captured request supplies the URL, headers, cookies, form payload, variables, and `doc_id`; no dynamic GraphQL values are hardcoded.
|
|
49
|
+
|
|
50
|
+
`days_limit` keeps posts newer than N days and stops pagination early once older posts are reached, using each post's `accessibility_caption` date; posts without a parseable date are kept. Combine with `max_pages` or `max_posts` for additional bounds.
|
|
51
|
+
|
|
52
|
+
For bounded asynchronous Embed enrichment (opt-in, not run automatically by `get_user_posts`):
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
import asyncio
|
|
56
|
+
from instagram_graphql_scraper import InstagramEmbedClient
|
|
57
|
+
|
|
58
|
+
async def enrich(posts):
|
|
59
|
+
async with InstagramEmbedClient(max_concurrency=10) as client:
|
|
60
|
+
return await client.fetch_many(posts)
|
|
61
|
+
|
|
62
|
+
details = asyncio.run(enrich(posts))
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The batch client preserves input order, deduplicates shortcodes per run, and returns a per-post error result instead of failing the whole batch. The single-post normalized CLI remains available:
|
|
66
|
+
|
|
67
|
+
```bash
|
|
68
|
+
python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/"
|
|
69
|
+
python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/" --full
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Tests
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
python3 -m pytest -q
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
The manual browser check is intentionally not part of CI:
|
|
79
|
+
|
|
80
|
+
```bash
|
|
81
|
+
python3 manual_integration.py 1989ivyshao --driver-path /path/to/chromedriver --max-pages 2
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## Documentation
|
|
85
|
+
|
|
86
|
+
See [docs/](docs/) for the system architecture diagram, class diagram, and a bilingual ([繁體中文](docs/sre_testing_runbook.md) / [English](docs/sre_testing_runbook.en.md)) SRE testing runbook covering environment setup, parameter boundaries, and known issues.
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
Do not place cookies, CSRF/LSD tokens, or captured payloads in source, fixtures, or logs.
|
|
90
|
+
|
|
91
|
+
## License
|
|
92
|
+
|
|
93
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# instagram-graphql-scraper
|
|
2
|
+
|
|
3
|
+
Collects posts from a public Instagram profile by capturing the browser's profile GraphQL request, then replaying subsequent pages through one `requests.Session`.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
pip install -r requirements.txt
|
|
9
|
+
# or
|
|
10
|
+
pip install -e .
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from instagram_graphql_scraper import InstagramGraphqlScraper
|
|
17
|
+
|
|
18
|
+
scraper = InstagramGraphqlScraper(driver_path="/path/to/chromedriver")
|
|
19
|
+
try:
|
|
20
|
+
posts = scraper.get_user_posts("username", max_pages=2, max_posts=24)
|
|
21
|
+
finally:
|
|
22
|
+
scraper.close()
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
The browser flow opens the profile, dismisses an optional login dialog, scrolls once, clears Selenium Wire requests, and clicks the dynamic show-more-posts button once. The captured request supplies the URL, headers, cookies, form payload, variables, and `doc_id`; no dynamic GraphQL values are hardcoded.
|
|
26
|
+
|
|
27
|
+
`days_limit` keeps posts newer than N days and stops pagination early once older posts are reached, using each post's `accessibility_caption` date; posts without a parseable date are kept. Combine with `max_pages` or `max_posts` for additional bounds.
|
|
28
|
+
|
|
29
|
+
For bounded asynchronous Embed enrichment (opt-in, not run automatically by `get_user_posts`):
|
|
30
|
+
|
|
31
|
+
```python
|
|
32
|
+
import asyncio
|
|
33
|
+
from instagram_graphql_scraper import InstagramEmbedClient
|
|
34
|
+
|
|
35
|
+
async def enrich(posts):
|
|
36
|
+
async with InstagramEmbedClient(max_concurrency=10) as client:
|
|
37
|
+
return await client.fetch_many(posts)
|
|
38
|
+
|
|
39
|
+
details = asyncio.run(enrich(posts))
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
The batch client preserves input order, deduplicates shortcodes per run, and returns a per-post error result instead of failing the whole batch. The single-post normalized CLI remains available:
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/"
|
|
46
|
+
python3 instagram_context_json.py "https://www.instagram.com/p/SHORTCODE/embed/captioned/" --full
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
## Tests
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python3 -m pytest -q
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
The manual browser check is intentionally not part of CI:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
python3 manual_integration.py 1989ivyshao --driver-path /path/to/chromedriver --max-pages 2
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
## Documentation
|
|
62
|
+
|
|
63
|
+
See [docs/](docs/) for the system architecture diagram, class diagram, and a bilingual ([繁體中文](docs/sre_testing_runbook.md) / [English](docs/sre_testing_runbook.en.md)) SRE testing runbook covering environment setup, parameter boundaries, and known issues.
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
Do not place cookies, CSRF/LSD tokens, or captured payloads in source, fixtures, or logs.
|
|
67
|
+
|
|
68
|
+
## License
|
|
69
|
+
|
|
70
|
+
MIT, see [LICENSE](LICENSE).
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Browser bootstrap package for the Instagram scraper."""
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""Selenium Wire browser bootstrap helpers."""
|
|
2
|
+
|
|
3
|
+
# -*- coding:utf-8 -*-
|
|
4
|
+
from seleniumwire import webdriver
|
|
5
|
+
from selenium.webdriver.chrome.service import Service
|
|
6
|
+
from typing import Optional
|
|
7
|
+
|
|
8
|
+
class BasePage:
|
|
9
|
+
"""Create and configure the Selenium Wire Chrome driver."""
|
|
10
|
+
|
|
11
|
+
def __init__(self, driver_path: Optional[str] = None, open_browser: bool = False):
|
|
12
|
+
"""Initialize a Chrome driver.
|
|
13
|
+
|
|
14
|
+
Args:
|
|
15
|
+
driver_path: Optional path to the ChromeDriver executable.
|
|
16
|
+
open_browser: Whether to show the browser instead of using headless mode.
|
|
17
|
+
"""
|
|
18
|
+
chrome_options = self._build_options(open_browser)
|
|
19
|
+
service = Service(driver_path) if driver_path else Service()
|
|
20
|
+
self.driver = webdriver.Chrome(service=service, options=chrome_options)
|
|
21
|
+
if open_browser:
|
|
22
|
+
self.driver.maximize_window()
|
|
23
|
+
else:
|
|
24
|
+
self.driver.set_window_size(1920, 1080)
|
|
25
|
+
|
|
26
|
+
@staticmethod
|
|
27
|
+
def _build_options(open_browser: bool) -> webdriver.ChromeOptions:
|
|
28
|
+
"""Build Chrome options used by the scraper browser.
|
|
29
|
+
|
|
30
|
+
Args:
|
|
31
|
+
open_browser: Whether headless mode should be disabled.
|
|
32
|
+
|
|
33
|
+
Returns:
|
|
34
|
+
Configured Selenium Chrome options.
|
|
35
|
+
"""
|
|
36
|
+
options = webdriver.ChromeOptions()
|
|
37
|
+
options.add_argument("--disable-blink-features")
|
|
38
|
+
options.add_argument("--disable-notifications")
|
|
39
|
+
options.add_argument("--disable-blink-features=AutomationControlled")
|
|
40
|
+
options.add_argument("--window-size=1920,1080")
|
|
41
|
+
if not open_browser:
|
|
42
|
+
options.add_argument("--headless=new")
|
|
43
|
+
options.add_argument("--blink-settings=imagesEnabled=false")
|
|
44
|
+
options.add_experimental_option("excludeSwitches", ["enable-automation"])
|
|
45
|
+
options.add_experimental_option("useAutomationExtension", False)
|
|
46
|
+
return options
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Instagram post HTML detail parsing and fail-soft merge helpers."""
|
|
2
|
+
|
|
3
|
+
import re
|
|
4
|
+
from datetime import datetime, timezone
|
|
5
|
+
from html.parser import HTMLParser
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class PostDetailParser(HTMLParser):
|
|
10
|
+
"""Collect metadata, timestamps, and exact count anchors from post HTML."""
|
|
11
|
+
|
|
12
|
+
def __init__(self):
|
|
13
|
+
"""Initialize parser state for one Instagram detail response."""
|
|
14
|
+
super().__init__()
|
|
15
|
+
self.meta: dict[str, str] = {}
|
|
16
|
+
self.times: list[str] = []
|
|
17
|
+
self._count_anchor: str | None = None
|
|
18
|
+
self._count_text: list[str] = []
|
|
19
|
+
self.exact_counts: dict[str, int] = {}
|
|
20
|
+
|
|
21
|
+
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
22
|
+
"""Capture relevant metadata from an opening HTML tag.
|
|
23
|
+
|
|
24
|
+
Args:
|
|
25
|
+
tag: HTML tag name.
|
|
26
|
+
attrs: Parsed HTML attributes.
|
|
27
|
+
"""
|
|
28
|
+
attributes = dict(attrs)
|
|
29
|
+
if tag == "meta":
|
|
30
|
+
key = attributes.get("name") or attributes.get("property")
|
|
31
|
+
content = attributes.get("content")
|
|
32
|
+
if key and content and key in {"description", "og:description"}:
|
|
33
|
+
self.meta[key] = content
|
|
34
|
+
elif tag == "time" and attributes.get("datetime"):
|
|
35
|
+
datetime_value = attributes.get("datetime")
|
|
36
|
+
if datetime_value:
|
|
37
|
+
self.times.append(datetime_value)
|
|
38
|
+
elif tag == "a":
|
|
39
|
+
event = attributes.get("data-log-event")
|
|
40
|
+
if event in {"likeCountClick", "captionCommentsClick"}:
|
|
41
|
+
self._count_anchor = event
|
|
42
|
+
self._count_text = []
|
|
43
|
+
|
|
44
|
+
def handle_data(self, data: str) -> None:
|
|
45
|
+
"""Collect text belonging to an exact count anchor."""
|
|
46
|
+
if self._count_anchor:
|
|
47
|
+
self._count_text.append(data)
|
|
48
|
+
|
|
49
|
+
def handle_endtag(self, tag: str) -> None:
|
|
50
|
+
"""Finalize an exact count when its anchor closes."""
|
|
51
|
+
if tag == "a" and self._count_anchor:
|
|
52
|
+
text = "".join(self._count_text)
|
|
53
|
+
match = re.search(r"([\d,]+)\s+(?:likes|comments)", text, re.IGNORECASE)
|
|
54
|
+
if match:
|
|
55
|
+
count = int(match.group(1).replace(",", ""))
|
|
56
|
+
field = "like_count" if self._count_anchor == "likeCountClick" else "comment_count"
|
|
57
|
+
self.exact_counts[field] = count
|
|
58
|
+
self._count_anchor = None
|
|
59
|
+
self._count_text = []
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _parse_count(description: str, label: str) -> tuple[int | None, bool]:
|
|
63
|
+
"""Parse a displayed count and report whether it uses a rounded suffix.
|
|
64
|
+
|
|
65
|
+
Args:
|
|
66
|
+
description: Post description metadata text.
|
|
67
|
+
label: Metric label such as ``likes`` or ``comments``.
|
|
68
|
+
|
|
69
|
+
Returns:
|
|
70
|
+
A tuple containing the parsed count and an approximation flag.
|
|
71
|
+
"""
|
|
72
|
+
match = re.search(rf"([\d,.]+)\s*([KMB])?\s+{label}\b", description, re.IGNORECASE)
|
|
73
|
+
if not match:
|
|
74
|
+
return None, False
|
|
75
|
+
number = float(match.group(1).replace(",", ""))
|
|
76
|
+
suffix = (match.group(2) or "").upper()
|
|
77
|
+
multiplier = {"": 1, "K": 1_000, "M": 1_000_000, "B": 1_000_000_000}[suffix]
|
|
78
|
+
return int(number * multiplier), bool(suffix)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def _parse_embedded_count(html: str, field_name: str) -> int | None:
|
|
82
|
+
"""Extract an exact count from embedded Instagram structured data."""
|
|
83
|
+
pattern = rf'\\?"{field_name}\\?"\s*:\s*\\?\{{\\?"count\\?"\s*:\s*(\d+)'
|
|
84
|
+
match = re.search(pattern, html)
|
|
85
|
+
return int(match.group(1)) if match else None
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def _parse_datetime(value: str | None) -> tuple[int | None, str | None]:
|
|
89
|
+
"""Convert an ISO timestamp to UTC Unix and ISO-8601 values."""
|
|
90
|
+
if not value:
|
|
91
|
+
return None, None
|
|
92
|
+
try:
|
|
93
|
+
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
94
|
+
except ValueError:
|
|
95
|
+
return None, None
|
|
96
|
+
if parsed.tzinfo is None:
|
|
97
|
+
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
98
|
+
parsed = parsed.astimezone(timezone.utc)
|
|
99
|
+
return int(parsed.timestamp()), parsed.isoformat()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _parse_description_date(description: str) -> str | None:
|
|
103
|
+
"""Extract a date-only fallback from an English description."""
|
|
104
|
+
match = re.search(r"\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2}),\s+(\d{4})\b", description)
|
|
105
|
+
if not match:
|
|
106
|
+
return None
|
|
107
|
+
try:
|
|
108
|
+
return datetime.strptime(" ".join(match.groups()), "%B %d %Y").date().isoformat()
|
|
109
|
+
except ValueError:
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def parse_post_detail_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
|
|
114
|
+
"""Normalize metrics and publication metadata from post HTML.
|
|
115
|
+
|
|
116
|
+
Args:
|
|
117
|
+
html: HTML returned by an Instagram post or Embed page.
|
|
118
|
+
shortcode: Optional shortcode associated with the response.
|
|
119
|
+
|
|
120
|
+
Returns:
|
|
121
|
+
Normalized detail fields with ``None`` for unavailable values.
|
|
122
|
+
"""
|
|
123
|
+
parser = PostDetailParser()
|
|
124
|
+
parser.feed(html)
|
|
125
|
+
description = parser.meta.get("description") or parser.meta.get("og:description") or ""
|
|
126
|
+
like_count, likes_approximate = _parse_count(description, "likes")
|
|
127
|
+
comment_count, comments_approximate = _parse_count(description, "comments")
|
|
128
|
+
if "like_count" in parser.exact_counts:
|
|
129
|
+
like_count, likes_approximate = parser.exact_counts["like_count"], False
|
|
130
|
+
if "comment_count" in parser.exact_counts:
|
|
131
|
+
comment_count, comments_approximate = parser.exact_counts["comment_count"], False
|
|
132
|
+
embedded_like_count = _parse_embedded_count(html, "edge_liked_by")
|
|
133
|
+
embedded_comment_count = _parse_embedded_count(html, "edge_media_to_comment")
|
|
134
|
+
if embedded_like_count is not None:
|
|
135
|
+
like_count, likes_approximate = embedded_like_count, False
|
|
136
|
+
if embedded_comment_count is not None:
|
|
137
|
+
comment_count, comments_approximate = embedded_comment_count, False
|
|
138
|
+
video_view_count, _ = _parse_count(description, "views")
|
|
139
|
+
if video_view_count is None:
|
|
140
|
+
video_view_count, _ = _parse_count(description, "plays")
|
|
141
|
+
timestamp, published_at = _parse_datetime(parser.times[0] if parser.times else None)
|
|
142
|
+
if published_at is None:
|
|
143
|
+
published_at = _parse_description_date(description)
|
|
144
|
+
return {
|
|
145
|
+
"shortcode": shortcode,
|
|
146
|
+
"like_count": like_count,
|
|
147
|
+
"comment_count": comment_count,
|
|
148
|
+
"video_view_count": video_view_count,
|
|
149
|
+
"taken_at_timestamp": timestamp,
|
|
150
|
+
"published_at": published_at,
|
|
151
|
+
"like_count_is_approximate": likes_approximate,
|
|
152
|
+
"comment_count_is_approximate": comments_approximate,
|
|
153
|
+
"detail_source": "post_embed_html" if parser.exact_counts or embedded_like_count is not None or embedded_comment_count is not None else ("post_html_metadata" if description or parser.times else None),
|
|
154
|
+
"detail_shortcode": shortcode,
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def merge_post_detail(post: dict[str, Any], detail: dict[str, Any]) -> dict[str, Any]:
|
|
159
|
+
"""Merge missing detail fields without replacing existing post values.
|
|
160
|
+
|
|
161
|
+
Args:
|
|
162
|
+
post: Existing normalized timeline post.
|
|
163
|
+
detail: Normalized Embed or detail response.
|
|
164
|
+
|
|
165
|
+
Returns:
|
|
166
|
+
The updated post dictionary.
|
|
167
|
+
"""
|
|
168
|
+
for field in (
|
|
169
|
+
"like_count", "comment_count", "video_view_count", "video_duration",
|
|
170
|
+
"video_url", "display_uri", "product_type", "media_type", "is_video",
|
|
171
|
+
"taken_at_timestamp", "published_at",
|
|
172
|
+
):
|
|
173
|
+
if post.get(field) is None and detail.get(field) is not None:
|
|
174
|
+
post[field] = detail[field]
|
|
175
|
+
for field in ("like_count_is_approximate", "comment_count_is_approximate", "detail_source"):
|
|
176
|
+
if detail.get(field) is not None:
|
|
177
|
+
post[field] = detail[field]
|
|
178
|
+
return post
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Asynchronous Instagram Embed fetching and normalization utilities."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Iterable, Mapping
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import httpx
|
|
10
|
+
|
|
11
|
+
try:
|
|
12
|
+
from .detail import parse_post_detail_html
|
|
13
|
+
except ImportError:
|
|
14
|
+
from detail import parse_post_detail_html
|
|
15
|
+
|
|
16
|
+
try:
|
|
17
|
+
from .instagram_context_json import extract_context_json, get_media, normalize_media
|
|
18
|
+
except ImportError:
|
|
19
|
+
from instagram_context_json import extract_context_json, get_media, normalize_media
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class InstagramEmbedClient:
|
|
23
|
+
"""Fetch and normalize Instagram Embed posts with bounded concurrency."""
|
|
24
|
+
|
|
25
|
+
def __init__(
|
|
26
|
+
self,
|
|
27
|
+
max_concurrency: int = 10,
|
|
28
|
+
timeout: float = 30,
|
|
29
|
+
max_retries: int = 2,
|
|
30
|
+
client: httpx.AsyncClient | None = None,
|
|
31
|
+
):
|
|
32
|
+
"""Configure a reusable async Embed client.
|
|
33
|
+
|
|
34
|
+
Args:
|
|
35
|
+
max_concurrency: Maximum number of requests running concurrently.
|
|
36
|
+
timeout: Per-request timeout in seconds.
|
|
37
|
+
max_retries: Number of retries for transient failures.
|
|
38
|
+
client: Optional externally managed ``httpx.AsyncClient``.
|
|
39
|
+
"""
|
|
40
|
+
if max_concurrency < 1:
|
|
41
|
+
raise ValueError("max_concurrency must be at least 1")
|
|
42
|
+
self.max_concurrency = max_concurrency
|
|
43
|
+
self.timeout = timeout
|
|
44
|
+
self.max_retries = max_retries
|
|
45
|
+
self._client = client
|
|
46
|
+
self._owns_client = client is None
|
|
47
|
+
self._semaphore = asyncio.Semaphore(max_concurrency)
|
|
48
|
+
self._tasks: dict[str, asyncio.Task[dict[str, Any]]] = {}
|
|
49
|
+
|
|
50
|
+
async def __aenter__(self) -> "InstagramEmbedClient":
|
|
51
|
+
"""Create the shared HTTP client when entering the context."""
|
|
52
|
+
await self._ensure_client()
|
|
53
|
+
return self
|
|
54
|
+
|
|
55
|
+
async def __aexit__(self, exc_type: Any, exc: Any, traceback: Any) -> None:
|
|
56
|
+
"""Close the internally owned HTTP client on context exit."""
|
|
57
|
+
await self.aclose()
|
|
58
|
+
|
|
59
|
+
async def _ensure_client(self) -> httpx.AsyncClient:
|
|
60
|
+
"""Return the shared async HTTP client, creating it if needed."""
|
|
61
|
+
if self._client is None:
|
|
62
|
+
self._client = httpx.AsyncClient(
|
|
63
|
+
timeout=self.timeout,
|
|
64
|
+
limits=httpx.Limits(
|
|
65
|
+
max_connections=self.max_concurrency,
|
|
66
|
+
max_keepalive_connections=self.max_concurrency,
|
|
67
|
+
),
|
|
68
|
+
headers={"User-Agent": "Mozilla/5.0"},
|
|
69
|
+
follow_redirects=True,
|
|
70
|
+
)
|
|
71
|
+
return self._client
|
|
72
|
+
|
|
73
|
+
@staticmethod
|
|
74
|
+
def embed_url(shortcode: str, media_type: Any = None) -> str:
|
|
75
|
+
"""Build the public Embed URL for a post or Reel."""
|
|
76
|
+
path = "reel" if media_type == 2 or media_type == "video" else "p"
|
|
77
|
+
return f"https://www.instagram.com/{path}/{shortcode}/embed/captioned/"
|
|
78
|
+
|
|
79
|
+
async def fetch_post(self, shortcode: str, media_type: Any = None) -> dict[str, Any]:
|
|
80
|
+
"""Fetch one post, reusing an in-flight task for duplicate shortcodes."""
|
|
81
|
+
if not shortcode:
|
|
82
|
+
return {"shortcode": shortcode, "error": "missing shortcode"}
|
|
83
|
+
if shortcode in self._tasks:
|
|
84
|
+
return await self._tasks[shortcode]
|
|
85
|
+
task = asyncio.create_task(self._fetch_post_once(shortcode, media_type))
|
|
86
|
+
self._tasks[shortcode] = task
|
|
87
|
+
return await task
|
|
88
|
+
|
|
89
|
+
async def _fetch_post_once(self, shortcode: str, media_type: Any) -> dict[str, Any]:
|
|
90
|
+
"""Fetch, parse, and normalize one Embed response with retry handling."""
|
|
91
|
+
client = await self._ensure_client()
|
|
92
|
+
url = self.embed_url(shortcode, media_type)
|
|
93
|
+
last_error: Exception | None = None
|
|
94
|
+
for attempt in range(self.max_retries + 1):
|
|
95
|
+
try:
|
|
96
|
+
async with self._semaphore:
|
|
97
|
+
response = await client.get(url)
|
|
98
|
+
if response.status_code == 404:
|
|
99
|
+
return {"shortcode": shortcode, "error": "HTTP 404"}
|
|
100
|
+
if response.status_code in {401, 403}:
|
|
101
|
+
return {"shortcode": shortcode, "error": f"HTTP {response.status_code}"}
|
|
102
|
+
if response.status_code == 429 or response.status_code >= 500:
|
|
103
|
+
response.raise_for_status()
|
|
104
|
+
response.raise_for_status()
|
|
105
|
+
return normalize_embed_html(response.text, shortcode)
|
|
106
|
+
except (httpx.TimeoutException, httpx.NetworkError, httpx.RemoteProtocolError) as error:
|
|
107
|
+
last_error = error
|
|
108
|
+
if attempt < self.max_retries:
|
|
109
|
+
await asyncio.sleep(2**attempt)
|
|
110
|
+
except httpx.HTTPStatusError as error:
|
|
111
|
+
status_code = error.response.status_code
|
|
112
|
+
if status_code in {429, 500, 502, 503, 504} and attempt < self.max_retries:
|
|
113
|
+
await asyncio.sleep(2**attempt)
|
|
114
|
+
continue
|
|
115
|
+
return {"shortcode": shortcode, "error": f"HTTP {status_code}"}
|
|
116
|
+
except (ValueError, KeyError) as error:
|
|
117
|
+
return {"shortcode": shortcode, "error": str(error)}
|
|
118
|
+
return {"shortcode": shortcode, "error": str(last_error or "request failed")}
|
|
119
|
+
|
|
120
|
+
async def fetch_many(self, posts: Iterable[Mapping[str, Any] | str]) -> list[dict[str, Any]]:
|
|
121
|
+
"""Fetch many posts while preserving input order and failure results."""
|
|
122
|
+
items = list(posts)
|
|
123
|
+
tasks = []
|
|
124
|
+
for item in items:
|
|
125
|
+
if isinstance(item, str):
|
|
126
|
+
tasks.append(self.fetch_post(item))
|
|
127
|
+
else:
|
|
128
|
+
tasks.append(self.fetch_post(str(item.get("shortcode") or ""), item.get("media_type")))
|
|
129
|
+
return await asyncio.gather(*tasks)
|
|
130
|
+
|
|
131
|
+
async def aclose(self) -> None:
|
|
132
|
+
"""Close the internally owned HTTP client."""
|
|
133
|
+
if self._owns_client and self._client is not None:
|
|
134
|
+
await self._client.aclose()
|
|
135
|
+
self._client = None
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
async def fetch_posts_embed(
|
|
139
|
+
posts: Iterable[Mapping[str, Any] | str],
|
|
140
|
+
max_concurrency: int = 10,
|
|
141
|
+
timeout: float = 30,
|
|
142
|
+
max_retries: int = 2,
|
|
143
|
+
) -> list[dict[str, Any]]:
|
|
144
|
+
"""Fetch a batch of Embed posts using a temporary bounded client.
|
|
145
|
+
|
|
146
|
+
Args:
|
|
147
|
+
posts: Shortcodes or timeline post dictionaries.
|
|
148
|
+
max_concurrency: Maximum number of concurrent requests.
|
|
149
|
+
timeout: Per-request timeout in seconds.
|
|
150
|
+
max_retries: Number of transient retries.
|
|
151
|
+
|
|
152
|
+
Returns:
|
|
153
|
+
Normalized detail results in input order.
|
|
154
|
+
"""
|
|
155
|
+
async with InstagramEmbedClient(max_concurrency, timeout, max_retries) as client:
|
|
156
|
+
return await client.fetch_many(posts)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def normalize_embed_html(html: str, shortcode: str | None = None) -> dict[str, Any]:
|
|
160
|
+
"""Normalize context JSON, falling back to exact HTML metrics when needed."""
|
|
161
|
+
try:
|
|
162
|
+
context = extract_context_json(html)
|
|
163
|
+
return normalize_media(get_media(context))
|
|
164
|
+
except (ValueError, KeyError) as error:
|
|
165
|
+
fallback = parse_post_detail_html(html, shortcode)
|
|
166
|
+
if fallback.get("detail_source"):
|
|
167
|
+
return fallback
|
|
168
|
+
raise ValueError(str(error)) from error
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Basic Instagram scraper usage examples."""
|
|
2
|
+
|
|
3
|
+
# -*- coding: utf-8 -*-
|
|
4
|
+
if __package__:
|
|
5
|
+
from . import InstagramGraphqlScraper as ig_graphql_scraper
|
|
6
|
+
else:
|
|
7
|
+
from scraper import InstagramGraphqlScraper as ig_graphql_scraper
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
## Example.1 - without logging in
|
|
11
|
+
if __name__ == "__main__":
|
|
12
|
+
instagram_user_name = "love.yuweishao"
|
|
13
|
+
instagram_user_id = "100044253168423"
|
|
14
|
+
days_limit = 100 # Number of days within which to scrape posts
|
|
15
|
+
driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver"
|
|
16
|
+
ig_spider = ig_graphql_scraper(driver_path=driver_path, open_browser=False)
|
|
17
|
+
res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_id, days_limit=days_limit, display_progress=True)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
## Example.2 - login in your Instagram account to collect data
|
|
21
|
+
# if __name__ == "__main__":
|
|
22
|
+
# instagram_user_name = "love.yuweishao"
|
|
23
|
+
# instagram_user_id = "100044253168423"
|
|
24
|
+
# ig_account = "instagram_account"
|
|
25
|
+
# ig_pwd = "instagram_password"
|
|
26
|
+
# days_limit = 30 # Number of days within which to scrape posts
|
|
27
|
+
# driver_path = "/Users/hongshangren/Downloads/chromedriver-mac-arm64_136/chromedriver"
|
|
28
|
+
# ig_spider = ig_graphql_scraper(ig_account=ig_account, ig_pwd=ig_pwd, driver_path=driver_path, open_browser=False)
|
|
29
|
+
# res = ig_spider.get_user_posts(ig_username_or_userid=instagram_user_name, days_limit=days_limit, display_progress=True)
|
|
30
|
+
# print(res)
|
|
31
|
+
|