myntra-scraper 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- myntra_scraper-0.1.0/LICENSE +21 -0
- myntra_scraper-0.1.0/MANIFEST.in +3 -0
- myntra_scraper-0.1.0/PKG-INFO +146 -0
- myntra_scraper-0.1.0/README.md +124 -0
- myntra_scraper-0.1.0/examples/listing_to_sizes.py +54 -0
- myntra_scraper-0.1.0/pyproject.toml +34 -0
- myntra_scraper-0.1.0/setup.cfg +4 -0
- myntra_scraper-0.1.0/src/myntra_scraper/__init__.py +32 -0
- myntra_scraper-0.1.0/src/myntra_scraper/cli.py +110 -0
- myntra_scraper-0.1.0/src/myntra_scraper/client.py +180 -0
- myntra_scraper-0.1.0/src/myntra_scraper/parser.py +212 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/PKG-INFO +146 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/SOURCES.txt +18 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/dependency_links.txt +1 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/entry_points.txt +2 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/requires.txt +1 -0
- myntra_scraper-0.1.0/src/myntra_scraper.egg-info/top_level.txt +1 -0
- myntra_scraper-0.1.0/tests/fixtures/listing_page.html +7 -0
- myntra_scraper-0.1.0/tests/fixtures/product_page.html +7 -0
- myntra_scraper-0.1.0/tests/test_parser.py +184 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Buğrahan Saka
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: myntra-scraper
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Scrape Myntra listings (price, MRP, discount, rating) with real pagination, and product pages (price and stock per size and seller) with Python.
|
|
5
|
+
Author: Buğrahan Saka
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/bugrahansss/myntra-scraper
|
|
8
|
+
Project-URL: Source, https://github.com/bugrahansss/myntra-scraper
|
|
9
|
+
Project-URL: Issues, https://github.com/bugrahansss/myntra-scraper/issues
|
|
10
|
+
Project-URL: Scrape.do, https://scrape.do/?utm_source=pypi&utm_medium=package&utm_campaign=myntra-scraper
|
|
11
|
+
Keywords: myntra,web scraping,fashion,ecommerce,scraper,akamai,india
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
17
|
+
Requires-Python: >=3.8
|
|
18
|
+
Description-Content-Type: text/markdown
|
|
19
|
+
License-File: LICENSE
|
|
20
|
+
Requires-Dist: requests>=2.25
|
|
21
|
+
Dynamic: license-file
|
|
22
|
+
|
|
23
|
+
# myntra-scraper
|
|
24
|
+
|
|
25
|
+
[](https://pypi.org/project/myntra-scraper/)
|
|
26
|
+
[](https://pypi.org/project/myntra-scraper/)
|
|
27
|
+
[](LICENSE)
|
|
28
|
+
|
|
29
|
+
A Python scraper for [Myntra](https://www.myntra.com), India's largest fashion marketplace. Two jobs:
|
|
30
|
+
|
|
31
|
+
- **Listings and search:** any listing or search URL (`/men-tshirts`, `/levis?rawQuery=levis`, with sort and filters) → name, brand, price, MRP, discount, rating, rating count, sizes, colour, image and URL. **Real pagination**, 50 products per page, sponsored products left out.
|
|
32
|
+
- **Product pages:** price, MRP, rating and attributes, plus **one row per size and seller with that seller's price and the exact units in stock**.
|
|
33
|
+
|
|
34
|
+
Output as Python dicts, JSON or CSV. One dependency (`requests`), no browser.
|
|
35
|
+
|
|
36
|
+
```bash
|
|
37
|
+
pip install myntra-scraper
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
## Why a plain request fails
|
|
41
|
+
|
|
42
|
+
Myntra sits behind Akamai Bot Manager (`_abck`, `bm_sz` cookies). A script doesn't get an error, it gets nothing:
|
|
43
|
+
|
|
44
|
+
| Request | Result |
|
|
45
|
+
| --- | --- |
|
|
46
|
+
| `requests`, default or Chrome user agent | connection opens, no bytes, timeout after 20 s |
|
|
47
|
+
| `curl` (HTTP/2) | `stream was not closed cleanly: INTERNAL_ERROR` |
|
|
48
|
+
| Through Scrape.do, no parameters | 200 in ~3 s, full page, 1 credit |
|
|
49
|
+
|
|
50
|
+
Pages are fetched through the [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=myntra-scraper) API with a plain request: no `render`, no `super`, 1 credit per page. You need a Scrape.do API token; the free plan includes 1,000 credits a month.
|
|
51
|
+
|
|
52
|
+
## Quick start
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from myntra_scraper import MyntraScraper
|
|
56
|
+
|
|
57
|
+
scraper = MyntraScraper(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
58
|
+
|
|
59
|
+
# 3 pages of 50 = 150 products (4 credits: one page visit + three API pages)
|
|
60
|
+
rows = scraper.get_listing("https://www.myntra.com/men-tshirts?sort=new", max_pages=3)
|
|
61
|
+
for row in rows[:3]:
|
|
62
|
+
print(row["brand"], row["name"], row["price_inr"], row["mrp_inr"], row["discount_pct"])
|
|
63
|
+
|
|
64
|
+
product = scraper.get_product(rows[0]["url"])
|
|
65
|
+
print(product["units_in_stock"], product["sizes_in_stock"], "/", product["sizes_total"], "sizes in stock")
|
|
66
|
+
for size in product["sizes"]:
|
|
67
|
+
print(size["size"], size["seller"], size["price_inr"], size["units"])
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
### Command line
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
export SCRAPEDO_TOKEN=your_token
|
|
74
|
+
|
|
75
|
+
# 3 pages of a listing, CSV
|
|
76
|
+
myntra-scraper listing "https://www.myntra.com/men-tshirts?sort=new" --pages 3 -f csv -o myntra-listing.csv
|
|
77
|
+
|
|
78
|
+
# product summaries for the first 10 products from that file
|
|
79
|
+
myntra-scraper product -i myntra-listing.csv --limit 10 -f csv -o myntra-products.csv
|
|
80
|
+
|
|
81
|
+
# one row per size and seller
|
|
82
|
+
myntra-scraper product -i myntra-listing.csv --limit 10 --sizes -f csv -o myntra-sizes.csv
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
`product` reads the `url` column of the listing CSV, so the commands chain.
|
|
86
|
+
|
|
87
|
+
### Output
|
|
88
|
+
|
|
89
|
+
Size rows (from the fictional test page in [`tests/fixtures`](tests/fixtures/product_page.html)):
|
|
90
|
+
|
|
91
|
+
```json
|
|
92
|
+
[
|
|
93
|
+
{"product_id": 90000001, "size": "M", "sku_id": 900000012, "available": true,
|
|
94
|
+
"seller": "Example Retail Two", "seller_id": 202, "price_inr": 449, "mrp_inr": 999, "units": 5},
|
|
95
|
+
{"product_id": 90000001, "size": "M", "sku_id": 900000012, "available": true,
|
|
96
|
+
"seller": "Example Retail One", "seller_id": 101, "price_inr": 469, "mrp_inr": 999, "units": 30}
|
|
97
|
+
]
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Listing columns: `product_id, name, brand, price_inr, mrp_inr, discount_pct, rating, rating_count, sizes, gender, article_type, colour, image, url`.
|
|
101
|
+
|
|
102
|
+
Product columns: `product_id, name, brand, price_inr, mrp_inr, discount_pct, rating, rating_count, colour, article_type, gender, country_of_origin, sizes_in_stock, sizes_total, units_in_stock, lowest_size_price_inr, highest_size_price_inr, sellers, url` (plus `attributes` and `sizes` in Python/JSON).
|
|
103
|
+
|
|
104
|
+
## How it works
|
|
105
|
+
|
|
106
|
+
**The data is one JSON object.** Myntra renders every page on the server and puts the data in an inline script, `window.__myx = {...}`. Listings keep their products in `searchData.results.products`, product pages keep everything in `pdpData`. The scraper finds the assignment and reads the object with `json.JSONDecoder.raw_decode`. No CSS selectors.
|
|
107
|
+
|
|
108
|
+
**Sponsored products are separate.** A listing page carries the organic results in `products` and the ads in `plaProducts` (18 of them on a men's T-shirts page). Only the organic products are returned.
|
|
109
|
+
|
|
110
|
+
**`?p=2` is ignored.** Ask the HTML for page 2, 3 or 50 and you get the first 50 products again. In our test, page 2 of a listing sorted by newest shared 49 of 50 products with page 1. The site pages through its search API, `/gateway/v4/search/<slug>`, which:
|
|
111
|
+
|
|
112
|
+
1. returns 401 without the cookies a page visit sets (`at`, `_abck`, `bm_sz` and others), and
|
|
113
|
+
2. ignores `p` and `o` as well, unless the request carries a `pagination-context` header. Every API response returns a new `pagination-context` header; sending it back moves to the next 50.
|
|
114
|
+
|
|
115
|
+
So `get_listing(max_pages=N)` visits the listing once for its cookies, then calls the search API N times, passing the cursor from each response to the next request (Scrape.do forwards the cookie and cursor headers with `extraHeaders=true`). In our test, four API pages returned 200 distinct products.
|
|
116
|
+
|
|
117
|
+
**Listing price vs size price.** A listing card has one price. On the product page every size lists its sellers, each with its own price and stock count. On a Levi's T-shirt listed at ₹389, sizes M and XL had a seller at ₹389 while sizes S and L were only available from a second seller at ₹402. The `sizes` rows keep that split.
|
|
118
|
+
|
|
119
|
+
## Limits
|
|
120
|
+
|
|
121
|
+
- **One location.** Prices and stock are what Myntra shows without a delivery pincode.
|
|
122
|
+
- **Credits.** `max_pages=1` reads the listing HTML (1 credit). More pages cost one page visit plus one credit per API page.
|
|
123
|
+
- **Errors.** Non-200 responses raise `ScrapeError` with the status code. A 200 without Myntra's data (an Akamai page) is treated as a failure. Server errors, 429s and empty pages are retried twice; client errors are not.
|
|
124
|
+
|
|
125
|
+
## Responsible use
|
|
126
|
+
|
|
127
|
+
The scraper reads listing, product and search pages, which Myntra's robots.txt allows (`Allow: /`); it does not touch cart, checkout, account or size-chart paths. Product data is not personal data. Read Myntra's terms for your use case and keep request rates modest. This is not legal advice.
|
|
128
|
+
|
|
129
|
+
## Development
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
git clone https://github.com/bugrahansss/myntra-scraper
|
|
133
|
+
cd myntra-scraper
|
|
134
|
+
pip install -e .
|
|
135
|
+
python -m unittest discover -s tests -v
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The tests run offline against fictional fixture pages.
|
|
139
|
+
|
|
140
|
+
## License
|
|
141
|
+
|
|
142
|
+
MIT
|
|
143
|
+
|
|
144
|
+
---
|
|
145
|
+
|
|
146
|
+
Built and tested with [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=myntra-scraper), November 2026.
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# myntra-scraper
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/myntra-scraper/)
|
|
4
|
+
[](https://pypi.org/project/myntra-scraper/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
A Python scraper for [Myntra](https://www.myntra.com), India's largest fashion marketplace. Two jobs:
|
|
8
|
+
|
|
9
|
+
- **Listings and search:** any listing or search URL (`/men-tshirts`, `/levis?rawQuery=levis`, with sort and filters) → name, brand, price, MRP, discount, rating, rating count, sizes, colour, image and URL. **Real pagination**, 50 products per page, sponsored products left out.
|
|
10
|
+
- **Product pages:** price, MRP, rating and attributes, plus **one row per size and seller with that seller's price and the exact units in stock**.
|
|
11
|
+
|
|
12
|
+
Output as Python dicts, JSON or CSV. One dependency (`requests`), no browser.
|
|
13
|
+
|
|
14
|
+
```bash
|
|
15
|
+
pip install myntra-scraper
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
## Why a plain request fails
|
|
19
|
+
|
|
20
|
+
Myntra sits behind Akamai Bot Manager (`_abck`, `bm_sz` cookies). A script doesn't get an error, it gets nothing:
|
|
21
|
+
|
|
22
|
+
| Request | Result |
|
|
23
|
+
| --- | --- |
|
|
24
|
+
| `requests`, default or Chrome user agent | connection opens, no bytes, timeout after 20 s |
|
|
25
|
+
| `curl` (HTTP/2) | `stream was not closed cleanly: INTERNAL_ERROR` |
|
|
26
|
+
| Through Scrape.do, no parameters | 200 in ~3 s, full page, 1 credit |
|
|
27
|
+
|
|
28
|
+
Pages are fetched through the [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=myntra-scraper) API with a plain request: no `render`, no `super`, 1 credit per page. You need a Scrape.do API token; the free plan includes 1,000 credits a month.
|
|
29
|
+
|
|
30
|
+
## Quick start
|
|
31
|
+
|
|
32
|
+
```python
|
|
33
|
+
from myntra_scraper import MyntraScraper
|
|
34
|
+
|
|
35
|
+
scraper = MyntraScraper(token="YOUR_SCRAPEDO_TOKEN") # or set SCRAPEDO_TOKEN
|
|
36
|
+
|
|
37
|
+
# 3 pages of 50 = 150 products (4 credits: one page visit + three API pages)
|
|
38
|
+
rows = scraper.get_listing("https://www.myntra.com/men-tshirts?sort=new", max_pages=3)
|
|
39
|
+
for row in rows[:3]:
|
|
40
|
+
print(row["brand"], row["name"], row["price_inr"], row["mrp_inr"], row["discount_pct"])
|
|
41
|
+
|
|
42
|
+
product = scraper.get_product(rows[0]["url"])
|
|
43
|
+
print(product["units_in_stock"], product["sizes_in_stock"], "/", product["sizes_total"], "sizes in stock")
|
|
44
|
+
for size in product["sizes"]:
|
|
45
|
+
print(size["size"], size["seller"], size["price_inr"], size["units"])
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
### Command line
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
export SCRAPEDO_TOKEN=your_token
|
|
52
|
+
|
|
53
|
+
# 3 pages of a listing, CSV
|
|
54
|
+
myntra-scraper listing "https://www.myntra.com/men-tshirts?sort=new" --pages 3 -f csv -o myntra-listing.csv
|
|
55
|
+
|
|
56
|
+
# product summaries for the first 10 products from that file
|
|
57
|
+
myntra-scraper product -i myntra-listing.csv --limit 10 -f csv -o myntra-products.csv
|
|
58
|
+
|
|
59
|
+
# one row per size and seller
|
|
60
|
+
myntra-scraper product -i myntra-listing.csv --limit 10 --sizes -f csv -o myntra-sizes.csv
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`product` reads the `url` column of the listing CSV, so the commands chain.
|
|
64
|
+
|
|
65
|
+
### Output
|
|
66
|
+
|
|
67
|
+
Size rows (from the fictional test page in [`tests/fixtures`](tests/fixtures/product_page.html)):
|
|
68
|
+
|
|
69
|
+
```json
|
|
70
|
+
[
|
|
71
|
+
{"product_id": 90000001, "size": "M", "sku_id": 900000012, "available": true,
|
|
72
|
+
"seller": "Example Retail Two", "seller_id": 202, "price_inr": 449, "mrp_inr": 999, "units": 5},
|
|
73
|
+
{"product_id": 90000001, "size": "M", "sku_id": 900000012, "available": true,
|
|
74
|
+
"seller": "Example Retail One", "seller_id": 101, "price_inr": 469, "mrp_inr": 999, "units": 30}
|
|
75
|
+
]
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
Listing columns: `product_id, name, brand, price_inr, mrp_inr, discount_pct, rating, rating_count, sizes, gender, article_type, colour, image, url`.
|
|
79
|
+
|
|
80
|
+
Product columns: `product_id, name, brand, price_inr, mrp_inr, discount_pct, rating, rating_count, colour, article_type, gender, country_of_origin, sizes_in_stock, sizes_total, units_in_stock, lowest_size_price_inr, highest_size_price_inr, sellers, url` (plus `attributes` and `sizes` in Python/JSON).
|
|
81
|
+
|
|
82
|
+
## How it works
|
|
83
|
+
|
|
84
|
+
**The data is one JSON object.** Myntra renders every page on the server and puts the data in an inline script, `window.__myx = {...}`. Listings keep their products in `searchData.results.products`, product pages keep everything in `pdpData`. The scraper finds the assignment and reads the object with `json.JSONDecoder.raw_decode`. No CSS selectors.
|
|
85
|
+
|
|
86
|
+
**Sponsored products are separate.** A listing page carries the organic results in `products` and the ads in `plaProducts` (18 of them on a men's T-shirts page). Only the organic products are returned.
|
|
87
|
+
|
|
88
|
+
**`?p=2` is ignored.** Ask the HTML for page 2, 3 or 50 and you get the first 50 products again. In our test, page 2 of a listing sorted by newest shared 49 of 50 products with page 1. The site pages through its search API, `/gateway/v4/search/<slug>`, which:
|
|
89
|
+
|
|
90
|
+
1. returns 401 without the cookies a page visit sets (`at`, `_abck`, `bm_sz` and others), and
|
|
91
|
+
2. ignores `p` and `o` as well, unless the request carries a `pagination-context` header. Every API response returns a new `pagination-context` header; sending it back moves to the next 50.
|
|
92
|
+
|
|
93
|
+
So `get_listing(max_pages=N)` visits the listing once for its cookies, then calls the search API N times, passing the cursor from each response to the next request (Scrape.do forwards the cookie and cursor headers with `extraHeaders=true`). In our test, four API pages returned 200 distinct products.
|
|
94
|
+
|
|
95
|
+
**Listing price vs size price.** A listing card has one price. On the product page every size lists its sellers, each with its own price and stock count. On a Levi's T-shirt listed at ₹389, sizes M and XL had a seller at ₹389 while sizes S and L were only available from a second seller at ₹402. The `sizes` rows keep that split.
|
|
96
|
+
|
|
97
|
+
## Limits
|
|
98
|
+
|
|
99
|
+
- **One location.** Prices and stock are what Myntra shows without a delivery pincode.
|
|
100
|
+
- **Credits.** `max_pages=1` reads the listing HTML (1 credit). More pages cost one page visit plus one credit per API page.
|
|
101
|
+
- **Errors.** Non-200 responses raise `ScrapeError` with the status code. A 200 without Myntra's data (an Akamai page) is treated as a failure. Server errors, 429s and empty pages are retried twice; client errors are not.
|
|
102
|
+
|
|
103
|
+
## Responsible use
|
|
104
|
+
|
|
105
|
+
The scraper reads listing, product and search pages, which Myntra's robots.txt allows (`Allow: /`); it does not touch cart, checkout, account or size-chart paths. Product data is not personal data. Read Myntra's terms for your use case and keep request rates modest. This is not legal advice.
|
|
106
|
+
|
|
107
|
+
## Development
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
git clone https://github.com/bugrahansss/myntra-scraper
|
|
111
|
+
cd myntra-scraper
|
|
112
|
+
pip install -e .
|
|
113
|
+
python -m unittest discover -s tests -v
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
The tests run offline against fictional fixture pages.
|
|
117
|
+
|
|
118
|
+
## License
|
|
119
|
+
|
|
120
|
+
MIT
|
|
121
|
+
|
|
122
|
+
---
|
|
123
|
+
|
|
124
|
+
Built and tested with [Scrape.do](https://scrape.do/?utm_source=github&utm_medium=readme&utm_campaign=myntra-scraper), November 2026.
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Read a few pages of a Myntra listing, then stock and price per size for the first products.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
export SCRAPEDO_TOKEN=your_token
|
|
5
|
+
python listing_to_sizes.py "https://www.myntra.com/men-tshirts?sort=new" 3 10
|
|
6
|
+
|
|
7
|
+
Writes myntra-listing.csv (price, MRP, discount, rating per product) and
|
|
8
|
+
myntra-sizes.csv (one row per size and seller: price, MRP, units in stock).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import csv
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
from myntra_scraper import LISTING_FIELDS, SIZE_FIELDS, MyntraScraper, ScrapeError
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def write(path, fields, rows):
|
|
18
|
+
with open(path, "w", newline="", encoding="utf-8") as handle:
|
|
19
|
+
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
|
|
20
|
+
writer.writeheader()
|
|
21
|
+
writer.writerows(rows)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def main(listing_url: str, pages: int = 3, max_products: int = 10) -> None:
|
|
25
|
+
scraper = MyntraScraper() # reads SCRAPEDO_TOKEN
|
|
26
|
+
listing = scraper.get_listing(listing_url, max_pages=pages)
|
|
27
|
+
write("myntra-listing.csv", LISTING_FIELDS, listing)
|
|
28
|
+
print(f"Saved {len(listing)} products to myntra-listing.csv")
|
|
29
|
+
|
|
30
|
+
sizes = []
|
|
31
|
+
for row in listing[:max_products]:
|
|
32
|
+
try:
|
|
33
|
+
product = scraper.get_product(row["url"])
|
|
34
|
+
except ScrapeError as exc:
|
|
35
|
+
print(f"skipped {row['url']}: {exc}")
|
|
36
|
+
continue
|
|
37
|
+
if not product:
|
|
38
|
+
continue
|
|
39
|
+
sizes.extend(product["sizes"])
|
|
40
|
+
print(
|
|
41
|
+
f"{product['name'][:50]} | listed Rs {row['price_inr']} | sizes Rs "
|
|
42
|
+
f"{product['lowest_size_price_inr']}-{product['highest_size_price_inr']} | "
|
|
43
|
+
f"{product['units_in_stock']} units, {product['sizes_in_stock']}/{product['sizes_total']} sizes in stock"
|
|
44
|
+
)
|
|
45
|
+
write("myntra-sizes.csv", SIZE_FIELDS, sizes)
|
|
46
|
+
print(f"Saved {len(sizes)} size rows to myntra-sizes.csv")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
if __name__ == "__main__":
|
|
50
|
+
main(
|
|
51
|
+
sys.argv[1],
|
|
52
|
+
int(sys.argv[2]) if len(sys.argv) > 2 else 3,
|
|
53
|
+
int(sys.argv[3]) if len(sys.argv) > 3 else 10,
|
|
54
|
+
)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "myntra-scraper"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Scrape Myntra listings (price, MRP, discount, rating) with real pagination, and product pages (price and stock per size and seller) with Python."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
license-files = ["LICENSE"]
|
|
13
|
+
authors = [{ name = "Buğrahan Saka" }]
|
|
14
|
+
keywords = ["myntra", "web scraping", "fashion", "ecommerce", "scraper", "akamai", "india"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 4 - Beta",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
21
|
+
]
|
|
22
|
+
dependencies = ["requests>=2.25"]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/bugrahansss/myntra-scraper"
|
|
26
|
+
Source = "https://github.com/bugrahansss/myntra-scraper"
|
|
27
|
+
Issues = "https://github.com/bugrahansss/myntra-scraper/issues"
|
|
28
|
+
"Scrape.do" = "https://scrape.do/?utm_source=pypi&utm_medium=package&utm_campaign=myntra-scraper"
|
|
29
|
+
|
|
30
|
+
[project.scripts]
|
|
31
|
+
myntra-scraper = "myntra_scraper.cli:main"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
where = ["src"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Scrape Myntra listings and product pages (price, MRP, sizes, stock per seller) with Python."""
|
|
2
|
+
|
|
3
|
+
from .client import MyntraScraper, ScrapeError
|
|
4
|
+
from .parser import (
|
|
5
|
+
LISTING_FIELDS,
|
|
6
|
+
PRODUCT_FIELDS,
|
|
7
|
+
SIZE_FIELDS,
|
|
8
|
+
extract_myx,
|
|
9
|
+
listing_info,
|
|
10
|
+
parse_listing,
|
|
11
|
+
parse_product,
|
|
12
|
+
parse_products,
|
|
13
|
+
parse_sizes,
|
|
14
|
+
product_url,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
__all__ = [
|
|
18
|
+
"LISTING_FIELDS",
|
|
19
|
+
"MyntraScraper",
|
|
20
|
+
"PRODUCT_FIELDS",
|
|
21
|
+
"SIZE_FIELDS",
|
|
22
|
+
"ScrapeError",
|
|
23
|
+
"extract_myx",
|
|
24
|
+
"listing_info",
|
|
25
|
+
"parse_listing",
|
|
26
|
+
"parse_product",
|
|
27
|
+
"parse_products",
|
|
28
|
+
"parse_sizes",
|
|
29
|
+
"product_url",
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"""Command line entry point: ``myntra-scraper listing URL`` / ``myntra-scraper product URL ...``."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import csv
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
from typing import List, Optional
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .client import TOKEN_ENV, MyntraScraper, ScrapeError
|
|
13
|
+
from .parser import LISTING_FIELDS, PRODUCT_FIELDS, SIZE_FIELDS
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
17
|
+
parser = argparse.ArgumentParser(
|
|
18
|
+
prog="myntra-scraper",
|
|
19
|
+
description="Scrape Myntra listings and product pages (price, MRP, sizes, stock per seller) into JSON or CSV.",
|
|
20
|
+
)
|
|
21
|
+
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
|
|
22
|
+
sub = parser.add_subparsers(dest="command", required=True)
|
|
23
|
+
|
|
24
|
+
lst = sub.add_parser("listing", help="Products of a listing or search page, 50 per page")
|
|
25
|
+
lst.add_argument("url", help="A Myntra listing or search URL, e.g. https://www.myntra.com/men-tshirts?sort=new")
|
|
26
|
+
lst.add_argument("--pages", type=int, default=1, help="Pages of 50 to read (default 1)")
|
|
27
|
+
|
|
28
|
+
prod = sub.add_parser("product", help="Product pages: summary, or one row per size and seller with --sizes")
|
|
29
|
+
prod.add_argument("urls", nargs="*", help="Myntra product URLs or style ids")
|
|
30
|
+
prod.add_argument("-i", "--input", help="Text file with one URL per line, or a CSV with a 'url' column")
|
|
31
|
+
prod.add_argument("--limit", type=int, default=0, help="Stop after this many products (0 = all)")
|
|
32
|
+
prod.add_argument("--sizes", action="store_true", help="Output one row per size and seller")
|
|
33
|
+
|
|
34
|
+
for p in (lst, prod):
|
|
35
|
+
p.add_argument("-t", "--token", help=f"Scrape.do API token (default: ${TOKEN_ENV})")
|
|
36
|
+
p.add_argument("-f", "--format", choices=("json", "csv"), default="json")
|
|
37
|
+
p.add_argument("-o", "--output", help="Write to this file instead of stdout")
|
|
38
|
+
return parser
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _read_urls(args: argparse.Namespace) -> List[str]:
|
|
42
|
+
urls = list(args.urls)
|
|
43
|
+
if args.input:
|
|
44
|
+
with open(args.input, encoding="utf-8", newline="") as handle:
|
|
45
|
+
if args.input.endswith(".csv"):
|
|
46
|
+
urls.extend(row["url"] for row in csv.DictReader(handle) if row.get("url"))
|
|
47
|
+
else:
|
|
48
|
+
urls.extend(line.strip() for line in handle if line.strip() and not line.startswith("#"))
|
|
49
|
+
return urls[: args.limit] if args.limit else urls
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _write(rows: List[dict], fields: List[str], fmt: str, stream) -> None:
|
|
53
|
+
if fmt == "csv":
|
|
54
|
+
writer = csv.DictWriter(stream, fieldnames=fields, extrasaction="ignore")
|
|
55
|
+
writer.writeheader()
|
|
56
|
+
writer.writerows(rows)
|
|
57
|
+
else:
|
|
58
|
+
json.dump(rows, stream, indent=2, ensure_ascii=False)
|
|
59
|
+
stream.write("\n")
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def main(argv: Optional[List[str]] = None) -> int:
|
|
63
|
+
args = build_parser().parse_args(argv)
|
|
64
|
+
try:
|
|
65
|
+
scraper = MyntraScraper(token=args.token)
|
|
66
|
+
except ValueError as exc:
|
|
67
|
+
print(exc, file=sys.stderr)
|
|
68
|
+
return 2
|
|
69
|
+
|
|
70
|
+
failures = 0
|
|
71
|
+
if args.command == "listing":
|
|
72
|
+
fields = LISTING_FIELDS
|
|
73
|
+
try:
|
|
74
|
+
rows = scraper.get_listing(args.url, max_pages=max(1, args.pages))
|
|
75
|
+
except (ScrapeError, ValueError) as exc:
|
|
76
|
+
print(f"fail {args.url}: {exc}", file=sys.stderr)
|
|
77
|
+
return 1
|
|
78
|
+
print(f"{len(rows)} products", file=sys.stderr)
|
|
79
|
+
else:
|
|
80
|
+
fields = SIZE_FIELDS if args.sizes else PRODUCT_FIELDS
|
|
81
|
+
urls = _read_urls(args)
|
|
82
|
+
if not urls:
|
|
83
|
+
print("No URLs given. Pass product URLs or --input FILE.", file=sys.stderr)
|
|
84
|
+
return 2
|
|
85
|
+
rows = []
|
|
86
|
+
for url in urls:
|
|
87
|
+
try:
|
|
88
|
+
product = scraper.get_product(url)
|
|
89
|
+
except (ScrapeError, ValueError) as exc:
|
|
90
|
+
failures += 1
|
|
91
|
+
print(f"fail {url}: {exc}", file=sys.stderr)
|
|
92
|
+
continue
|
|
93
|
+
if not product:
|
|
94
|
+
failures += 1
|
|
95
|
+
print(f"fail {url}: no product data", file=sys.stderr)
|
|
96
|
+
continue
|
|
97
|
+
rows.extend(product["sizes"] if args.sizes else [product])
|
|
98
|
+
print(f"ok {url}", file=sys.stderr)
|
|
99
|
+
|
|
100
|
+
if args.output:
|
|
101
|
+
with open(args.output, "w", encoding="utf-8", newline="" if args.format == "csv" else None) as handle:
|
|
102
|
+
_write(rows, fields, args.format, handle)
|
|
103
|
+
print(f"Saved {len(rows)} rows to {args.output}", file=sys.stderr)
|
|
104
|
+
else:
|
|
105
|
+
_write(rows, fields, args.format, sys.stdout)
|
|
106
|
+
return 1 if failures and not rows else 0
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
if __name__ == "__main__":
|
|
110
|
+
raise SystemExit(main())
|