tokopaedi-async 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokopaedi_async-0.1.0/LICENSE +21 -0
- tokopaedi_async-0.1.0/PKG-INFO +278 -0
- tokopaedi_async-0.1.0/README.md +248 -0
- tokopaedi_async-0.1.0/pyproject.toml +36 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/__init__.py +54 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/custom_logging.py +34 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/get_fingerprint.py +160 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/get_product.py +197 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/get_reviews.py +120 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/search.py +161 -0
- tokopaedi_async-0.1.0/src/tokopaedi_async/tokopaedi_types.py +192 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Hilmi Azizi
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tokopaedi-async
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: High-performance Async Python scraper for Tokopedia (Fork of tokopaedi)
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: tokopedia,scraper,ecommerce,product,data,web-scraping,python,data-analysis,tokopedia-reviews,async,asyncio
|
|
8
|
+
Author: pandamoon21
|
|
9
|
+
Author-email: pandamoon21@waifu.club
|
|
10
|
+
Requires-Python: >=3.9,<4.0
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Operating System :: OS Independent
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
22
|
+
Classifier: Topic :: Internet :: WWW/HTTP :: Indexing/Search
|
|
23
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
+
Requires-Dist: curl-cffi (>=0.11.4,<0.12.0)
|
|
25
|
+
Project-URL: Documentation, https://github.com/pandamoon21/tokopaedi-async
|
|
26
|
+
Project-URL: Homepage, https://github.com/pandamoon21/tokopaedi-async
|
|
27
|
+
Project-URL: Repository, https://github.com/pandamoon21/tokopaedi-async
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
# Tokopaedi Async - High-Performance Async Python Scraper for Tokopedia
|
|
32
|
+
 [](https://pepy.tech/projects/tokopaedi-async)  [](https://github.com/pandamoon21/tokopaedi-async/blob/main/LICENSE) 
|
|
33
|
+
|
|
34
|
+
**Extract product data, reviews, and search results from Tokopedia with ease.**
|
|
35
|
+
|
|
36
|
+
tokopaedi-async is a high-performance fork of the original [**Tokopaedi**](https://github.com/hilmiazizi/tokopaedi) library. It leverages Python's asyncio and curl_cffi to perform massive data extraction concurrently, making it significantly faster for bulk operations like enriching product details or fetching thousands of reviews.
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
## Key Features
|
|
42
|
+
|
|
43
|
+
- 🚀 **Asynchronous & Concurrent**: Built on asyncio for non-blocking I/O. Fetch details for more products in the time it takes to fetch one.
|
|
44
|
+
- 🔍 **Product Search**: Search products with advanced filters (price, rating, condition, official store, etc.).
|
|
45
|
+
- 📦 **Detailed Product Data**: Retrieve rich product details, including variants, pricing, stock, and media.
|
|
46
|
+
- 💬 **Reviews Scraper**: Scrape product reviews with ratings, timestamps, and more.
|
|
47
|
+
- 🛡️ **Smart Anti-Detection**: Uses curl_cffi to mimic real browser fingerprints (JA3/TLS) and rotating user agents.
|
|
48
|
+
- 📊 **JSON Export**: Easy serialization to JSON or Pandas DataFrame.
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
## Installation
|
|
52
|
+
Tokopaedi is available on PyPi: [https://pypi.org/project/tokopaedi-async/](https://pypi.org/project/tokopaedi-async/)
|
|
53
|
+
|
|
54
|
+
Install Tokopaedi via pip:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
pip install tokopaedi-async
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Quick Start
|
|
61
|
+
Since this library is asynchronous, you must run it within an `async` function using `asyncio`.
|
|
62
|
+
```python
|
|
63
|
+
import asyncio
|
|
64
|
+
import json
|
|
65
|
+
from tokopaedi import search, SearchFilters, get_product
|
|
66
|
+
|
|
67
|
+
async def main():
|
|
68
|
+
# 1. Setup Filters (Optional)
|
|
69
|
+
filters = SearchFilters(
|
|
70
|
+
bebas_ongkir_extra=True,
|
|
71
|
+
pmin=15000000,
|
|
72
|
+
pmax=25000000,
|
|
73
|
+
rt=4.5
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
print("🔍 Searching...")
|
|
77
|
+
# 'await' is required for search
|
|
78
|
+
results = await search("Asus Zenbook S14 32GB", max_result=100, debug=True, filters=filters)
|
|
79
|
+
|
|
80
|
+
# 2. Parallel Enrichment
|
|
81
|
+
# This runs 20+ concurrent requests to fetch details/reviews almost instantly
|
|
82
|
+
print(f"⚡ Enriching data for {len(results)} products...")
|
|
83
|
+
await results.enrich_details(debug=True)
|
|
84
|
+
await results.enrich_reviews(max_result=50, debug=True)
|
|
85
|
+
|
|
86
|
+
# 3. Save to JSON
|
|
87
|
+
with open('result.json', 'w') as f:
|
|
88
|
+
f.write(json.dumps(results.json(), indent=4))
|
|
89
|
+
print("✅ Done!")
|
|
90
|
+
|
|
91
|
+
if __name__ == "__main__":
|
|
92
|
+
asyncio.run(main())
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## 📘 API Overview
|
|
96
|
+
|
|
97
|
+
### 🔍 `async search(keyword: str, max_result: int = 100, filters: Optional[SearchFilters] = None, debug: bool = False) -> SearchResults`
|
|
98
|
+
|
|
99
|
+
Search for products from Tokopedia.
|
|
100
|
+
|
|
101
|
+
```python
|
|
102
|
+
results = await search(
|
|
103
|
+
keyword="Laptop Gaming",
|
|
104
|
+
max_result=100,
|
|
105
|
+
filters=filters,
|
|
106
|
+
debug=True
|
|
107
|
+
)
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
**Parameters:**
|
|
111
|
+
|
|
112
|
+
- `keyword`: string keyword (e.g., `"logitech mouse"`).
|
|
113
|
+
|
|
114
|
+
- `max_result`: Expected number of results to return.
|
|
115
|
+
|
|
116
|
+
- `filters`: Optional `SearchFilters` instance to narrow search results.
|
|
117
|
+
|
|
118
|
+
- `debug`: Show debug message if True
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
**Returns:**
|
|
122
|
+
|
|
123
|
+
- A `SearchResults` instance (list-like object of `ProductSearchResult`), supporting `.json()` for easy export.
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
----------
|
|
127
|
+
|
|
128
|
+
### 📦 `async get_product(product_id: Optional[Union[int, str]] = None, url: Optional[str] = None, debug: bool = False) -> ProductData`
|
|
129
|
+
|
|
130
|
+
Fetch detailed information for a given Tokopedia product.
|
|
131
|
+
|
|
132
|
+
**Parameters:**
|
|
133
|
+
|
|
134
|
+
- `product_id`: (Optional) The product ID returned from `search()`. If provided, this will take precedence over `url`.
|
|
135
|
+
- `url`: (Optional) The full product URL. Used only if `product_id` is not provided.
|
|
136
|
+
- `debug`: If `True`, prints debug output for troubleshooting.
|
|
137
|
+
|
|
138
|
+
> ⚠️ Either `product_id` or `url` must be provided. If both are given, `product_id` is used and `url` is ignored.
|
|
139
|
+
|
|
140
|
+
**Returns:**
|
|
141
|
+
|
|
142
|
+
- A `ProductData` instance containing detailed information such as product name, pricing, variants, media, stock, rating, etc.
|
|
143
|
+
- Supports `.json()` for easy serialization (e.g., to use with `pandas` or export as `.json`).
|
|
144
|
+
|
|
145
|
+
----------
|
|
146
|
+
|
|
147
|
+
### 🗣️ `async get_reviews(product_id: Optional[Union[int, str]] = None, url: Optional[str] = None, max_count: int = 20, debug: bool = False) -> List[ProductReview]`
|
|
148
|
+
|
|
149
|
+
Scrape customer reviews for a given product.
|
|
150
|
+
|
|
151
|
+
**Parameters:**
|
|
152
|
+
|
|
153
|
+
- `product_id`: (Optional) The product ID to fetch reviews for. Takes precedence over `url` if both are provided.
|
|
154
|
+
- `url`: (Optional) Full product URL. Used only if `product_id` is not provided.
|
|
155
|
+
- `max_count`: Maximum number of reviews to fetch (default: 20).
|
|
156
|
+
- `debug`: Show debug messages if `True`.
|
|
157
|
+
|
|
158
|
+
> ⚠️ Either `product_id` or `url` must be provided.
|
|
159
|
+
|
|
160
|
+
**Returns:**
|
|
161
|
+
|
|
162
|
+
- A list of `ProductReview` objects.
|
|
163
|
+
- Each object supports `.json()` for serialization (e.g., for use with `pandas` or JSON export).
|
|
164
|
+
|
|
165
|
+
**Returns:**
|
|
166
|
+
|
|
167
|
+
- A new `SearchResults` object with `.product_detail` and `.product_reviews` fields filled in (if data was provided).
|
|
168
|
+
|
|
169
|
+
----------
|
|
170
|
+
### `SearchFilters` – Optional Search Filters
|
|
171
|
+
|
|
172
|
+
Use `SearchFilters` to refine your search results. All fields are optional. Pass it into the `search()` function via the `filters` argument.
|
|
173
|
+
|
|
174
|
+
#### Example:
|
|
175
|
+
```python
|
|
176
|
+
from tokopaedi_async import SearchFilters
|
|
177
|
+
|
|
178
|
+
filters = SearchFilters(
|
|
179
|
+
pmin=100000, # Min Price
|
|
180
|
+
pmax=5000000, # Max Price
|
|
181
|
+
condition=1, # 1=New, 2=Used
|
|
182
|
+
shop_tier=2, # 2=Official Store, 3=Power Merchant
|
|
183
|
+
rt=4.5, # Min Rating
|
|
184
|
+
latest_product=30, # Added in last 30 days
|
|
185
|
+
is_discount=True, # Only discounted items
|
|
186
|
+
is_fulfillment=True, # "Dilayani Tokopedia"
|
|
187
|
+
is_plus=True # Tokopedia PLUS
|
|
188
|
+
)
|
|
189
|
+
|
|
190
|
+
results = await search("logitech mouse", filters=filters)
|
|
191
|
+
```
|
|
192
|
+
|
|
193
|
+
#### Available Fields:
|
|
194
|
+
|
|
195
|
+
| Field | Type | Description | Accepted Values |
|
|
196
|
+
|----------------------|----------|---------------------------------------------------|----------------------------------|
|
|
197
|
+
| `pmin` | `int` | Minimum price (in IDR) | e.g., `100000` |
|
|
198
|
+
| `pmax` | `int` | Maximum price (in IDR) | e.g., `1000000` |
|
|
199
|
+
| `condition` | `int` | Product condition | `1` = New, `2` = Used |
|
|
200
|
+
| `shop_tier` | `int` | Type of shop | `2` = Mall, `3` = Power Shop |
|
|
201
|
+
| `rt` | `float` | Minimum rating | e.g., `4.5` |
|
|
202
|
+
| `latest_product` | `int` | Product recency filter | `7`, `30`, `90` |
|
|
203
|
+
| `bebas_ongkir_extra` | `bool` | Filter for extra free shipping | `True` / `False` |
|
|
204
|
+
| `is_discount` | `bool` | Only show discounted products | `True` / `False` |
|
|
205
|
+
| `is_fulfillment` | `bool` | Only Fulfilled by Tokopedia | `True` / `False` |
|
|
206
|
+
| `is_plus` | `bool` | Only Tokopedia PLUS sellers | `True` / `False` |
|
|
207
|
+
| `cod` | `bool` | Cash on delivery available | `True` / `False` |
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
## Product Details & Reviews Enrichment
|
|
211
|
+
|
|
212
|
+
Tokopaedi supports data enrichment to attach detailed product information and customer reviews directly to search results. This is useful when you want to go beyond basic search metadata and analyze full product details or customer feedback.
|
|
213
|
+
#### Example:
|
|
214
|
+
```python
|
|
215
|
+
# Enrich search results
|
|
216
|
+
results = search("Asus Zenbook S14 32GB", max_result=10, debug=True, filters=filters)
|
|
217
|
+
results.enrich_details(debug=True)
|
|
218
|
+
results.enrich_reviews(max_result=50, debug=True)
|
|
219
|
+
|
|
220
|
+
# Enrich product detail with reviews
|
|
221
|
+
product = get_product(url="https://www.tokopedia.com/asusrogindonesia/asus-tuf-a15-fa506ncr-ryzen-7-7435hs-rtx3050-4gb-8gb-512gb-w11-ohs-o365-15-6fhd-144hz-ips-rgb-blk-r735b1t-om-laptop-8gb-512gb-4970d?extParam=whid%3D17186756&aff_unique_id=&channel=others&chain_key=")
|
|
222
|
+
product.enrich_reviews(max_result=50, debug=True)
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Enrichment methods are available on both the `SearchResults` container and individual `ProductData` objects:
|
|
226
|
+
|
|
227
|
+
### On `SearchResults`
|
|
228
|
+
|
|
229
|
+
```python
|
|
230
|
+
# Fetches details for ALL items in the list concurrently
|
|
231
|
+
await results.enrich_details()
|
|
232
|
+
|
|
233
|
+
# Fetches reviews for ALL items in the list concurrently
|
|
234
|
+
await results.enrich_reviews(max_result=20)
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
- `enrich_details(debug: bool = False) -> None`
|
|
238
|
+
Enriches all items in the result with detailed product info.
|
|
239
|
+
- `debug`: If `True`, logs each enrichment step.
|
|
240
|
+
|
|
241
|
+
- `enrich_reviews(max_result: int = 10, debug: bool = False) -> None`
|
|
242
|
+
Enriches all items with customer reviews (up to `max_result` per product).
|
|
243
|
+
- `max_result`: Number of reviews to fetch for each product.
|
|
244
|
+
- `debug`: If `True`, logs the review enrichment process.
|
|
245
|
+
|
|
246
|
+
### On `ProductData`
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
await product.enrich_details()
|
|
250
|
+
await product.enrich_reviews()
|
|
251
|
+
```
|
|
252
|
+
|
|
253
|
+
- `enrich_details(debug: bool = False) -> None`
|
|
254
|
+
Enriches this specific product with detailed information.
|
|
255
|
+
|
|
256
|
+
- `enrich_reviews(max_result: int = 10, debug: bool = False) -> None`
|
|
257
|
+
Enriches this product with customer reviews.
|
|
258
|
+
|
|
259
|
+
This design allows for flexibility: enrich a full result set at once, or enrich individual items selectively as needed.
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
## 📋 Changelog
|
|
263
|
+
|
|
264
|
+
### 0.1.0
|
|
265
|
+
- Initial release based on [tokopaedi 0.2.3](https://github.com/hilmiazizi/tokopaedi/tree/3d74be192ff9c4e278df24eb930329a1117c68ea)
|
|
266
|
+
|
|
267
|
+
## Disclaimer
|
|
268
|
+
|
|
269
|
+
This is an unofficial library and is not affiliated with, endorsed, or supported by Tokopedia. It is intended for educational and research purposes only. Please use responsibly and respect the website's Terms of Service.
|
|
270
|
+
|
|
271
|
+
## Credits & License
|
|
272
|
+
|
|
273
|
+
Original library by [**Hilmi Azizi**](https://hilmiazizi.com) - Thanks :)
|
|
274
|
+
|
|
275
|
+
Async fork maintained by [**pandamoon21**](https://github.com/pandamoon21).
|
|
276
|
+
|
|
277
|
+
Distributed under the MIT License. See [LICENSE](https://github.com/pandamoon21/tokopaedi-async/blob/main/LICENSE) for more information.
|
|
278
|
+
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
|
|
2
|
+
# Tokopaedi Async - High-Performance Async Python Scraper for Tokopedia
|
|
3
|
+
 [](https://pepy.tech/projects/tokopaedi-async)  [](https://github.com/pandamoon21/tokopaedi-async/blob/main/LICENSE) 
|
|
4
|
+
|
|
5
|
+
**Extract product data, reviews, and search results from Tokopedia with ease.**
|
|
6
|
+
|
|
7
|
+
tokopaedi-async is a high-performance fork of the original [**Tokopaedi**](https://github.com/hilmiazizi/tokopaedi) library. It leverages Python's asyncio and curl_cffi to perform massive data extraction concurrently, making it significantly faster for bulk operations like enriching product details or fetching thousands of reviews.
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
## Key Features
|
|
13
|
+
|
|
14
|
+
- 🚀 **Asynchronous & Concurrent**: Built on asyncio for non-blocking I/O. Fetch details for more products in the time it takes to fetch one.
|
|
15
|
+
- 🔍 **Product Search**: Search products with advanced filters (price, rating, condition, official store, etc.).
|
|
16
|
+
- 📦 **Detailed Product Data**: Retrieve rich product details, including variants, pricing, stock, and media.
|
|
17
|
+
- 💬 **Reviews Scraper**: Scrape product reviews with ratings, timestamps, and more.
|
|
18
|
+
- 🛡️ **Smart Anti-Detection**: Uses curl_cffi to mimic real browser fingerprints (JA3/TLS) and rotating user agents.
|
|
19
|
+
- 📊 **JSON Export**: Easy serialization to JSON or Pandas DataFrame.
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
## Installation
|
|
23
|
+
Tokopaedi is available on PyPi: [https://pypi.org/project/tokopaedi-async/](https://pypi.org/project/tokopaedi-async/)
|
|
24
|
+
|
|
25
|
+
Install Tokopaedi via pip:
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
pip install tokopaedi-async
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## Quick Start
|
|
32
|
+
Since this library is asynchronous, you must run it within an `async` function using `asyncio`.
|
|
33
|
+
```python
|
|
34
|
+
import asyncio
|
|
35
|
+
import json
|
|
36
|
+
from tokopaedi import search, SearchFilters, get_product
|
|
37
|
+
|
|
38
|
+
async def main():
|
|
39
|
+
# 1. Setup Filters (Optional)
|
|
40
|
+
filters = SearchFilters(
|
|
41
|
+
bebas_ongkir_extra=True,
|
|
42
|
+
pmin=15000000,
|
|
43
|
+
pmax=25000000,
|
|
44
|
+
rt=4.5
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
print("🔍 Searching...")
|
|
48
|
+
# 'await' is required for search
|
|
49
|
+
results = await search("Asus Zenbook S14 32GB", max_result=100, debug=True, filters=filters)
|
|
50
|
+
|
|
51
|
+
# 2. Parallel Enrichment
|
|
52
|
+
# This runs 20+ concurrent requests to fetch details/reviews almost instantly
|
|
53
|
+
print(f"⚡ Enriching data for {len(results)} products...")
|
|
54
|
+
await results.enrich_details(debug=True)
|
|
55
|
+
await results.enrich_reviews(max_result=50, debug=True)
|
|
56
|
+
|
|
57
|
+
# 3. Save to JSON
|
|
58
|
+
with open('result.json', 'w') as f:
|
|
59
|
+
f.write(json.dumps(results.json(), indent=4))
|
|
60
|
+
print("✅ Done!")
|
|
61
|
+
|
|
62
|
+
if __name__ == "__main__":
|
|
63
|
+
asyncio.run(main())
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## 📘 API Overview
|
|
67
|
+
|
|
68
|
+
### 🔍 `async search(keyword: str, max_result: int = 100, filters: Optional[SearchFilters] = None, debug: bool = False) -> SearchResults`
|
|
69
|
+
|
|
70
|
+
Search for products from Tokopedia.
|
|
71
|
+
|
|
72
|
+
```python
|
|
73
|
+
results = await search(
|
|
74
|
+
keyword="Laptop Gaming",
|
|
75
|
+
max_result=100,
|
|
76
|
+
filters=filters,
|
|
77
|
+
debug=True
|
|
78
|
+
)
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
**Parameters:**
|
|
82
|
+
|
|
83
|
+
- `keyword`: string keyword (e.g., `"logitech mouse"`).
|
|
84
|
+
|
|
85
|
+
- `max_result`: Expected number of results to return.
|
|
86
|
+
|
|
87
|
+
- `filters`: Optional `SearchFilters` instance to narrow search results.
|
|
88
|
+
|
|
89
|
+
- `debug`: Show debug message if True
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
**Returns:**
|
|
93
|
+
|
|
94
|
+
- A `SearchResults` instance (list-like object of `ProductSearchResult`), supporting `.json()` for easy export.
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
----------
|
|
98
|
+
|
|
99
|
+
### 📦 `async get_product(product_id: Optional[Union[int, str]] = None, url: Optional[str] = None, debug: bool = False) -> ProductData`
|
|
100
|
+
|
|
101
|
+
Fetch detailed information for a given Tokopedia product.
|
|
102
|
+
|
|
103
|
+
**Parameters:**
|
|
104
|
+
|
|
105
|
+
- `product_id`: (Optional) The product ID returned from `search()`. If provided, this will take precedence over `url`.
|
|
106
|
+
- `url`: (Optional) The full product URL. Used only if `product_id` is not provided.
|
|
107
|
+
- `debug`: If `True`, prints debug output for troubleshooting.
|
|
108
|
+
|
|
109
|
+
> ⚠️ Either `product_id` or `url` must be provided. If both are given, `product_id` is used and `url` is ignored.
|
|
110
|
+
|
|
111
|
+
**Returns:**
|
|
112
|
+
|
|
113
|
+
- A `ProductData` instance containing detailed information such as product name, pricing, variants, media, stock, rating, etc.
|
|
114
|
+
- Supports `.json()` for easy serialization (e.g., to use with `pandas` or export as `.json`).
|
|
115
|
+
|
|
116
|
+
----------
|
|
117
|
+
|
|
118
|
+
### 🗣️ `async get_reviews(product_id: Optional[Union[int, str]] = None, url: Optional[str] = None, max_count: int = 20, debug: bool = False) -> List[ProductReview]`
|
|
119
|
+
|
|
120
|
+
Scrape customer reviews for a given product.
|
|
121
|
+
|
|
122
|
+
**Parameters:**
|
|
123
|
+
|
|
124
|
+
- `product_id`: (Optional) The product ID to fetch reviews for. Takes precedence over `url` if both are provided.
|
|
125
|
+
- `url`: (Optional) Full product URL. Used only if `product_id` is not provided.
|
|
126
|
+
- `max_count`: Maximum number of reviews to fetch (default: 20).
|
|
127
|
+
- `debug`: Show debug messages if `True`.
|
|
128
|
+
|
|
129
|
+
> ⚠️ Either `product_id` or `url` must be provided.
|
|
130
|
+
|
|
131
|
+
**Returns:**
|
|
132
|
+
|
|
133
|
+
- A list of `ProductReview` objects.
|
|
134
|
+
- Each object supports `.json()` for serialization (e.g., for use with `pandas` or JSON export).
|
|
135
|
+
|
|
136
|
+
**Returns:**
|
|
137
|
+
|
|
138
|
+
- A new `SearchResults` object with `.product_detail` and `.product_reviews` fields filled in (if data was provided).
|
|
139
|
+
|
|
140
|
+
----------
|
|
141
|
+
### `SearchFilters` – Optional Search Filters
|
|
142
|
+
|
|
143
|
+
Use `SearchFilters` to refine your search results. All fields are optional. Pass it into the `search()` function via the `filters` argument.
|
|
144
|
+
|
|
145
|
+
#### Example:
|
|
146
|
+
```python
|
|
147
|
+
from tokopaedi_async import SearchFilters
|
|
148
|
+
|
|
149
|
+
filters = SearchFilters(
|
|
150
|
+
pmin=100000, # Min Price
|
|
151
|
+
pmax=5000000, # Max Price
|
|
152
|
+
condition=1, # 1=New, 2=Used
|
|
153
|
+
shop_tier=2, # 2=Official Store, 3=Power Merchant
|
|
154
|
+
rt=4.5, # Min Rating
|
|
155
|
+
latest_product=30, # Added in last 30 days
|
|
156
|
+
is_discount=True, # Only discounted items
|
|
157
|
+
is_fulfillment=True, # "Dilayani Tokopedia"
|
|
158
|
+
is_plus=True # Tokopedia PLUS
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
results = await search("logitech mouse", filters=filters)
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
#### Available Fields:
|
|
165
|
+
|
|
166
|
+
| Field | Type | Description | Accepted Values |
|
|
167
|
+
|----------------------|----------|---------------------------------------------------|----------------------------------|
|
|
168
|
+
| `pmin` | `int` | Minimum price (in IDR) | e.g., `100000` |
|
|
169
|
+
| `pmax` | `int` | Maximum price (in IDR) | e.g., `1000000` |
|
|
170
|
+
| `condition` | `int` | Product condition | `1` = New, `2` = Used |
|
|
171
|
+
| `shop_tier` | `int` | Type of shop | `2` = Mall, `3` = Power Shop |
|
|
172
|
+
| `rt` | `float` | Minimum rating | e.g., `4.5` |
|
|
173
|
+
| `latest_product` | `int` | Product recency filter | `7`, `30`, `90` |
|
|
174
|
+
| `bebas_ongkir_extra` | `bool` | Filter for extra free shipping | `True` / `False` |
|
|
175
|
+
| `is_discount` | `bool` | Only show discounted products | `True` / `False` |
|
|
176
|
+
| `is_fulfillment` | `bool` | Only Fulfilled by Tokopedia | `True` / `False` |
|
|
177
|
+
| `is_plus` | `bool` | Only Tokopedia PLUS sellers | `True` / `False` |
|
|
178
|
+
| `cod` | `bool` | Cash on delivery available | `True` / `False` |
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
## Product Details & Reviews Enrichment
|
|
182
|
+
|
|
183
|
+
Tokopaedi supports data enrichment to attach detailed product information and customer reviews directly to search results. This is useful when you want to go beyond basic search metadata and analyze full product details or customer feedback.
|
|
184
|
+
#### Example:
|
|
185
|
+
```python
|
|
186
|
+
# Enrich search results
|
|
187
|
+
results = search("Asus Zenbook S14 32GB", max_result=10, debug=True, filters=filters)
|
|
188
|
+
results.enrich_details(debug=True)
|
|
189
|
+
results.enrich_reviews(max_result=50, debug=True)
|
|
190
|
+
|
|
191
|
+
# Enrich product detail with reviews
|
|
192
|
+
product = get_product(url="https://www.tokopedia.com/asusrogindonesia/asus-tuf-a15-fa506ncr-ryzen-7-7435hs-rtx3050-4gb-8gb-512gb-w11-ohs-o365-15-6fhd-144hz-ips-rgb-blk-r735b1t-om-laptop-8gb-512gb-4970d?extParam=whid%3D17186756&aff_unique_id=&channel=others&chain_key=")
|
|
193
|
+
product.enrich_reviews(max_result=50, debug=True)
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
Enrichment methods are available on both the `SearchResults` container and individual `ProductData` objects:
|
|
197
|
+
|
|
198
|
+
### On `SearchResults`
|
|
199
|
+
|
|
200
|
+
```python
|
|
201
|
+
# Fetches details for ALL items in the list concurrently
|
|
202
|
+
await results.enrich_details()
|
|
203
|
+
|
|
204
|
+
# Fetches reviews for ALL items in the list concurrently
|
|
205
|
+
await results.enrich_reviews(max_result=20)
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
- `enrich_details(debug: bool = False) -> None`
|
|
209
|
+
Enriches all items in the result with detailed product info.
|
|
210
|
+
- `debug`: If `True`, logs each enrichment step.
|
|
211
|
+
|
|
212
|
+
- `enrich_reviews(max_result: int = 10, debug: bool = False) -> None`
|
|
213
|
+
Enriches all items with customer reviews (up to `max_result` per product).
|
|
214
|
+
- `max_result`: Number of reviews to fetch for each product.
|
|
215
|
+
- `debug`: If `True`, logs the review enrichment process.
|
|
216
|
+
|
|
217
|
+
### On `ProductData`
|
|
218
|
+
|
|
219
|
+
```python
|
|
220
|
+
await product.enrich_details()
|
|
221
|
+
await product.enrich_reviews()
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
- `enrich_details(debug: bool = False) -> None`
|
|
225
|
+
Enriches this specific product with detailed information.
|
|
226
|
+
|
|
227
|
+
- `enrich_reviews(max_result: int = 10, debug: bool = False) -> None`
|
|
228
|
+
Enriches this product with customer reviews.
|
|
229
|
+
|
|
230
|
+
This design allows for flexibility: enrich a full result set at once, or enrich individual items selectively as needed.
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
## 📋 Changelog
|
|
234
|
+
|
|
235
|
+
### 0.1.0
|
|
236
|
+
- Initial release based on [tokopaedi 0.2.3](https://github.com/hilmiazizi/tokopaedi/tree/3d74be192ff9c4e278df24eb930329a1117c68ea)
|
|
237
|
+
|
|
238
|
+
## Disclaimer
|
|
239
|
+
|
|
240
|
+
This is an unofficial library and is not affiliated with, endorsed, or supported by Tokopedia. It is intended for educational and research purposes only. Please use responsibly and respect the website's Terms of Service.
|
|
241
|
+
|
|
242
|
+
## Credits & License
|
|
243
|
+
|
|
244
|
+
Original library by [**Hilmi Azizi**](https://hilmiazizi.com) - Thanks :)
|
|
245
|
+
|
|
246
|
+
Async fork maintained by [**pandamoon21**](https://github.com/pandamoon21).
|
|
247
|
+
|
|
248
|
+
Distributed under the MIT License. See [LICENSE](https://github.com/pandamoon21/tokopaedi-async/blob/main/LICENSE) for more information.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[tool.poetry]
|
|
2
|
+
name = "tokopaedi-async"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "High-performance Async Python scraper for Tokopedia (Fork of tokopaedi)"
|
|
5
|
+
authors = ["pandamoon21 <pandamoon21@waifu.club>"]
|
|
6
|
+
readme = "README.md"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
packages = [{ include = "tokopaedi_async", from = "src" }]
|
|
9
|
+
include = ["LICENSE"]
|
|
10
|
+
|
|
11
|
+
homepage = "https://github.com/pandamoon21/tokopaedi-async"
|
|
12
|
+
repository = "https://github.com/pandamoon21/tokopaedi-async"
|
|
13
|
+
documentation = "https://github.com/pandamoon21/tokopaedi-async"
|
|
14
|
+
keywords = ["tokopedia", "scraper", "ecommerce", "product", "data", "web-scraping", "python", "data-analysis", "tokopedia-reviews", "async", "asyncio"]
|
|
15
|
+
|
|
16
|
+
classifiers = [
|
|
17
|
+
"Development Status :: 4 - Beta",
|
|
18
|
+
"Intended Audience :: Developers",
|
|
19
|
+
"Topic :: Internet :: WWW/HTTP :: Indexing/Search",
|
|
20
|
+
"Topic :: Software Development :: Libraries :: Python Modules",
|
|
21
|
+
"License :: OSI Approved :: MIT License",
|
|
22
|
+
"Programming Language :: Python :: 3",
|
|
23
|
+
"Programming Language :: Python :: 3.9",
|
|
24
|
+
"Programming Language :: Python :: 3.10",
|
|
25
|
+
"Programming Language :: Python :: 3.11",
|
|
26
|
+
"Programming Language :: Python :: 3.12",
|
|
27
|
+
"Operating System :: OS Independent"
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
[tool.poetry.dependencies]
|
|
31
|
+
python = "^3.9"
|
|
32
|
+
curl-cffi = "^0.11.4"
|
|
33
|
+
|
|
34
|
+
[build-system]
|
|
35
|
+
requires = ["poetry-core>=1.1.0"]
|
|
36
|
+
build-backend = "poetry.core.masonry.api"
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
__version__ = '0.1.0'
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
from typing import Optional
|
|
4
|
+
from .search import search
|
|
5
|
+
from .get_product import get_product
|
|
6
|
+
from .get_reviews import get_reviews
|
|
7
|
+
from .tokopaedi_types import ProductData, ProductReview
|
|
8
|
+
|
|
9
|
+
@dataclass
|
|
10
|
+
class SearchFilters:
|
|
11
|
+
# Free shipping benefit (bebas ongkir ekstra)
|
|
12
|
+
# true = only show products with free shipping
|
|
13
|
+
bebas_ongkir_extra: Optional[bool] = None
|
|
14
|
+
|
|
15
|
+
# Discounted products
|
|
16
|
+
# true = only show products with active discounts
|
|
17
|
+
is_discount: Optional[bool] = None
|
|
18
|
+
|
|
19
|
+
# Product condition
|
|
20
|
+
# 1 = New
|
|
21
|
+
# 2 = Used
|
|
22
|
+
condition: Optional[int] = None
|
|
23
|
+
|
|
24
|
+
# Shop tier
|
|
25
|
+
# 2 = Mall
|
|
26
|
+
# 3 = Power Shop
|
|
27
|
+
shop_tier: Optional[int] = None
|
|
28
|
+
|
|
29
|
+
# Minimum price (in IDR)
|
|
30
|
+
pmin: Optional[int] = None
|
|
31
|
+
|
|
32
|
+
# Maximum price (in IDR)
|
|
33
|
+
pmax: Optional[int] = None
|
|
34
|
+
|
|
35
|
+
# Fulfilled by Tokopedia
|
|
36
|
+
# true = only show products fulfilled by Tokopedia
|
|
37
|
+
is_fulfillment: Optional[bool] = None
|
|
38
|
+
|
|
39
|
+
# Tokopedia Plus membership products
|
|
40
|
+
# true = only show products available under Tokopedia Plus
|
|
41
|
+
is_plus: Optional[bool] = None
|
|
42
|
+
|
|
43
|
+
# Cash on Delivery (COD)
|
|
44
|
+
# true = only show products eligible for COD
|
|
45
|
+
cod: Optional[bool] = None
|
|
46
|
+
|
|
47
|
+
# Minimum average rating (0.0 to 5.0)
|
|
48
|
+
rt: Optional[float] = None
|
|
49
|
+
|
|
50
|
+
# Product age in days
|
|
51
|
+
# 7 = added in the last 7 days
|
|
52
|
+
# 30 = added in the last 30 days
|
|
53
|
+
# 90 = added in the last 90 days
|
|
54
|
+
latest_product: Optional[int] = None
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
def setup_custom_logging():
|
|
4
|
+
SEARCH_LEVEL = 25
|
|
5
|
+
DETAIL_LEVEL = 26
|
|
6
|
+
REVIEWS_LEVEL = 27
|
|
7
|
+
|
|
8
|
+
logging.addLevelName(SEARCH_LEVEL, "SEARCH")
|
|
9
|
+
logging.addLevelName(DETAIL_LEVEL, "DETAIL")
|
|
10
|
+
logging.addLevelName(REVIEWS_LEVEL, "REVIEW")
|
|
11
|
+
|
|
12
|
+
class CustomLogger(logging.Logger):
|
|
13
|
+
def search(self, message, *args, **kwargs):
|
|
14
|
+
if self.isEnabledFor(SEARCH_LEVEL):
|
|
15
|
+
self._log(SEARCH_LEVEL, message, args, **kwargs)
|
|
16
|
+
|
|
17
|
+
def detail(self, message, *args, **kwargs):
|
|
18
|
+
if self.isEnabledFor(DETAIL_LEVEL):
|
|
19
|
+
self._log(DETAIL_LEVEL, message, args, **kwargs)
|
|
20
|
+
|
|
21
|
+
def reviews(self, message, *args, **kwargs):
|
|
22
|
+
if self.isEnabledFor(REVIEWS_LEVEL):
|
|
23
|
+
self._log(REVIEWS_LEVEL, message, args, **kwargs)
|
|
24
|
+
|
|
25
|
+
logging.setLoggerClass(CustomLogger)
|
|
26
|
+
|
|
27
|
+
logging.basicConfig(
|
|
28
|
+
level=SEARCH_LEVEL,
|
|
29
|
+
format="{asctime} - {levelname} - {message}",
|
|
30
|
+
style="{",
|
|
31
|
+
datefmt="%Y-%m-%d %H:%M",
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
return logging.getLogger(__name__)
|