search1688api 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- search1688api-1.0.0/PKG-INFO +15 -0
- search1688api-1.0.0/README.md +165 -0
- search1688api-1.0.0/search1688api/__init__.py +9 -0
- search1688api-1.0.0/search1688api/async_session.py +358 -0
- search1688api-1.0.0/search1688api/models.py +243 -0
- search1688api-1.0.0/search1688api/sync_session.py +388 -0
- search1688api-1.0.0/search1688api/utils.py +38 -0
- search1688api-1.0.0/search1688api.egg-info/PKG-INFO +15 -0
- search1688api-1.0.0/search1688api.egg-info/SOURCES.txt +12 -0
- search1688api-1.0.0/search1688api.egg-info/dependency_links.txt +1 -0
- search1688api-1.0.0/search1688api.egg-info/requires.txt +3 -0
- search1688api-1.0.0/search1688api.egg-info/top_level.txt +1 -0
- search1688api-1.0.0/setup.cfg +4 -0
- search1688api-1.0.0/setup.py +16 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: search1688api
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Python library for searching products on 1688.com by image
|
|
5
|
+
Author: netkaruma
|
|
6
|
+
Author-email: suzumekaruma@gmail.com
|
|
7
|
+
Requires-Python: >=3.7
|
|
8
|
+
Requires-Dist: requests>=2.25.0
|
|
9
|
+
Requires-Dist: aiohttp>=3.8.0
|
|
10
|
+
Requires-Dist: yarl>=1.6.0
|
|
11
|
+
Dynamic: author
|
|
12
|
+
Dynamic: author-email
|
|
13
|
+
Dynamic: requires-dist
|
|
14
|
+
Dynamic: requires-python
|
|
15
|
+
Dynamic: summary
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
# 1688 Search API by Image
|
|
2
|
+
|
|
3
|
+
Python библиотека для поиска товаров на 1688.com по изображению.
|
|
4
|
+
А также парсинга данных товаров по ID/Артикулу
|
|
5
|
+
|
|
6
|
+
## Установка
|
|
7
|
+
|
|
8
|
+
```bash
|
|
9
|
+
pip install search1688api
|
|
10
|
+
```
|
|
11
|
+
|
|
12
|
+
## Использование
|
|
13
|
+
|
|
14
|
+
### Синхронная версия
|
|
15
|
+
|
|
16
|
+
```python
|
|
17
|
+
from search1688api import Sync1688Session
|
|
18
|
+
|
|
19
|
+
with Sync1688Session() as session:
|
|
20
|
+
# Поиск по изображению
|
|
21
|
+
products = session.search_by_image("path/to/image.jpg")
|
|
22
|
+
for product in products:
|
|
23
|
+
print(product.name)
|
|
24
|
+
print(product.url)
|
|
25
|
+
print(product.company)
|
|
26
|
+
print(product.price)
|
|
27
|
+
|
|
28
|
+
# Получение детальной информации о товаре по ID
|
|
29
|
+
detailed_product = session.get_by_id("741375431874")
|
|
30
|
+
if detailed_product:
|
|
31
|
+
print(detailed_product.to_dict())
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
### Асинхронная версия
|
|
35
|
+
|
|
36
|
+
```python
|
|
37
|
+
import asyncio
|
|
38
|
+
from search1688api import Async1688Session
|
|
39
|
+
|
|
40
|
+
async def main():
|
|
41
|
+
async with Async1688Session() as session:
|
|
42
|
+
# Поиск по изображению
|
|
43
|
+
products = await session.search_by_image("path/to/image.jpg")
|
|
44
|
+
for product in products:
|
|
45
|
+
print(product.name)
|
|
46
|
+
print(product.url)
|
|
47
|
+
print(product.company)
|
|
48
|
+
print(product.price)
|
|
49
|
+
|
|
50
|
+
# Получение детальной информации о товаре по ID
|
|
51
|
+
detailed_product = await session.get_by_id("741375431874")
|
|
52
|
+
if detailed_product:
|
|
53
|
+
print(detailed_product.to_dict())
|
|
54
|
+
|
|
55
|
+
asyncio.run(main())
|
|
56
|
+
```
|
|
57
|
+
## Настройка таймаута
|
|
58
|
+
### При создании сессии
|
|
59
|
+
|
|
60
|
+
```python
|
|
61
|
+
# Таймаут по умолчанию 30 секунд
|
|
62
|
+
session = Sync1688Session()
|
|
63
|
+
|
|
64
|
+
# Установить кастомный таймаут
|
|
65
|
+
session = Sync1688Session(default_timeout=60) # 60 секунд
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
### Для отдельных запросов
|
|
69
|
+
```python
|
|
70
|
+
# Использовать таймаут по умолчанию
|
|
71
|
+
products = session.search_by_image("image.jpg")
|
|
72
|
+
detailed = session.get_by_id("123456")
|
|
73
|
+
|
|
74
|
+
# Установить таймаут для конкретного запроса
|
|
75
|
+
products = session.search_by_image("image.jpg", timeout=10) # 10 секунд
|
|
76
|
+
detailed = session.get_by_id("123456", timeout=15) # 15 секунд
|
|
77
|
+
```
|
|
78
|
+
### Изменение таймаута после создания
|
|
79
|
+
```python
|
|
80
|
+
session = Sync1688Session()
|
|
81
|
+
session.default_timeout = 45 # Теперь все запросы будут с таймаутом 45 секунд
|
|
82
|
+
```
|
|
83
|
+
## Модель Product
|
|
84
|
+
|
|
85
|
+
## Класс Product содержит следующие атрибуты:
|
|
86
|
+
| Атрибут | Тип | Описание |
|
|
87
|
+
| ------------------- | ------------ | ----------------------------------------------------------------------------------- |
|
|
88
|
+
| `id` | `str` | Уникальный идентификатор товара на 1688 |
|
|
89
|
+
| `name` | `str` | Полное название товара |
|
|
90
|
+
| `simple_name` | `str` | Упрощённое/сокращённое название товара |
|
|
91
|
+
| `brief` | `str` | Краткое описание товара |
|
|
92
|
+
| `company` | `str` | Название компании-продавца |
|
|
93
|
+
| `company_location` | `str` | Регион продавца (провинция и город) |
|
|
94
|
+
| `credit_level` | `str` | Уровень доверия продавца (рейтинг) |
|
|
95
|
+
| `is_factory` | `bool` | Является ли продавец фабрикой (`True`/`False`) |
|
|
96
|
+
| `price` | `str` | Основная цена товара |
|
|
97
|
+
| `quantity_prices` | `List[Dict]` | Список цен в зависимости от количества. Пример: `[{"quantity": 2, "price": "¥10"}]` |
|
|
98
|
+
| `image_url` | `str` | Ссылка на основное изображение товара |
|
|
99
|
+
| `image_url_220x220` | `str` | Ссылка на изображение размером 220x220 |
|
|
100
|
+
| `booked_count` | `int` | Количество забронированных единиц |
|
|
101
|
+
| `sale_quantity` | `int` | Количество проданных единиц |
|
|
102
|
+
| `quantity_begin` | `int` | Минимальное количество заказа |
|
|
103
|
+
| `url` | `str` | Прямая ссылка на страницу товара (`https://detail.1688.com/offer/{id}.html`) |
|
|
104
|
+
| `category_id` | `int` | Идентификатор категории товара |
|
|
105
|
+
| `tags` | `List[str]` | Список идентификаторов тегов (рыночные, членские, праздничные и т. д.) |
|
|
106
|
+
| `service_labels` | `List[str]` | Список активных сервисных меток (например, «Быстрая доставка») |
|
|
107
|
+
| `raw_data` | `Dict` | Исходные данные товара из JSON ответа |
|
|
108
|
+
## Класс DetailProduct (парсинг по ID)
|
|
109
|
+
| Атрибут | Тип | Описание |
|
|
110
|
+
|---------|-----|-----------|
|
|
111
|
+
| `id` | `str` | ID товара |
|
|
112
|
+
| `name` | `str` | Название товара |
|
|
113
|
+
| `brand` | `str` | Бренд товара |
|
|
114
|
+
| `company` | `str` | Название компании |
|
|
115
|
+
| `grade` | `str` | Рейтинг товара |
|
|
116
|
+
| `weight` | `str` | Вес/объем товара |
|
|
117
|
+
| `goods` | `List[Dict]` | Список вариантов товара с ценами и изображениями |
|
|
118
|
+
| `repeat_orders` | `str` | Процент повторных заказов |
|
|
119
|
+
| `saledCount` | `int` | Количество продаж |
|
|
120
|
+
| `raw_data` | `Dict` | Исходные данные товара из JSON ответа |
|
|
121
|
+
|
|
122
|
+
## Методы
|
|
123
|
+
```python
|
|
124
|
+
search_by_image(image_path: str, timeout: float = None) -> List[Product]
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### Параметры:
|
|
128
|
+
1. **image_path** - путь к файлу изображения
|
|
129
|
+
2. **timeout** - таймаут запроса в секундах (по умолчанию используется default_timeout сессии)
|
|
130
|
+
### Возвращает: список объектов Product
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
get_by_id(offer_id: str, timeout: float = None) -> DetailProduct
|
|
134
|
+
```
|
|
135
|
+
### Получение детальной информации о товаре по его ID.
|
|
136
|
+
### Параметры:
|
|
137
|
+
1. **offer_id** - ID товара на 1688.com
|
|
138
|
+
2. **timeout** - таймаут запроса в секундах (по умолчанию используется default_timeout сессии)
|
|
139
|
+
### Возвращает: объект DetailProduct или None если товар не найден
|
|
140
|
+
|
|
141
|
+
## Конвертация в словарь
|
|
142
|
+
```python
|
|
143
|
+
product_dict = product.to_dict()
|
|
144
|
+
detailed_product_dict = detailed_product.to_dict()
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
## Пример конвертации в словарь
|
|
148
|
+
```python
|
|
149
|
+
product_dict = product.to_dict()
|
|
150
|
+
print(product_dict)
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
## Работа с вариантами товара
|
|
154
|
+
```python
|
|
155
|
+
detailed_product = session.get_by_id("741375431874", timeout=30)
|
|
156
|
+
if detailed_product:
|
|
157
|
+
for variant in detailed_product.goods:
|
|
158
|
+
print(f"Вариант: {variant['name']}")
|
|
159
|
+
print(f"Цена: {variant['price']}")
|
|
160
|
+
print(f"Изображение: {variant['imageUrl']}")
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
## Лицензия
|
|
164
|
+
|
|
165
|
+
MIT
|
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
from urllib import response
|
|
3
|
+
import aiohttp
|
|
4
|
+
import json
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
from typing import List, Dict, Any
|
|
8
|
+
from yarl import URL
|
|
9
|
+
|
|
10
|
+
from .models import Product, DetailProduct, extract_products_from_html
|
|
11
|
+
from .utils import prepare_image_request, generate_sign, read_and_encode_image
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Async1688Session:
|
|
15
|
+
def __init__(self, default_timeout: float = 30):
|
|
16
|
+
self._session = None
|
|
17
|
+
self._token = None
|
|
18
|
+
self._token_part = None
|
|
19
|
+
self.app_key = "12574478"
|
|
20
|
+
self.base_url = "https://h5api.m.1688.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/"
|
|
21
|
+
self._initialized = False
|
|
22
|
+
self.default_timeout = default_timeout
|
|
23
|
+
self.cookies_dict = {}
|
|
24
|
+
|
|
25
|
+
async def __aenter__(self):
|
|
26
|
+
if not self._initialized:
|
|
27
|
+
await self.start()
|
|
28
|
+
return self
|
|
29
|
+
|
|
30
|
+
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
|
31
|
+
await self.close()
|
|
32
|
+
|
|
33
|
+
async def _initialize(self):
|
|
34
|
+
if self._initialized:
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
await self.start()
|
|
38
|
+
self._initialized = True
|
|
39
|
+
|
|
40
|
+
async def start(self):
|
|
41
|
+
if self._session and not self._session.closed:
|
|
42
|
+
await self.close()
|
|
43
|
+
|
|
44
|
+
self._session = aiohttp.ClientSession()
|
|
45
|
+
|
|
46
|
+
try:
|
|
47
|
+
test_params = {
|
|
48
|
+
"jsv": "2.7.2",
|
|
49
|
+
"appKey": self.app_key,
|
|
50
|
+
"t": str(int(time.time() * 1000)),
|
|
51
|
+
"api": "mtop.relationrecommend.WirelessRecommend.recommend",
|
|
52
|
+
"v": "2.0",
|
|
53
|
+
"type": "originaljson"
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async with self._session.get(
|
|
57
|
+
self.base_url,
|
|
58
|
+
params=test_params,
|
|
59
|
+
timeout=self.default_timeout
|
|
60
|
+
) as response:
|
|
61
|
+
|
|
62
|
+
url_obj = URL(self.base_url)
|
|
63
|
+
cookies = self._session.cookie_jar.filter_cookies(url_obj)
|
|
64
|
+
token_cookie = cookies.get('_m_h5_tk')
|
|
65
|
+
|
|
66
|
+
if token_cookie:
|
|
67
|
+
self._token = token_cookie.value
|
|
68
|
+
self._token_part = self._token.split('_')[0]
|
|
69
|
+
self._initialized = True
|
|
70
|
+
return True
|
|
71
|
+
else:
|
|
72
|
+
raise Exception("Не удалось получить токен")
|
|
73
|
+
|
|
74
|
+
except Exception as e:
|
|
75
|
+
await self.close()
|
|
76
|
+
raise Exception(f"Ошибка запуска сессии: {e}")
|
|
77
|
+
|
|
78
|
+
async def close(self):
|
|
79
|
+
if self._session and not self._session.closed:
|
|
80
|
+
await self._session.close()
|
|
81
|
+
self._session = None
|
|
82
|
+
self._token = None
|
|
83
|
+
self._token_part = None
|
|
84
|
+
self._initialized = False
|
|
85
|
+
self.cookies_dict = {}
|
|
86
|
+
|
|
87
|
+
async def _ensure_initialized(self):
|
|
88
|
+
if not self._initialized or not self.is_active:
|
|
89
|
+
await self._initialize()
|
|
90
|
+
|
|
91
|
+
async def _get_image_id(self, image_path, timeout=None):
|
|
92
|
+
await self._ensure_initialized()
|
|
93
|
+
|
|
94
|
+
if timeout is None:
|
|
95
|
+
timeout = self.default_timeout
|
|
96
|
+
|
|
97
|
+
image_b64 = read_and_encode_image(image_path)
|
|
98
|
+
data_string = prepare_image_request(image_b64)
|
|
99
|
+
timestamp = str(int(time.time() * 1000))
|
|
100
|
+
sign = generate_sign(self._token_part, timestamp, self.app_key, data_string)
|
|
101
|
+
|
|
102
|
+
params = {
|
|
103
|
+
"jsv": "2.7.2",
|
|
104
|
+
"appKey": self.app_key,
|
|
105
|
+
"t": timestamp,
|
|
106
|
+
"sign": sign,
|
|
107
|
+
"api": "mtop.relationrecommend.WirelessRecommend.recommend",
|
|
108
|
+
"ignoreLogin": "true",
|
|
109
|
+
"prefix": "h5api",
|
|
110
|
+
"v": "2.0",
|
|
111
|
+
"type": "originaljson",
|
|
112
|
+
"dataType": "jsonp",
|
|
113
|
+
"jsonpIncPrefix": "search1688",
|
|
114
|
+
"timeout": "20000"
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
headers = {
|
|
118
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
119
|
+
"origin": "https://s.1688.com",
|
|
120
|
+
"referer": "https://s.1688.com/",
|
|
121
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
try:
|
|
125
|
+
async with self._session.post(
|
|
126
|
+
self.base_url,
|
|
127
|
+
params=params,
|
|
128
|
+
data={"data": data_string},
|
|
129
|
+
headers=headers,
|
|
130
|
+
timeout=timeout
|
|
131
|
+
) as response:
|
|
132
|
+
|
|
133
|
+
if response.status == 200:
|
|
134
|
+
result = await response.json()
|
|
135
|
+
if result.get("data", {}).get("success"):
|
|
136
|
+
image_id = result["data"].get("imageId")
|
|
137
|
+
if image_id:
|
|
138
|
+
return image_id
|
|
139
|
+
return None
|
|
140
|
+
|
|
141
|
+
except Exception as e:
|
|
142
|
+
raise Exception(f"Ошибка при запросе: {e}")
|
|
143
|
+
|
|
144
|
+
async def search_by_image(self, image_path: str, timeout: float = None) -> List[Product]:
|
|
145
|
+
await self._ensure_initialized()
|
|
146
|
+
|
|
147
|
+
if timeout is None:
|
|
148
|
+
timeout = self.default_timeout
|
|
149
|
+
|
|
150
|
+
image_id = await self._get_image_id(image_path, timeout=timeout)
|
|
151
|
+
|
|
152
|
+
if not image_id:
|
|
153
|
+
return []
|
|
154
|
+
|
|
155
|
+
products = await self._search_by_image_id(image_id, timeout=timeout)
|
|
156
|
+
return products
|
|
157
|
+
|
|
158
|
+
async def _search_by_image_id(self, image_id: str, timeout=None) -> List[Product]:
|
|
159
|
+
await self._ensure_initialized()
|
|
160
|
+
|
|
161
|
+
if timeout is None:
|
|
162
|
+
timeout = self.default_timeout
|
|
163
|
+
|
|
164
|
+
search_url = f"https://s.1688.com/youyuan/index.htm"
|
|
165
|
+
params = {
|
|
166
|
+
"tab": "imageSearch",
|
|
167
|
+
"imageId": image_id
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
headers = {
|
|
171
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
172
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
173
|
+
"accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
174
|
+
"referer": "https://s.1688.com/"
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
try:
|
|
178
|
+
async with self._session.get(
|
|
179
|
+
search_url,
|
|
180
|
+
params=params,
|
|
181
|
+
headers=headers,
|
|
182
|
+
timeout=timeout
|
|
183
|
+
) as response:
|
|
184
|
+
|
|
185
|
+
if response.status == 200:
|
|
186
|
+
html_content = await response.text()
|
|
187
|
+
products = extract_products_from_html(html_content)
|
|
188
|
+
return products
|
|
189
|
+
else:
|
|
190
|
+
return []
|
|
191
|
+
|
|
192
|
+
except Exception:
|
|
193
|
+
return []
|
|
194
|
+
|
|
195
|
+
async def _get_cookies_with_aiohttp(self):
|
|
196
|
+
"""Получает куки через aiohttp запросы"""
|
|
197
|
+
temp_session = None
|
|
198
|
+
try:
|
|
199
|
+
# Используем временную сессию для получения кук
|
|
200
|
+
temp_session = aiohttp.ClientSession()
|
|
201
|
+
|
|
202
|
+
headers = {
|
|
203
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
204
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
|
205
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
206
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
207
|
+
"DNT": "1",
|
|
208
|
+
"Connection": "keep-alive",
|
|
209
|
+
"Upgrade-Insecure-Requests": "1",
|
|
210
|
+
"Sec-Fetch-Dest": "document",
|
|
211
|
+
"Sec-Fetch-Mode": "navigate",
|
|
212
|
+
"Sec-Fetch-Site": "none",
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
# Запрос к главной странице 1688
|
|
216
|
+
async with temp_session.get("https://www.1688.com/", headers=headers, allow_redirects=True) as response:
|
|
217
|
+
# Получаем куки из ответа
|
|
218
|
+
response_cookies = response.cookies
|
|
219
|
+
for cookie in response_cookies:
|
|
220
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
221
|
+
|
|
222
|
+
# Дополнительный запрос для симуляции поведения браузера
|
|
223
|
+
async with temp_session.get("https://login.1688.com/", headers=headers, allow_redirects=True) as response:
|
|
224
|
+
# Обновляем куки
|
|
225
|
+
response_cookies = response.cookies
|
|
226
|
+
for cookie in response_cookies:
|
|
227
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
228
|
+
|
|
229
|
+
# Запрос к API для получения дополнительных кук
|
|
230
|
+
api_headers = headers.copy()
|
|
231
|
+
api_headers.update({
|
|
232
|
+
"Accept": "application/json, text/plain, */*",
|
|
233
|
+
"Sec-Fetch-Dest": "empty",
|
|
234
|
+
"Sec-Fetch-Mode": "cors",
|
|
235
|
+
"Sec-Fetch-Site": "same-origin",
|
|
236
|
+
})
|
|
237
|
+
|
|
238
|
+
async with temp_session.get("https://www.1688.com/api/token/init", headers=api_headers) as response:
|
|
239
|
+
# Обновляем куки
|
|
240
|
+
response_cookies = response.cookies
|
|
241
|
+
for cookie in response_cookies:
|
|
242
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
243
|
+
|
|
244
|
+
except Exception:
|
|
245
|
+
# Fallback на базовые куки
|
|
246
|
+
self.cookies_dict = {
|
|
247
|
+
"cna": "Ez7rHJABCwKCAXrD2Q==",
|
|
248
|
+
"_m_h5_tk": "random_token_12345",
|
|
249
|
+
"_m_h5_tk_enc": "random_enc_token_12345",
|
|
250
|
+
}
|
|
251
|
+
finally:
|
|
252
|
+
# Всегда закрываем временную сессию
|
|
253
|
+
if temp_session and not temp_session.closed:
|
|
254
|
+
await temp_session.close()
|
|
255
|
+
|
|
256
|
+
async def get_by_id(self, offer_id: str, timeout: float = None) -> DetailProduct:
|
|
257
|
+
"""Парсит товар по ID"""
|
|
258
|
+
await self._ensure_initialized()
|
|
259
|
+
|
|
260
|
+
if timeout is None:
|
|
261
|
+
timeout = self.default_timeout
|
|
262
|
+
|
|
263
|
+
# Если куки еще не получены, получаем их
|
|
264
|
+
if not self.cookies_dict:
|
|
265
|
+
await self._get_cookies_with_aiohttp()
|
|
266
|
+
|
|
267
|
+
url = f"https://detail.1688.com/offer/{offer_id}.html"
|
|
268
|
+
|
|
269
|
+
cookies_str = '; '.join([f'{k}={v}' for k, v in self.cookies_dict.items()])
|
|
270
|
+
|
|
271
|
+
headers = {
|
|
272
|
+
"cookie": cookies_str,
|
|
273
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
274
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
275
|
+
"accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
try:
|
|
279
|
+
async with self._session.get(
|
|
280
|
+
url,
|
|
281
|
+
headers=headers,
|
|
282
|
+
timeout=timeout
|
|
283
|
+
) as response:
|
|
284
|
+
|
|
285
|
+
if response.status == 200:
|
|
286
|
+
html_content = await response.text()
|
|
287
|
+
|
|
288
|
+
# Ищем JSON данные
|
|
289
|
+
json_str = self._extract_json_string(html_content)
|
|
290
|
+
|
|
291
|
+
if json_str:
|
|
292
|
+
# Исправляем JSON перед парсингом
|
|
293
|
+
fixed_json_str = self._fix_json_issues(json_str)
|
|
294
|
+
|
|
295
|
+
try:
|
|
296
|
+
product_data = json.loads(fixed_json_str)
|
|
297
|
+
product = DetailProduct(product_data)
|
|
298
|
+
return product
|
|
299
|
+
except json.JSONDecodeError as e:
|
|
300
|
+
print(f"JSON decode error: {e}")
|
|
301
|
+
return None
|
|
302
|
+
else:
|
|
303
|
+
print("No JSON data found in HTML")
|
|
304
|
+
return None
|
|
305
|
+
else:
|
|
306
|
+
print(f"HTTP error: {response.status}")
|
|
307
|
+
return None
|
|
308
|
+
|
|
309
|
+
except asyncio.TimeoutError:
|
|
310
|
+
print("Request timeout")
|
|
311
|
+
return None
|
|
312
|
+
except Exception as e:
|
|
313
|
+
print(f"Request error: {e}")
|
|
314
|
+
return None
|
|
315
|
+
|
|
316
|
+
def _extract_json_string(self, html_content: str) -> str:
|
|
317
|
+
"""Извлекает JSON строку из HTML"""
|
|
318
|
+
pattern = r'window\.contextPath\s*,\s*({.*?})\);'
|
|
319
|
+
match = re.search(pattern, html_content, re.DOTALL)
|
|
320
|
+
|
|
321
|
+
if match:
|
|
322
|
+
json_str = match.group(1)
|
|
323
|
+
return json_str
|
|
324
|
+
|
|
325
|
+
return ""
|
|
326
|
+
|
|
327
|
+
def _fix_json_issues(self, json_str: str) -> str:
|
|
328
|
+
"""Исправляет проблемы в JSON"""
|
|
329
|
+
# Исправляем объект skuWeight
|
|
330
|
+
sku_weight_pattern = r'"skuWeight":\s*\{[^}]+\}'
|
|
331
|
+
def fix_sku_weight(match):
|
|
332
|
+
sku_weight_obj = match.group(0)
|
|
333
|
+
fixed = re.sub(r'(\s*)(\d+)(\s*):(\s*)', r'\1"\2"\3:\4', sku_weight_obj)
|
|
334
|
+
return fixed
|
|
335
|
+
|
|
336
|
+
fixed_json = re.sub(sku_weight_pattern, fix_sku_weight, json_str)
|
|
337
|
+
|
|
338
|
+
# Исправляем объект skuFeatures
|
|
339
|
+
sku_features_pattern = r'"skuFeatures":\s*\{[^}]+\}'
|
|
340
|
+
def fix_sku_features(match):
|
|
341
|
+
sku_features_obj = match.group(0)
|
|
342
|
+
fixed = re.sub(r'(\s*)(\d+)(\s*):(\s*)', r'\1"\2"\3:\4', sku_features_obj)
|
|
343
|
+
return fixed
|
|
344
|
+
|
|
345
|
+
fixed_json = re.sub(sku_features_pattern, fix_sku_features, fixed_json)
|
|
346
|
+
|
|
347
|
+
return fixed_json
|
|
348
|
+
|
|
349
|
+
@property
|
|
350
|
+
def is_active(self):
|
|
351
|
+
return self._session is not None and not self._session.closed
|
|
352
|
+
|
|
353
|
+
def __await__(self):
|
|
354
|
+
return self._create_initialized().__await__()
|
|
355
|
+
|
|
356
|
+
async def _create_initialized(self):
|
|
357
|
+
await self._initialize()
|
|
358
|
+
return self
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import re
|
|
3
|
+
from typing import List, Dict, Any
|
|
4
|
+
|
|
5
|
+
class Product:
|
|
6
|
+
def __init__(self, data: Dict[str, Any]):
|
|
7
|
+
self.raw_data = data
|
|
8
|
+
self.id = data.get("id")
|
|
9
|
+
self.name = data.get("information", {}).get("subject", "")
|
|
10
|
+
self.simple_name = data.get("information", {}).get("simpleSubject", "")
|
|
11
|
+
self.brief = data.get("information", {}).get("brief", "")
|
|
12
|
+
|
|
13
|
+
company = data.get("company", {})
|
|
14
|
+
self.company = company.get("name", "")
|
|
15
|
+
self.company_location = f"{company.get('province', '')} {company.get('city', '')}".strip()
|
|
16
|
+
self.credit_level = company.get("creditLevelText", "")
|
|
17
|
+
self.is_factory = company.get("isFactory") == "Y"
|
|
18
|
+
|
|
19
|
+
price_info = data.get("tradePrice", {}).get("offerPrice", {})
|
|
20
|
+
self.price = price_info.get("valueString", "")
|
|
21
|
+
self.quantity_prices = self._parse_quantity_prices(price_info.get("quantityPrices", []))
|
|
22
|
+
|
|
23
|
+
image_info = data.get("image", {})
|
|
24
|
+
self.image_url = image_info.get("imgUrl", "")
|
|
25
|
+
self.image_url_220x220 = image_info.get("imgUrlOf220x220", "")
|
|
26
|
+
|
|
27
|
+
trade_quantity = data.get("tradeQuantity", {})
|
|
28
|
+
self.booked_count = trade_quantity.get("bookedCount", 0)
|
|
29
|
+
self.sale_quantity = trade_quantity.get("saleQuantity", 0)
|
|
30
|
+
self.quantity_begin = trade_quantity.get("quantityBegin", 1)
|
|
31
|
+
|
|
32
|
+
self.url = f"https://detail.1688.com/offer/{self.id}.html" if self.id else ""
|
|
33
|
+
self.category_id = data.get("information", {}).get("categoryId")
|
|
34
|
+
self.features = data.get("features", {}).get("list", [])
|
|
35
|
+
|
|
36
|
+
self.tags = []
|
|
37
|
+
market_tags = data.get("marketOfferTag", {})
|
|
38
|
+
self.tags.extend(market_tags.get("offerTagIds", []))
|
|
39
|
+
self.tags.extend(market_tags.get("memberTagIds", []))
|
|
40
|
+
self.tags.extend(market_tags.get("holidayTagIds", []))
|
|
41
|
+
|
|
42
|
+
self.service_labels = []
|
|
43
|
+
common_labels = data.get("commonPositionLabels", {}).get("offerMiddle", [])
|
|
44
|
+
for label in common_labels:
|
|
45
|
+
if label.get("enable"):
|
|
46
|
+
self.service_labels.append(label.get("text", ""))
|
|
47
|
+
|
|
48
|
+
def _parse_quantity_prices(self, quantity_prices: List[Dict]) -> List[Dict]:
|
|
49
|
+
parsed_prices = []
|
|
50
|
+
for qp in quantity_prices:
|
|
51
|
+
quantity = qp.get("quantity", "")
|
|
52
|
+
price = qp.get("valueString", "")
|
|
53
|
+
if price:
|
|
54
|
+
parsed_prices.append({
|
|
55
|
+
"quantity": quantity,
|
|
56
|
+
"price": price
|
|
57
|
+
})
|
|
58
|
+
return parsed_prices
|
|
59
|
+
|
|
60
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
61
|
+
return {
|
|
62
|
+
"id": self.id,
|
|
63
|
+
"name": self.name,
|
|
64
|
+
"simple_name": self.simple_name,
|
|
65
|
+
"brief": self.brief,
|
|
66
|
+
"company": self.company,
|
|
67
|
+
"company_location": self.company_location,
|
|
68
|
+
"credit_level": self.credit_level,
|
|
69
|
+
"is_factory": self.is_factory,
|
|
70
|
+
"price": self.price,
|
|
71
|
+
"quantity_prices": self.quantity_prices,
|
|
72
|
+
"image_url": self.image_url,
|
|
73
|
+
"image_url_220x220": self.image_url_220x220,
|
|
74
|
+
"booked_count": self.booked_count,
|
|
75
|
+
"sale_quantity": self.sale_quantity,
|
|
76
|
+
"min_quantity": self.quantity_begin,
|
|
77
|
+
"url": self.url,
|
|
78
|
+
"category_id": self.category_id,
|
|
79
|
+
"tags": self.tags,
|
|
80
|
+
"service_labels": self.service_labels
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
def __str__(self):
|
|
84
|
+
return f"Product(id={self.id}, name={self.simple_name[:50]}..., price={self.price}, company={self.company})"
|
|
85
|
+
|
|
86
|
+
def __repr__(self):
|
|
87
|
+
return f"Product(id={self.id}, name={self.simple_name[:30]}...)"
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class DetailProduct:
|
|
91
|
+
def __init__(self, data: Dict[str, Any]):
|
|
92
|
+
self.raw_data = data
|
|
93
|
+
self._extract_data()
|
|
94
|
+
|
|
95
|
+
def _extract_data(self):
|
|
96
|
+
"""Извлекает все данные из JSON структуры"""
|
|
97
|
+
try:
|
|
98
|
+
# ID товара
|
|
99
|
+
self.id = self._get_nested_value(["result", "data", "shippingServices", "fields", "deliveryLimitTimeModel", "offerId"])
|
|
100
|
+
|
|
101
|
+
# Название товара
|
|
102
|
+
self.name = self._get_nested_value(["result", "data", "gallery", "fields", "subject"])
|
|
103
|
+
|
|
104
|
+
# Бренд и вес из featureAttributes
|
|
105
|
+
attributes = self._get_nested_value(["result", "global", "globalData", "model", "offerDetail", "featureAttributes"])
|
|
106
|
+
self.brand = None
|
|
107
|
+
self.weight = None
|
|
108
|
+
if attributes:
|
|
109
|
+
for attr in attributes:
|
|
110
|
+
if attr.get("name") == "品牌":
|
|
111
|
+
self.brand = attr.get("value")
|
|
112
|
+
if attr.get("name") == "净含量":
|
|
113
|
+
self.weight = attr.get("value")
|
|
114
|
+
|
|
115
|
+
# Компания
|
|
116
|
+
self.company = self._get_nested_value(["result", "data", "Root", "fields", "dataJson", "tempModel", "companyName"])
|
|
117
|
+
|
|
118
|
+
# Названия товаров (goods) с ценами и картинками
|
|
119
|
+
self.goods = []
|
|
120
|
+
|
|
121
|
+
# ПРАВИЛЬНЫЙ путь к данным
|
|
122
|
+
sku_props = self._get_nested_value(["result", "data", "Root", "fields", "dataJson", "skuModel", "skuProps"])
|
|
123
|
+
sku_info_map = self._get_nested_value(["result", "data", "Root", "fields", "dataJson", "skuModel", "skuInfoMap"])
|
|
124
|
+
|
|
125
|
+
if sku_props and len(sku_props) > 0 and 'value' in sku_props[0]:
|
|
126
|
+
goods_data = sku_props[0]['value']
|
|
127
|
+
|
|
128
|
+
if goods_data and sku_info_map:
|
|
129
|
+
for good in goods_data:
|
|
130
|
+
good_name = good.get("name")
|
|
131
|
+
image_url = good.get("imageUrl")
|
|
132
|
+
|
|
133
|
+
# Получаем цену из skuInfoMap
|
|
134
|
+
price = None
|
|
135
|
+
if good_name in sku_info_map:
|
|
136
|
+
price = sku_info_map[good_name].get("price")
|
|
137
|
+
|
|
138
|
+
# Добавляем товар с названием, ценой и картинкой
|
|
139
|
+
self.goods.append({
|
|
140
|
+
"name": good_name,
|
|
141
|
+
"price": price,
|
|
142
|
+
"imageUrl": image_url
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
# Рейтинг
|
|
146
|
+
self.grade = self._get_nested_value(["result", "data", "productTitle", "fields", "rateInfo", "goodsGrade"])
|
|
147
|
+
|
|
148
|
+
# Процент повторных заказов
|
|
149
|
+
self.repeat_orders = self._get_nested_value(["result", "data", "productTitle", "fields", "shopInfo", "byrRepeatRate3m"])
|
|
150
|
+
|
|
151
|
+
# Количество продаж
|
|
152
|
+
self.saledCount = self._get_nested_value(["result", "data", "Root", "fields", "dataJson", "tempModel", "saledCount"])
|
|
153
|
+
|
|
154
|
+
except Exception:
|
|
155
|
+
pass
|
|
156
|
+
|
|
157
|
+
def _get_nested_value(self, keys: List[str]):
|
|
158
|
+
"""Безопасно получает значение из вложенной структуры"""
|
|
159
|
+
current = self.raw_data
|
|
160
|
+
for key in keys:
|
|
161
|
+
if isinstance(current, dict) and key in current:
|
|
162
|
+
current = current[key]
|
|
163
|
+
else:
|
|
164
|
+
return None
|
|
165
|
+
return current
|
|
166
|
+
|
|
167
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
168
|
+
"""Возвращает данные в виде словаря"""
|
|
169
|
+
return {
|
|
170
|
+
"id": self.id,
|
|
171
|
+
"name": self.name,
|
|
172
|
+
"brand": self.brand,
|
|
173
|
+
"company": self.company,
|
|
174
|
+
"grade": self.grade,
|
|
175
|
+
"weight": self.weight,
|
|
176
|
+
"goods": self.goods,
|
|
177
|
+
"repeat_orders": self.repeat_orders,
|
|
178
|
+
"saledCount": self.saledCount
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def extract_products_from_html(html_content: str) -> List[Product]:
|
|
183
|
+
products = []
|
|
184
|
+
|
|
185
|
+
try:
|
|
186
|
+
patterns = [
|
|
187
|
+
r'window\.data\.offerresultData\s*=\s*successDataCheck\(\s*({.*?})\s*\)',
|
|
188
|
+
r'window\.data\.offerresultData\s*=\s*({.*?});',
|
|
189
|
+
r'offerresultData\s*=\s*successDataCheck\(\s*({.*?})\s*\)'
|
|
190
|
+
]
|
|
191
|
+
|
|
192
|
+
json_data = None
|
|
193
|
+
for pattern in patterns:
|
|
194
|
+
match = re.search(pattern, html_content, re.DOTALL)
|
|
195
|
+
if match:
|
|
196
|
+
try:
|
|
197
|
+
json_str = match.group(1)
|
|
198
|
+
json_str = re.sub(r',\s*]', ']', json_str)
|
|
199
|
+
json_str = re.sub(r',\s*}', '}', json_str)
|
|
200
|
+
data = json.loads(json_str)
|
|
201
|
+
json_data = data
|
|
202
|
+
break
|
|
203
|
+
except json.JSONDecodeError:
|
|
204
|
+
continue
|
|
205
|
+
|
|
206
|
+
if not json_data:
|
|
207
|
+
return products
|
|
208
|
+
|
|
209
|
+
offer_list = []
|
|
210
|
+
|
|
211
|
+
if "data" in json_data and "offerList" in json_data["data"]:
|
|
212
|
+
offer_list = json_data["data"]["offerList"]
|
|
213
|
+
elif "offerList" in json_data:
|
|
214
|
+
offer_list = json_data["offerList"]
|
|
215
|
+
else:
|
|
216
|
+
def find_offer_list(obj):
|
|
217
|
+
if isinstance(obj, dict):
|
|
218
|
+
for key, value in obj.items():
|
|
219
|
+
if key == "offerList" and isinstance(value, list):
|
|
220
|
+
return value
|
|
221
|
+
result = find_offer_list(value)
|
|
222
|
+
if result is not None:
|
|
223
|
+
return result
|
|
224
|
+
elif isinstance(obj, list):
|
|
225
|
+
for item in obj:
|
|
226
|
+
result = find_offer_list(item)
|
|
227
|
+
if result is not None:
|
|
228
|
+
return result
|
|
229
|
+
return None
|
|
230
|
+
|
|
231
|
+
offer_list = find_offer_list(json_data) or []
|
|
232
|
+
|
|
233
|
+
for offer_data in offer_list:
|
|
234
|
+
try:
|
|
235
|
+
product = Product(offer_data)
|
|
236
|
+
products.append(product)
|
|
237
|
+
except Exception:
|
|
238
|
+
continue
|
|
239
|
+
|
|
240
|
+
except Exception:
|
|
241
|
+
pass
|
|
242
|
+
|
|
243
|
+
return products
|
|
@@ -0,0 +1,388 @@
|
|
|
1
|
+
import asyncio
|
|
2
|
+
import requests
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
import base64
|
|
7
|
+
from typing import List, Dict, Any
|
|
8
|
+
from urllib.parse import urlencode
|
|
9
|
+
import aiohttp
|
|
10
|
+
from yarl import URL
|
|
11
|
+
|
|
12
|
+
from .models import Product, DetailProduct, extract_products_from_html
|
|
13
|
+
from .utils import prepare_image_request, generate_sign, read_and_encode_image
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Sync1688Session:
|
|
17
|
+
def __init__(self, default_timeout: float = 30):
|
|
18
|
+
self._async_session = None
|
|
19
|
+
self._token = None
|
|
20
|
+
self._token_part = None
|
|
21
|
+
self.app_key = "12574478"
|
|
22
|
+
self.base_url = "https://h5api.m.1688.com/h5/mtop.relationrecommend.wirelessrecommend.recommend/2.0/"
|
|
23
|
+
self._initialized = False
|
|
24
|
+
self.default_timeout = default_timeout
|
|
25
|
+
self.cookies_dict = {}
|
|
26
|
+
self._loop = None
|
|
27
|
+
|
|
28
|
+
def __enter__(self):
|
|
29
|
+
if not self._initialized:
|
|
30
|
+
self.start()
|
|
31
|
+
return self
|
|
32
|
+
|
|
33
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
34
|
+
self.close()
|
|
35
|
+
|
|
36
|
+
def _run_async(self, coro):
|
|
37
|
+
"""Запускает асинхронную функцию в синхронном контексте"""
|
|
38
|
+
if self._loop is None:
|
|
39
|
+
try:
|
|
40
|
+
self._loop = asyncio.get_event_loop()
|
|
41
|
+
except RuntimeError:
|
|
42
|
+
self._loop = asyncio.new_event_loop()
|
|
43
|
+
asyncio.set_event_loop(self._loop)
|
|
44
|
+
|
|
45
|
+
if self._loop.is_running():
|
|
46
|
+
# Если loop уже запущен, создаем новую задачу
|
|
47
|
+
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
|
48
|
+
return future.result()
|
|
49
|
+
else:
|
|
50
|
+
# Если loop не запущен, запускаем его
|
|
51
|
+
return self._loop.run_until_complete(coro)
|
|
52
|
+
|
|
53
|
+
def _initialize(self):
|
|
54
|
+
if self._initialized:
|
|
55
|
+
return
|
|
56
|
+
|
|
57
|
+
self.start()
|
|
58
|
+
self._initialized = True
|
|
59
|
+
|
|
60
|
+
def start(self):
|
|
61
|
+
if self._async_session and not self._async_session.closed:
|
|
62
|
+
self.close()
|
|
63
|
+
|
|
64
|
+
async def _start_async():
|
|
65
|
+
self._async_session = aiohttp.ClientSession()
|
|
66
|
+
|
|
67
|
+
try:
|
|
68
|
+
test_params = {
|
|
69
|
+
"jsv": "2.7.2",
|
|
70
|
+
"appKey": self.app_key,
|
|
71
|
+
"t": str(int(time.time() * 1000)),
|
|
72
|
+
"api": "mtop.relationrecommend.WirelessRecommend.recommend",
|
|
73
|
+
"v": "2.0",
|
|
74
|
+
"type": "originaljson"
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async with self._async_session.get(
|
|
78
|
+
self.base_url,
|
|
79
|
+
params=test_params,
|
|
80
|
+
timeout=self.default_timeout
|
|
81
|
+
) as response:
|
|
82
|
+
|
|
83
|
+
url_obj = URL(self.base_url)
|
|
84
|
+
cookies = self._async_session.cookie_jar.filter_cookies(url_obj)
|
|
85
|
+
token_cookie = cookies.get('_m_h5_tk')
|
|
86
|
+
|
|
87
|
+
if token_cookie:
|
|
88
|
+
self._token = token_cookie.value
|
|
89
|
+
self._token_part = self._token.split('_')[0]
|
|
90
|
+
self._initialized = True
|
|
91
|
+
return True
|
|
92
|
+
else:
|
|
93
|
+
raise Exception("Не удалось получить токен")
|
|
94
|
+
|
|
95
|
+
except Exception as e:
|
|
96
|
+
await self._async_session.close()
|
|
97
|
+
self._async_session = None
|
|
98
|
+
raise Exception(f"Ошибка запуска сессии: {e}")
|
|
99
|
+
|
|
100
|
+
return self._run_async(_start_async())
|
|
101
|
+
|
|
102
|
+
def close(self):
|
|
103
|
+
if self._async_session and not self._async_session.closed:
|
|
104
|
+
async def _close_async():
|
|
105
|
+
await self._async_session.close()
|
|
106
|
+
|
|
107
|
+
self._run_async(_close_async())
|
|
108
|
+
self._async_session = None
|
|
109
|
+
self._token = None
|
|
110
|
+
self._token_part = None
|
|
111
|
+
self._initialized = False
|
|
112
|
+
self.cookies_dict = {}
|
|
113
|
+
|
|
114
|
+
def _ensure_initialized(self):
|
|
115
|
+
if not self._initialized or not self.is_active:
|
|
116
|
+
self._initialize()
|
|
117
|
+
|
|
118
|
+
def _get_image_id(self, image_path, timeout=None):
|
|
119
|
+
self._ensure_initialized()
|
|
120
|
+
|
|
121
|
+
if timeout is None:
|
|
122
|
+
timeout = self.default_timeout
|
|
123
|
+
|
|
124
|
+
async def _get_image_id_async():
|
|
125
|
+
image_b64 = read_and_encode_image(image_path)
|
|
126
|
+
data_string = prepare_image_request(image_b64)
|
|
127
|
+
timestamp = str(int(time.time() * 1000))
|
|
128
|
+
sign = generate_sign(self._token_part, timestamp, self.app_key, data_string)
|
|
129
|
+
|
|
130
|
+
params = {
|
|
131
|
+
"jsv": "2.7.2",
|
|
132
|
+
"appKey": self.app_key,
|
|
133
|
+
"t": timestamp,
|
|
134
|
+
"sign": sign,
|
|
135
|
+
"api": "mtop.relationrecommend.WirelessRecommend.recommend",
|
|
136
|
+
"ignoreLogin": "true",
|
|
137
|
+
"prefix": "h5api",
|
|
138
|
+
"v": "2.0",
|
|
139
|
+
"type": "originaljson",
|
|
140
|
+
"dataType": "jsonp",
|
|
141
|
+
"jsonpIncPrefix": "search1688",
|
|
142
|
+
"timeout": "20000"
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
headers = {
|
|
146
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
147
|
+
"origin": "https://s.1688.com",
|
|
148
|
+
"referer": "https://s.1688.com/",
|
|
149
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
try:
|
|
153
|
+
async with self._async_session.post(
|
|
154
|
+
self.base_url,
|
|
155
|
+
params=params,
|
|
156
|
+
data={"data": data_string},
|
|
157
|
+
headers=headers,
|
|
158
|
+
timeout=timeout
|
|
159
|
+
) as response:
|
|
160
|
+
|
|
161
|
+
if response.status == 200:
|
|
162
|
+
result = await response.json()
|
|
163
|
+
if result.get("data", {}).get("success"):
|
|
164
|
+
image_id = result["data"].get("imageId")
|
|
165
|
+
if image_id:
|
|
166
|
+
return image_id
|
|
167
|
+
return None
|
|
168
|
+
|
|
169
|
+
except Exception as e:
|
|
170
|
+
raise Exception(f"Ошибка при запросе: {e}")
|
|
171
|
+
|
|
172
|
+
return self._run_async(_get_image_id_async())
|
|
173
|
+
|
|
174
|
+
def search_by_image(self, image_path: str, timeout: float = None) -> List[Product]:
|
|
175
|
+
self._ensure_initialized()
|
|
176
|
+
|
|
177
|
+
if timeout is None:
|
|
178
|
+
timeout = self.default_timeout
|
|
179
|
+
|
|
180
|
+
image_id = self._get_image_id(image_path, timeout=timeout)
|
|
181
|
+
|
|
182
|
+
if not image_id:
|
|
183
|
+
return []
|
|
184
|
+
|
|
185
|
+
products = self._search_by_image_id(image_id, timeout=timeout)
|
|
186
|
+
return products
|
|
187
|
+
|
|
188
|
+
def _search_by_image_id(self, image_id: str, timeout=None) -> List[Product]:
|
|
189
|
+
self._ensure_initialized()
|
|
190
|
+
|
|
191
|
+
if timeout is None:
|
|
192
|
+
timeout = self.default_timeout
|
|
193
|
+
|
|
194
|
+
async def _search_by_image_id_async():
|
|
195
|
+
search_url = f"https://s.1688.com/youyuan/index.htm"
|
|
196
|
+
params = {
|
|
197
|
+
"tab": "imageSearch",
|
|
198
|
+
"imageId": image_id
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
headers = {
|
|
202
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
203
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
204
|
+
"accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
205
|
+
"referer": "https://s.1688.com/"
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
try:
|
|
209
|
+
async with self._async_session.get(
|
|
210
|
+
search_url,
|
|
211
|
+
params=params,
|
|
212
|
+
headers=headers,
|
|
213
|
+
timeout=timeout
|
|
214
|
+
) as response:
|
|
215
|
+
|
|
216
|
+
if response.status == 200:
|
|
217
|
+
html_content = await response.text()
|
|
218
|
+
products = extract_products_from_html(html_content)
|
|
219
|
+
return products
|
|
220
|
+
else:
|
|
221
|
+
return []
|
|
222
|
+
|
|
223
|
+
except Exception:
|
|
224
|
+
return []
|
|
225
|
+
|
|
226
|
+
return self._run_async(_search_by_image_id_async())
|
|
227
|
+
|
|
228
|
+
def _get_cookies_with_aiohttp(self):
|
|
229
|
+
"""Получает куки через aiohttp запросы"""
|
|
230
|
+
async def _get_cookies_async():
|
|
231
|
+
temp_session = None
|
|
232
|
+
try:
|
|
233
|
+
temp_session = aiohttp.ClientSession()
|
|
234
|
+
|
|
235
|
+
headers = {
|
|
236
|
+
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
|
237
|
+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
|
238
|
+
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
239
|
+
"Accept-Encoding": "gzip, deflate, br",
|
|
240
|
+
"DNT": "1",
|
|
241
|
+
"Connection": "keep-alive",
|
|
242
|
+
"Upgrade-Insecure-Requests": "1",
|
|
243
|
+
"Sec-Fetch-Dest": "document",
|
|
244
|
+
"Sec-Fetch-Mode": "navigate",
|
|
245
|
+
"Sec-Fetch-Site": "none",
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
# Запрос к главной странице 1688
|
|
249
|
+
async with temp_session.get("https://www.1688.com/", headers=headers, allow_redirects=True) as response:
|
|
250
|
+
# Получаем куки из ответа
|
|
251
|
+
response_cookies = response.cookies
|
|
252
|
+
for cookie in response_cookies:
|
|
253
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
254
|
+
|
|
255
|
+
# Дополнительный запрос для симуляции поведения браузера
|
|
256
|
+
async with temp_session.get("https://login.1688.com/", headers=headers, allow_redirects=True) as response:
|
|
257
|
+
# Обновляем куки
|
|
258
|
+
response_cookies = response.cookies
|
|
259
|
+
for cookie in response_cookies:
|
|
260
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
261
|
+
|
|
262
|
+
# Запрос к API для получения дополнительных кук
|
|
263
|
+
api_headers = headers.copy()
|
|
264
|
+
api_headers.update({
|
|
265
|
+
"Accept": "application/json, text/plain, */*",
|
|
266
|
+
"Sec-Fetch-Dest": "empty",
|
|
267
|
+
"Sec-Fetch-Mode": "cors",
|
|
268
|
+
"Sec-Fetch-Site": "same-origin",
|
|
269
|
+
})
|
|
270
|
+
|
|
271
|
+
async with temp_session.get("https://www.1688.com/api/token/init", headers=api_headers) as response:
|
|
272
|
+
# Обновляем куки
|
|
273
|
+
response_cookies = response.cookies
|
|
274
|
+
for cookie in response_cookies:
|
|
275
|
+
self.cookies_dict[cookie.key] = cookie.value
|
|
276
|
+
|
|
277
|
+
except Exception:
|
|
278
|
+
# Fallback на базовые куки
|
|
279
|
+
self.cookies_dict = {
|
|
280
|
+
"cna": "Ez7rHJABCwKCAXrD2Q==",
|
|
281
|
+
"_m_h5_tk": "random_token_12345",
|
|
282
|
+
"_m_h5_tk_enc": "random_enc_token_12345",
|
|
283
|
+
}
|
|
284
|
+
finally:
|
|
285
|
+
if temp_session and not temp_session.closed:
|
|
286
|
+
await temp_session.close()
|
|
287
|
+
|
|
288
|
+
self._run_async(_get_cookies_async())
|
|
289
|
+
|
|
290
|
+
def get_by_id(self, offer_id: str, timeout: float = None) -> DetailProduct:
|
|
291
|
+
"""Парсит товар по ID"""
|
|
292
|
+
self._ensure_initialized()
|
|
293
|
+
|
|
294
|
+
if timeout is None:
|
|
295
|
+
timeout = self.default_timeout
|
|
296
|
+
|
|
297
|
+
# Если куки еще не получены, получаем их
|
|
298
|
+
if not self.cookies_dict:
|
|
299
|
+
self._get_cookies_with_aiohttp()
|
|
300
|
+
|
|
301
|
+
async def _get_by_id_async():
|
|
302
|
+
url = f"https://detail.1688.com/offer/{offer_id}.html"
|
|
303
|
+
|
|
304
|
+
cookies_str = '; '.join([f'{k}={v}' for k, v in self.cookies_dict.items()])
|
|
305
|
+
|
|
306
|
+
headers = {
|
|
307
|
+
"cookie": cookies_str,
|
|
308
|
+
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
|
|
309
|
+
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
|
310
|
+
"accept-language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
try:
|
|
314
|
+
async with self._async_session.get(
|
|
315
|
+
url,
|
|
316
|
+
headers=headers,
|
|
317
|
+
timeout=timeout
|
|
318
|
+
) as response:
|
|
319
|
+
|
|
320
|
+
if response.status == 200:
|
|
321
|
+
html_content = await response.text()
|
|
322
|
+
|
|
323
|
+
# Ищем JSON данные
|
|
324
|
+
json_str = self._extract_json_string(html_content)
|
|
325
|
+
|
|
326
|
+
if json_str:
|
|
327
|
+
# Исправляем JSON перед парсингом
|
|
328
|
+
fixed_json_str = self._fix_json_issues(json_str)
|
|
329
|
+
|
|
330
|
+
try:
|
|
331
|
+
product_data = json.loads(fixed_json_str)
|
|
332
|
+
product = DetailProduct(product_data)
|
|
333
|
+
return product
|
|
334
|
+
except json.JSONDecodeError as e:
|
|
335
|
+
print(f"JSON decode error: {e}")
|
|
336
|
+
return None
|
|
337
|
+
else:
|
|
338
|
+
print("No JSON data found in HTML")
|
|
339
|
+
return None
|
|
340
|
+
else:
|
|
341
|
+
print(f"HTTP error: {response.status}")
|
|
342
|
+
return None
|
|
343
|
+
|
|
344
|
+
except asyncio.TimeoutError:
|
|
345
|
+
print("Request timeout")
|
|
346
|
+
return None
|
|
347
|
+
except Exception as e:
|
|
348
|
+
print(f"Request error: {e}")
|
|
349
|
+
return None
|
|
350
|
+
|
|
351
|
+
return self._run_async(_get_by_id_async())
|
|
352
|
+
|
|
353
|
+
def _extract_json_string(self, html_content: str) -> str:
|
|
354
|
+
"""Извлекает JSON строку из HTML"""
|
|
355
|
+
pattern = r'window\.contextPath\s*,\s*({.*?})\);'
|
|
356
|
+
match = re.search(pattern, html_content, re.DOTALL)
|
|
357
|
+
|
|
358
|
+
if match:
|
|
359
|
+
json_str = match.group(1)
|
|
360
|
+
return json_str
|
|
361
|
+
|
|
362
|
+
return ""
|
|
363
|
+
|
|
364
|
+
def _fix_json_issues(self, json_str: str) -> str:
|
|
365
|
+
"""Исправляет проблемы в JSON"""
|
|
366
|
+
# Исправляем объект skuWeight
|
|
367
|
+
sku_weight_pattern = r'"skuWeight":\s*\{[^}]+\}'
|
|
368
|
+
def fix_sku_weight(match):
|
|
369
|
+
sku_weight_obj = match.group(0)
|
|
370
|
+
fixed = re.sub(r'(\s*)(\d+)(\s*):(\s*)', r'\1"\2"\3:\4', sku_weight_obj)
|
|
371
|
+
return fixed
|
|
372
|
+
|
|
373
|
+
fixed_json = re.sub(sku_weight_pattern, fix_sku_weight, json_str)
|
|
374
|
+
|
|
375
|
+
# Исправляем объект skuFeatures
|
|
376
|
+
sku_features_pattern = r'"skuFeatures":\s*\{[^}]+\}'
|
|
377
|
+
def fix_sku_features(match):
|
|
378
|
+
sku_features_obj = match.group(0)
|
|
379
|
+
fixed = re.sub(r'(\s*)(\d+)(\s*):(\s*)', r'\1"\2"\3:\4', sku_features_obj)
|
|
380
|
+
return fixed
|
|
381
|
+
|
|
382
|
+
fixed_json = re.sub(sku_features_pattern, fix_sku_features, fixed_json)
|
|
383
|
+
|
|
384
|
+
return fixed_json
|
|
385
|
+
|
|
386
|
+
@property
|
|
387
|
+
def is_active(self):
|
|
388
|
+
return self._async_session is not None and not self._async_session.closed
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import time
|
|
3
|
+
import hashlib
|
|
4
|
+
import json
|
|
5
|
+
from typing import Dict, Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def prepare_image_request(image_b64: str) -> str:
|
|
9
|
+
params_data = {
|
|
10
|
+
"searchScene": "imageEx",
|
|
11
|
+
"interfaceName": "imageBase64ToImageId",
|
|
12
|
+
"serviceParam.extendParam[imageBase64]": image_b64,
|
|
13
|
+
"subChannel": "pc_image_search_image_id"
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
request_data = {
|
|
17
|
+
"appId": 32517,
|
|
18
|
+
"params": json.dumps(params_data, separators=(',', ':'))
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
return json.dumps(request_data, separators=(',', ':'))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def generate_sign(token_part: str, timestamp: str, app_key: str, data_string: str) -> str:
|
|
25
|
+
if not token_part:
|
|
26
|
+
raise ValueError("Токен не установлен")
|
|
27
|
+
|
|
28
|
+
sign_string = f"{token_part}&{timestamp}&{app_key}&{data_string}"
|
|
29
|
+
return hashlib.md5(sign_string.encode('utf-8')).hexdigest()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def read_and_encode_image(image_path: str) -> str:
|
|
33
|
+
try:
|
|
34
|
+
with open(image_path, 'rb') as f:
|
|
35
|
+
image_b64 = base64.b64encode(f.read()).decode('utf-8')
|
|
36
|
+
return image_b64
|
|
37
|
+
except Exception as e:
|
|
38
|
+
raise ValueError(f"Ошибка чтения файла: {e}")
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: search1688api
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Python library for searching products on 1688.com by image
|
|
5
|
+
Author: netkaruma
|
|
6
|
+
Author-email: suzumekaruma@gmail.com
|
|
7
|
+
Requires-Python: >=3.7
|
|
8
|
+
Requires-Dist: requests>=2.25.0
|
|
9
|
+
Requires-Dist: aiohttp>=3.8.0
|
|
10
|
+
Requires-Dist: yarl>=1.6.0
|
|
11
|
+
Dynamic: author
|
|
12
|
+
Dynamic: author-email
|
|
13
|
+
Dynamic: requires-dist
|
|
14
|
+
Dynamic: requires-python
|
|
15
|
+
Dynamic: summary
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
setup.py
|
|
3
|
+
search1688api/__init__.py
|
|
4
|
+
search1688api/async_session.py
|
|
5
|
+
search1688api/models.py
|
|
6
|
+
search1688api/sync_session.py
|
|
7
|
+
search1688api/utils.py
|
|
8
|
+
search1688api.egg-info/PKG-INFO
|
|
9
|
+
search1688api.egg-info/SOURCES.txt
|
|
10
|
+
search1688api.egg-info/dependency_links.txt
|
|
11
|
+
search1688api.egg-info/requires.txt
|
|
12
|
+
search1688api.egg-info/top_level.txt
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
search1688api
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
from setuptools import setup, find_packages
|
|
2
|
+
|
|
3
|
+
setup(
|
|
4
|
+
name="search1688api",
|
|
5
|
+
version="1.0.0",
|
|
6
|
+
author="netkaruma",
|
|
7
|
+
author_email="suzumekaruma@gmail.com",
|
|
8
|
+
description="Python library for searching products on 1688.com by image",
|
|
9
|
+
packages=find_packages(),
|
|
10
|
+
python_requires=">=3.7",
|
|
11
|
+
install_requires=[
|
|
12
|
+
"requests>=2.25.0",
|
|
13
|
+
"aiohttp>=3.8.0",
|
|
14
|
+
"yarl>=1.6.0",
|
|
15
|
+
],
|
|
16
|
+
)
|