xhs-note-extractor 0.1.5.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,985 @@
1
+ """
2
+ 小红书笔记提取器模块
3
+
4
+ 该模块提供了从小红书URL中提取笔记信息的功能,包括:
5
+ - URL解析和转换
6
+ - 设备连接和页面跳转
7
+ - 笔记内容提取(正文、图片、点赞数等)
8
+ - 结构化数据返回
9
+
10
+ 作者: JoyCode Agent
11
+ 版本: 1.0.0
12
+ """
13
+
14
+ import uiautomator2 as u2
15
+ import time
16
+ import re
17
+ import requests
18
+ import logging
19
+ from typing import Dict, List, Optional, Union
20
+ from urllib.parse import urlparse, parse_qs
21
+ import xml.etree.ElementTree as ET
22
+
23
+ # 延迟加载agent_login模块以避免不必要的依赖
24
+ from .date_desc_utils import parse_time_to_timestamp_ms
25
+ from .number_utils import parse_count_to_int
26
+
27
+ # 配置日志
28
+ logging.basicConfig(
29
+ level=logging.INFO,
30
+ format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
31
+ )
32
+ logger = logging.getLogger(__name__)
33
+
34
+
35
+ class XHSNoteExtractor:
36
+ """
37
+ 小红书笔记提取器类
38
+
39
+ 提供了从小红书URL中提取笔记信息的完整功能,
40
+ 包括URL解析、设备连接、页面跳转和笔记内容提取。
41
+ """
42
+
43
+ def __init__(self, devices:dict = None):
44
+ """
45
+ 初始化小红书笔记提取器
46
+
47
+ Args:
48
+ devices (dict, optional): 设备信息字典,包含设备序列号和对应小红书账号可选手机号
49
+ {
50
+ "b520805": ["13800000000"]
51
+ }
52
+
53
+ Raises:
54
+ ValueError: 当设备信息为空或无效时抛出异常
55
+ """
56
+ if not devices:
57
+ raise ValueError("设备信息必须从外部传入")
58
+
59
+ self.device = None # 当前设备
60
+ self.next_phone = None # 下一个手机号
61
+ self.devices_info = devices # 存储设备信息字典
62
+ self.problematic_devices = [] # 存储无法获取笔记的设备信息
63
+ self.enable_time_logging = True # 默认启用耗时打印
64
+
65
+ # 日志记录设备信息
66
+ logger.info(f"已配置设备信息: {self.devices_info}")
67
+ logger.info("设备将在需要时连接")
68
+
69
+ def _get_next_phone_number(self, device_serial: str) -> Optional[str]:
70
+ """
71
+ 获取指定设备的下一个手机号(循环)
72
+
73
+ Args:
74
+ device_serial (str): 设备序列号
75
+
76
+ Returns:
77
+ str: 下一个手机号,如果没有则返回None
78
+ """
79
+ if device_serial not in self.devices_info:
80
+ return None
81
+
82
+ phone_list = self.devices_info[device_serial]
83
+ if not phone_list:
84
+ return None
85
+
86
+ # 如果当前没有设置下一个手机号,返回第一个
87
+ if not self.next_phone:
88
+ self.next_phone = phone_list[0]
89
+ return self.next_phone
90
+
91
+ # 找到当前手机号在列表中的索引
92
+ try:
93
+ current_index = phone_list.index(self.next_phone)
94
+ # 循环到下一个
95
+ next_index = (current_index + 1) % len(phone_list)
96
+ self.next_phone = phone_list[next_index]
97
+ except ValueError:
98
+ # 如果当前手机号不在列表中,返回第一个
99
+ self.next_phone = phone_list[0]
100
+ return self.next_phone
101
+ def _time_method(self, method_name, start_time):
102
+ """
103
+ 记录方法执行时间
104
+
105
+ Args:
106
+ method_name (str): 方法名称
107
+ start_time (float): 开始时间
108
+ """
109
+ if self.enable_time_logging:
110
+ elapsed_time = time.time() - start_time
111
+ if elapsed_time < 1:
112
+ logger.info(f"⏱️ [{method_name}] 耗时: {elapsed_time*1000:.0f}ms")
113
+ else:
114
+ logger.info(f"⏱️ [{method_name}] 耗时: {elapsed_time:.2f}s")
115
+
116
+
117
+
118
+ def connect_device(self, device_serial: Optional[str] = None) -> bool:
119
+ """
120
+ 连接设备
121
+
122
+ Args:
123
+ device_serial (str, optional): 指定设备序列号,如果为None则使用devices_info中的第一个设备
124
+
125
+ Returns:
126
+ bool: 是否成功连接设备
127
+ """
128
+ start_time = time.time()
129
+
130
+ # 如果指定了设备序列号,则使用指定的设备
131
+ target_device = device_serial
132
+
133
+ # 如果没有指定设备序列号,尝试使用devices_info中的第一个设备
134
+ if not target_device and self.devices_info:
135
+ target_device = next(iter(self.devices_info.keys()))
136
+
137
+ try:
138
+ if not target_device:
139
+ logger.error("✗ 设备连接失败: 无法确定设备序列号")
140
+ self._time_method("connect_device", start_time)
141
+ return False
142
+
143
+ self.device = u2.connect(target_device)
144
+ logger.info(f"✓ 已连接设备: {self.device.serial}")
145
+ self._time_method("connect_device", start_time)
146
+ # 重启小红书应用以确保登录状态
147
+ logger.info("🔄 重启小红书应用...")
148
+ self.device.app_stop("com.xingin.xhs")
149
+ time.sleep(1)
150
+ self.device.app_start("com.xingin.xhs")
151
+ time.sleep(3)
152
+ # 获取下一个手机号
153
+ self.next_phone = self._get_next_phone_number(target_device)
154
+ logger.warning(f'next_phone:{self.next_phone}')
155
+ return True
156
+ except Exception as e:
157
+ logger.error(f"✗ 设备连接失败: {e}")
158
+ self._time_method("connect_device", start_time)
159
+ return False
160
+
161
+ def switch_to_next_device(self) -> bool:
162
+ """
163
+ 切换到下一个可用设备
164
+
165
+ Returns:
166
+ bool: 是否成功切换到下一个设备
167
+ """
168
+ self.next_phone = None # 重置下一个手机号为None
169
+ if not self.devices_info or len(self.devices_info) <= 1:
170
+ logger.warning("没有更多可用设备可以切换")
171
+ return False
172
+
173
+ # 获取当前设备的序列号
174
+ current_serial = self.device.serial if self.device else None
175
+ logger.info(f"当前设备: {current_serial}")
176
+ # 转换为列表以便切换
177
+ device_serials = list(self.devices_info.keys())
178
+ logger.info(f"device_serials: {device_serials}")
179
+ # 找到当前设备的索引
180
+ current_index = device_serials.index(current_serial) if current_serial in device_serials else -1
181
+ logger.info(f"current_index: {current_index}")
182
+
183
+ # 如果当前设备不在列表中,并且有尝试过的设备记录,则从尝试过的设备之后开始
184
+ attempted_serials = [d['serial'] for d in self.problematic_devices]
185
+ if current_index == -1 and attempted_serials:
186
+ # 找到最后一个尝试过的设备的索引
187
+ last_attempted = attempted_serials[-1]
188
+ if last_attempted in device_serials:
189
+ current_index = device_serials.index(last_attempted)
190
+
191
+ # 移动到下一个设备
192
+ next_index = (current_index + 1) % len(device_serials)
193
+ next_device_serial = device_serials[next_index]
194
+ logger.info(f"next_device_serial: {next_device_serial}")
195
+ logger.info(f"尝试切换到设备: {next_device_serial}")
196
+ return self.connect_device(next_device_serial)
197
+ def is_device_connected(self) -> bool:
198
+ """
199
+ 检查设备是否仍然连接
200
+
201
+ Returns:
202
+ bool: 设备是否连接
203
+ """
204
+ if not self.device:
205
+ return False
206
+ try:
207
+ # 通过获取设备信息来验证连接
208
+ self.device.info
209
+ return True
210
+ except:
211
+ return False
212
+
213
+ def get_problematic_devices(self) -> List[Dict[str, Union[str, float]]]:
214
+ """
215
+ 获取无法获取笔记的设备列表
216
+
217
+ Returns:
218
+ List[Dict[str, Union[str, float]]]: 包含有问题设备信息的列表,每个设备信息包括:
219
+ - serial: 设备序列号
220
+ - reason: 问题原因
221
+ - note_id: 尝试提取的笔记ID
222
+ - timestamp: 记录时间戳
223
+ """
224
+ return self.problematic_devices
225
+
226
+ def clear_problematic_devices(self) -> None:
227
+ """
228
+ 清空有问题的设备列表
229
+ """
230
+ self.problematic_devices.clear()
231
+ # 清除缓存并重启APP
232
+ def clear_login_state(self, device_serial=None):
233
+ import uiautomator2 as u2
234
+ import time
235
+
236
+ # 连接设备
237
+ d = u2.connect(device_serial)
238
+
239
+ # 启动APP
240
+ d.app_stop('com.xingin.xhs')
241
+ time.sleep(1) # 等待APP启动
242
+ d.app_start('com.xingin.xhs')
243
+ time.sleep(2) # 等待APP启动
244
+ try:
245
+ if not d(text='我').exists():
246
+ print("已退出登录,无需退出登录")
247
+ return
248
+
249
+ # 点击我的/个人中心按钮
250
+ d(description='我').click()
251
+ time.sleep(2)
252
+
253
+ if d(text='微信登录').exists() or d(text='手机号登录').exists():
254
+ print("已登录,无需退出登录")
255
+ return
256
+
257
+ # 点击设置按钮
258
+ d(description='设置').click()
259
+ time.sleep(2)
260
+
261
+ # 滚动到退出登录选项
262
+ d.swipe_ext('up', scale=0.5)
263
+ time.sleep(1)
264
+
265
+ # 点击退出登录
266
+ d(text='退出登录').click()
267
+ time.sleep(1)
268
+
269
+ # 确认退出
270
+ d(text='退出登录').click()
271
+ time.sleep(2)
272
+
273
+ print("退出登录成功")
274
+ except Exception as e:
275
+ print(f"退出登录失败: {e}")
276
+
277
+ @staticmethod
278
+ def parse_xhs_url(url: str) -> Dict[str, str]:
279
+ """
280
+ 解析小红书URL,提取note_id和xsec_token
281
+
282
+ Args:
283
+ url (str): 小红书URL,支持标准格式或xhsdiscover协议格式
284
+
285
+ Returns:
286
+ Dict[str, str]: 包含note_id和xsec_token的字典
287
+
288
+ Raises:
289
+ ValueError: 当URL格式不正确时抛出异常
290
+ """
291
+ start_time = time.time()
292
+ # 处理xhsdiscover协议格式
293
+ if url.startswith("xhsdiscover://"):
294
+ # 提取note_id
295
+ note_id_match = re.search(r'item/([^?]+)', url)
296
+ if not note_id_match:
297
+ raise ValueError("无法从xhsdiscover URL中提取note_id")
298
+
299
+ note_id = note_id_match.group(1)
300
+
301
+ # 尝试从open_url参数中提取原始URL
302
+ open_url_match = re.search(r'open_url=([^&]+)', url)
303
+ xsec_token = ""
304
+ if open_url_match:
305
+ open_url = open_url_match.group(1)
306
+ # 解码URL
307
+ import urllib.parse
308
+ decoded_url = urllib.parse.unquote(open_url)
309
+ # 从原始URL中提取xsec_token
310
+ token_match = re.search(r'xsec_token=([^&]+)', decoded_url)
311
+ if token_match:
312
+ xsec_token = token_match.group(1)
313
+
314
+ return {
315
+ "note_id": note_id,
316
+ "xsec_token": xsec_token,
317
+ "original_url": url
318
+ }
319
+
320
+ # 处理标准URL格式
321
+ elif "xiaohongshu.com" in url:
322
+ parsed_url = urlparse(url)
323
+ path_parts = parsed_url.path.strip('/').split('/')
324
+
325
+ # 查找explore部分和note_id
326
+ if 'explore' in path_parts:
327
+ explore_index = path_parts.index('explore')
328
+ if explore_index + 1 < len(path_parts):
329
+ note_id = path_parts[explore_index + 1]
330
+ else:
331
+ raise ValueError("URL中缺少note_id")
332
+ # 兼容 /discovery/item/ 格式
333
+ elif 'discovery' in path_parts and 'item' in path_parts:
334
+ item_index = path_parts.index('item')
335
+ if item_index + 1 < len(path_parts):
336
+ note_id = path_parts[item_index + 1]
337
+ else:
338
+ raise ValueError("URL中缺少note_id")
339
+ else:
340
+ raise ValueError("URL格式不正确,缺少/explore/或/discovery/item/路径")
341
+
342
+ # 提取查询参数中的xsec_token
343
+ query_params = parse_qs(parsed_url.query)
344
+ xsec_token = query_params.get('xsec_token', [''])[0]
345
+
346
+ elapsed_time = time.time() - start_time
347
+ logger.info(f"[parse_xhs_url] 耗时: {elapsed_time:.3f}秒")
348
+ return {
349
+ "note_id": note_id,
350
+ "xsec_token": xsec_token,
351
+ "original_url": url
352
+ }
353
+
354
+ else:
355
+ elapsed_time = time.time() - start_time
356
+ logger.info(f"[parse_xhs_url] 耗时: {elapsed_time:.3f}秒")
357
+ raise ValueError("不支持的URL格式")
358
+
359
+ @staticmethod
360
+ def validate_url(url: str) -> bool:
361
+ """
362
+ 验证URL是否是有效的小红书URL
363
+
364
+ Args:
365
+ url (str): 要验证的URL
366
+
367
+ Returns:
368
+ bool: URL是否有效
369
+ """
370
+ try:
371
+ XHSNoteExtractor.parse_xhs_url(url)
372
+ return True
373
+ except ValueError:
374
+ return False
375
+
376
+ @staticmethod
377
+ def convert_to_xhsdiscover_format(note_id: str, xsec_token: str = "") -> str:
378
+ """
379
+ 将note_id和xsec_token转换为xhsdiscover协议格式
380
+
381
+ Args:
382
+ note_id (str): 笔记ID
383
+ xsec_token (str): xsec_token参数
384
+
385
+ Returns:
386
+ str: xhsdiscover协议格式的URL
387
+ """
388
+ start_time = time.time()
389
+ result = ""
390
+ if xsec_token:
391
+ original_url = f"http://www.xiaohongshu.com/explore/{note_id}?xsec_token={xsec_token}&xsec_source=pc_feed"
392
+ encoded_url = requests.utils.quote(original_url)
393
+ result = f"xhsdiscover://item/{note_id}?open_url={encoded_url}"
394
+ else:
395
+ result = f"xhsdiscover://item/{note_id}"
396
+
397
+ elapsed_time = time.time() - start_time
398
+ logger.info(f"[convert_to_xhsdiscover_format] 耗时: {elapsed_time:.3f}秒")
399
+ return result
400
+
401
+ def extract_note_data(self, url: Optional[str] = None, note_id: Optional[str] = None,
402
+ xsec_token: Optional[str] = None) -> Optional[Dict[str, Union[str, List[str]]]]:
403
+ """
404
+ 从小红书笔记中提取数据,支持设备重试机制
405
+
406
+ Args:
407
+ url (str, optional): 小红书URL,如果提供则会解析其中的note_id和xsec_token
408
+ note_id (str, optional): 笔记ID,如果提供则直接使用
409
+ xsec_token (str, optional): xsec_token参数
410
+
411
+ Returns:
412
+ Optional[Dict[str, Union[str, List[str]]]]: 包含笔记数据的字典,如果没有成功则返回None
413
+
414
+ Raises:
415
+ Exception: 当提取过程中出现错误时抛出异常
416
+ """
417
+ start_time = time.time()
418
+ # 如果提供了URL,则先解析它(验证URL有效性)
419
+ if url:
420
+ parsed_data = self.parse_xhs_url(url)
421
+ note_id = parsed_data["note_id"]
422
+ xsec_token = parsed_data["xsec_token"]
423
+
424
+ max_retries = len(self.devices_info) if self.devices_info else 1
425
+ attempted_devices = []
426
+
427
+ for attempt in range(max_retries):
428
+ logger.info(f"尝试第 {attempt + 1}/{max_retries} 次提取笔记: {note_id}")
429
+
430
+ # 检查设备是否连接,如果没有则尝试连接
431
+ if self.device is None:
432
+ if not self.connect_device():
433
+ logger.warning("设备连接失败,尝试下一个设备")
434
+ # 记录连接失败的设备
435
+ device_serials = list(self.devices_info.keys())
436
+ if device_serials and attempt < len(device_serials):
437
+ failed_device = device_serials[attempt]
438
+ if failed_device not in [d['serial'] for d in self.problematic_devices]:
439
+ self.problematic_devices.append({
440
+ 'serial': failed_device,
441
+ 'reason': '设备连接失败',
442
+ 'note_id': note_id,
443
+ 'timestamp': time.time()
444
+ })
445
+ if self.switch_to_next_device():
446
+ continue
447
+ else:
448
+ break
449
+
450
+ # 构建跳转URL
451
+ jump_url = self.convert_to_xhsdiscover_format(note_id, xsec_token)
452
+
453
+ logger.info(f"正在尝试跳转至笔记: {note_id} (设备: {self.device.serial if self.device else '未知'})")
454
+
455
+ try:
456
+ # # 在跳转链接前重启APP
457
+ # logger.info(f"🔄 准备跳转至笔记 {note_id},正在重启APP...")
458
+ # self.restart_xhs_app()
459
+
460
+ # 发起跳转
461
+ self.device.open_url(jump_url)
462
+ logger.info("✓ 已发送跳转指令,等待页面加载...")
463
+
464
+ # 使用现有的xhs_utils功能提取数据
465
+ data = self._get_detail_data(jump_url)
466
+
467
+ # 如果返回None,说明需要登录,尝试下一个设备
468
+ if data is None:
469
+ logger.warning(f"当前设备{self.device.serial}需要登录,尝试切换到下一个设备")
470
+ attempted_devices.append(self.device.serial if self.device else "未知设备")
471
+ # 尝试重新登录
472
+ # 触发退出登录
473
+ self.clear_login_state(self.device.serial)
474
+ # 触发登录
475
+ try:
476
+ # 延迟加载agent_login模块以避免不必要的依赖
477
+ from .agent_login import do_login
478
+
479
+ # 获取当前设备的所有手机号列表
480
+ phone_list = self.devices_info.get(self.device.serial, [])
481
+ if not phone_list:
482
+ logger.warning(f"设备{self.device.serial}没有配置手机号")
483
+ attempted_devices.append(self.device.serial) # 记录尝试过的设备
484
+ failed_device_serial = self.device.serial
485
+ self.device = None
486
+ else:
487
+ # 找到当前手机号在列表中的索引
488
+ current_phone = self.next_phone
489
+ current_index = phone_list.index(current_phone) if current_phone in phone_list else -1
490
+
491
+ # 从下一个手机号开始循环尝试,不包括当前手机号
492
+ phone_count = len(phone_list)
493
+ login_success = False
494
+
495
+ # 如果当前手机号在列表中,从下一个开始尝试;否则从第一个开始
496
+ start_index = (current_index + 1) % phone_count if current_index != -1 else 0
497
+
498
+ # 尝试当前手机号之后的所有手机号(循环一次)
499
+ for i in range(phone_count):
500
+ # 计算当前要尝试的手机号索引
501
+ next_index = (start_index + i) % phone_count
502
+ self.next_phone = phone_list[next_index]
503
+
504
+ # 如果已经尝试过所有手机号,或者回到了当前手机号(如果当前手机号在列表中),则停止
505
+ if current_index != -1 and next_index == current_index:
506
+ break
507
+
508
+ logger.warning(f'attempting phone:{self.next_phone}')
509
+
510
+ # 尝试登录
511
+ login_result = do_login(phone_number=self.next_phone, device_id=self.device.serial)
512
+
513
+ if login_result:
514
+ logger.info(f"✓ 设备{self.device.serial}使用手机号{self.next_phone}登录成功")
515
+ login_success = True
516
+ break
517
+ else:
518
+ logger.warning(f"✗ 设备{self.device.serial}使用手机号{self.next_phone}登录失败")
519
+
520
+ if login_success:
521
+ continue
522
+ else:
523
+ logger.warning(f"✗ 设备{self.device.serial}尝试所有手机号均登录失败")
524
+ attempted_devices.append(self.device.serial) # 记录尝试过的设备
525
+ failed_device_serial = self.device.serial
526
+ self.device = None
527
+
528
+ # 手动记录失败的设备信息
529
+ if failed_device_serial not in [d['serial'] for d in self.problematic_devices]:
530
+ self.problematic_devices.append({
531
+ 'serial': failed_device_serial,
532
+ 'reason': '设备登录失败',
533
+ 'note_id': note_id,
534
+ 'timestamp': time.time()
535
+ })
536
+
537
+ # 尝试切换到下一个设备
538
+ if not self.switch_to_next_device():
539
+ logger.error("没有更多可用设备,提取失败")
540
+ self._time_method("extract_note_data", start_time)
541
+ return {}
542
+ continue
543
+ except ImportError as e:
544
+ logger.warning(f"无法导入登录模块: {e}")
545
+ logger.warning("将尝试跳过登录步骤,继续使用当前设备")
546
+ continue
547
+
548
+ logger.info(f"✓ 成功提取笔记数据,点赞数: {data['likes']}, 图片数: {len(data['image_urls'])}")
549
+ self._time_method("extract_note_data", start_time)
550
+ return data
551
+
552
+ except Exception as e:
553
+ logger.error(f"✗ 提取笔记数据失败: {e}")
554
+ attempted_devices.append(self.device.serial if self.device else "未知设备")
555
+
556
+ # 记录有问题的设备
557
+ if self.device and self.device.serial not in [d['serial'] for d in self.problematic_devices]:
558
+ self.problematic_devices.append({
559
+ 'serial': self.device.serial,
560
+ 'reason': f'提取异常: {str(e)}',
561
+ 'note_id': note_id,
562
+ 'timestamp': time.time()
563
+ })
564
+
565
+ # 如果还有设备可用,尝试下一个
566
+ if attempt < max_retries - 1 and self.switch_to_next_device():
567
+ continue
568
+ else:
569
+ logger.error("所有设备尝试完毕,提取失败")
570
+ self._time_method("extract_note_data", start_time)
571
+ logger.error(f"所有设备尝试完毕,提取失败。尝试过的设备: {attempted_devices}")
572
+ self._time_method("extract_note_data", start_time)
573
+ return {}
574
+
575
+ def _get_detail_data(self, jump_url: str) -> Dict[str, Union[str, List[str]]]:
576
+ """
577
+ 从当前已经打开的小红书详情页提取完整正文、图片和点赞数。
578
+ 优化版本: 使用 dump_hierarchy 替代遍历,大幅提升速度。
579
+
580
+ Args:
581
+ jump_url (str): 笔记的跳转URL,用于白屏时重新加载
582
+
583
+ Returns:
584
+ Dict[str, Union[str, List[str]]]: 包含笔记数据的字典
585
+ """
586
+ start_time = time.time()
587
+ logger.info("🔍 进入深度提取模式 (XML优化版)...")
588
+
589
+ # 1. 验证是否进入详情页 & 展开全文
590
+ detail_loaded = False
591
+ try:
592
+ if self.device(text="展开").exists:
593
+ self.device(text="展开").click()
594
+ except: pass
595
+
596
+ # 超快速检查 - 只等0.2秒
597
+ time.sleep(0.2)
598
+
599
+ # 快速检查登录状态
600
+ if self.device(textContains="其他登录方式").exists or self.device(textContains="微信登录").exists or self.device(textContains="登录发现更多精彩").exists:
601
+ logger.error("✗ 需要登录才能查看详情页内容,提取终止")
602
+ return None
603
+
604
+ # 极简检查 - 只检查一次
605
+ time.sleep(0.3)
606
+ detail_count = 5
607
+ detail_loaded = False
608
+ while(detail_count > 0):
609
+ if not self.device(textContains="关注").exists:
610
+ detail_count -= 1
611
+ time.sleep(0.1)
612
+ continue
613
+ detail_loaded = True
614
+ break
615
+
616
+ if not detail_loaded:
617
+ logger.warning("⚠ 警告:详情页特征未发现,提取可能不完整")
618
+
619
+ # 智能滚动 - 确保看到发布时间和评论区 (优化速度版)
620
+ scroll_phase_start = time.time()
621
+ try:
622
+ # 定义需要查找的目标元素 (正则匹配)
623
+ target_pattern = re.compile(r"条评论|发布于|小时前|天前|月前|年前|昨天|今天")
624
+
625
+ # 最多滚动6次,单次距离加大
626
+ for i in range(6):
627
+ # 向下滚动
628
+ swipe_start = time.time()
629
+ self.device.swipe(540, 1600, 540, 600, 0.1)
630
+ self._time_method(f"scroll_swipe_{i+1}", swipe_start)
631
+
632
+ # 核心优化:只 dump 一次,在字符串中搜索,避免多次 exists() 调用的开销
633
+ dump_start = time.time()
634
+ xml_temp = self.device.dump_hierarchy()
635
+ self._time_method(f"scroll_dump_{i+1}", dump_start)
636
+
637
+ if target_pattern.search(xml_temp):
638
+ logger.info(f"✓ 已检测到目标元素 (第 {i+1} 次滚动)")
639
+ break
640
+
641
+ # 极短间隔
642
+ time.sleep(0.1)
643
+
644
+ time.sleep(0.3) # 稳定时间
645
+ self._time_method("intelligent_scroll_total", scroll_phase_start)
646
+ logger.info("✓ 滚动完成")
647
+ except Exception as e:
648
+ logger.warning(f"滚动失败: {e}")
649
+
650
+ # 初始化提取变量
651
+ content = ""
652
+ likes = 0
653
+ collects = 0
654
+ comments = 0
655
+ author_name = "Unknown"
656
+ publish_time = 0
657
+ date_desc = ""
658
+ image_urls = []
659
+
660
+ # 2. 获取 UI层级 (核心优化)
661
+ # 增加一次重试逻辑,如果第一次没抓到日期
662
+ text_nodes = []
663
+ limit_y = 2500
664
+
665
+ for attempt in range(2):
666
+ xml_dump_start = time.time()
667
+ xml_content = self.device.dump_hierarchy()
668
+ self._time_method("dump_hierarchy", xml_dump_start)
669
+
670
+ # 检测白屏状态 - 检查文本节点数量
671
+ current_text_nodes = []
672
+ root = ET.fromstring(xml_content)
673
+
674
+ def parse_nodes(node):
675
+ text = node.attrib.get('text', '') or node.attrib.get('content-desc', '')
676
+ bounds_str = node.attrib.get('bounds', '[0,0][0,0]')
677
+ try:
678
+ coords = bounds_str.replace('][', ',').replace('[', '').replace(']', '').split(',')
679
+ x1, y1, x2, y2 = map(int, coords)
680
+ if text:
681
+ current_text_nodes.append({
682
+ 'text': text,
683
+ 'l': x1, 't': y1, 'r': x2, 'b': y2,
684
+ 'cx': (x1 + x2) / 2, 'cy': (y1 + y2) / 2
685
+ })
686
+ except: pass
687
+ for child in node: parse_nodes(child)
688
+
689
+ parse_nodes(root)
690
+
691
+ # 白屏检测:如果文本节点太少,可能是白屏
692
+ print(f'当前文本节点数量: {len(current_text_nodes)}')
693
+ if len(current_text_nodes) < 11:
694
+ logger.error(f"✗ 检测到白屏状态 - 文本节点数量异常少 ({len(current_text_nodes)}个节点)")
695
+ logger.info("--- 调试: 捕获的文本节点 ---")
696
+ for i, n in enumerate(current_text_nodes):
697
+ logger.info(f"[{i}] {n['text']} (t={n['t']}, b={n['b']}, l={n['l']}, r={n['r']})")
698
+ logger.info("--- 调试结束 ---")
699
+
700
+ # 如果是第一次尝试,重新加载页面
701
+ if attempt == 0:
702
+ logger.info("🔄 尝试重新加载页面...")
703
+ # 重新发送跳转指令
704
+ self.device.open_url(jump_url)
705
+ time.sleep(2) # 等待页面重新加载
706
+ continue
707
+ else:
708
+ # 第二次尝试仍白屏,直接返回None
709
+ logger.error("✗ 页面加载失败 - 白屏状态")
710
+ return None
711
+
712
+ # 检查是否存在加载指示器
713
+ loading_found = False
714
+ for node in current_text_nodes:
715
+ if re.search(r'(加载|loading|等待|waiting|\.\.\.|\\u231a|\\u25ba)', node['text'], re.IGNORECASE):
716
+ loading_found = True
717
+ break
718
+
719
+ if loading_found:
720
+ logger.warning("⚠ 检测到页面正在加载中")
721
+ if attempt == 0:
722
+ logger.info("🔄 等待页面加载完成...")
723
+ time.sleep(2)
724
+ continue
725
+
726
+ text_nodes = current_text_nodes # 保留最新的节点供后续提取使用
727
+
728
+ # 4. 分析节点数据 (简化版日期快速检查)
729
+ found_date_in_this_xml = False
730
+ follow_node = None
731
+ for n in text_nodes:
732
+ if n['text'] in ["关注", "已关注"]:
733
+ follow_node = n
734
+ break
735
+
736
+ if follow_node:
737
+ # 寻找作者名
738
+ best_dist = 999
739
+ for n in text_nodes:
740
+ if n == follow_node: continue
741
+ if abs(n['cy'] - follow_node['cy']) < 100 and n['r'] <= follow_node['l'] + 50:
742
+ dist = follow_node['l'] - n['r']
743
+ if dist < best_dist:
744
+ best_dist = dist
745
+ author_name = n['text']
746
+
747
+ # 寻找日期
748
+ min_y = follow_node['b'] if follow_node else 150
749
+ # 提前寻找 limit_y
750
+ current_limit_y = 2500
751
+ for n in text_nodes:
752
+ if re.match(r"^共\s*\d+\s*条评论$", n['text']) or n['text'] in ["说点什么", "写评论", "写点什么", "这里是评论区"]:
753
+ current_limit_y = min(current_limit_y, n['t'])
754
+ limit_y = current_limit_y
755
+
756
+ for n in text_nodes:
757
+ if n['t'] > min_y - 200 and n['b'] < limit_y + 150:
758
+ txt = n['text'].strip()
759
+ if 2 <= len(txt) <= 50 and txt not in ["点赞", "收藏", "评论", "关注", "分享", "回复", "不喜欢"]:
760
+ try:
761
+ ts = parse_time_to_timestamp_ms(txt)
762
+ publish_time = ts
763
+ date_desc = txt
764
+ found_date_in_this_xml = True
765
+ # 不要 break,因为日期通常在最后
766
+ except: continue
767
+
768
+ if found_date_in_this_xml:
769
+ break
770
+
771
+ if attempt == 0:
772
+ logger.warning("⚠ 未识别到发布时间,尝试额外滚动并重试...")
773
+ self.device.swipe(540, 1500, 540, 1000, 0.2)
774
+ time.sleep(0.5)
775
+
776
+ if not date_desc:
777
+ logger.warning("未识别到发布时间")
778
+ # 埋点调试: 打印出识别到的所有节点及其坐标
779
+ logger.info("--- 调试: 所有捕获的文本节点 ---")
780
+ for i, n in enumerate(text_nodes):
781
+ logger.info(f"[{i}] {n['text']} (t={n['t']}, b={n['b']}, l={n['l']}, r={n['r']})")
782
+ logger.info("--- 调试结束 ---")
783
+ else:
784
+ logger.info(f"✓ 识别到发布时间: {date_desc} -> {publish_time}")
785
+
786
+ logger.info(f"text_nodes: {text_nodes}")
787
+
788
+
789
+ # B. 互动数据提取 (底部区域)
790
+ # 使用 limit_y 作为分割线大概率更准确
791
+ bottom_nodes = [n for n in text_nodes if n['t'] >= limit_y - 300] # 互动栏通常在 limit_y 上方一点点 或者 就在 mask 区域
792
+ bottom_nodes.sort(key=lambda x: x['l']) # 从左到右
793
+
794
+ for n in bottom_nodes:
795
+ txt = n['text']
796
+ # 保留数字、小数点、w/W 和 "万" 字
797
+ num_txt = ''.join(c for c in txt if c.isdigit() or c in ['.', 'w', 'W', '万'])
798
+ if not num_txt: continue
799
+
800
+ cx = n['cx']
801
+ if 500 < cx < 750:
802
+ likes = parse_count_to_int(num_txt)
803
+ elif 750 < cx < 900:
804
+ collects = parse_count_to_int(num_txt)
805
+ elif cx >= 900:
806
+ comments = parse_count_to_int(num_txt)
807
+
808
+ # C. 正文提取
809
+ # 过滤掉非正文内容
810
+ content_lines = []
811
+ # exclude_keywords = ['收藏', '点赞', '评论', '分享', '发布于', '说点什么', '条评论', '关注', author_name]
812
+ # if date_desc:
813
+ # exclude_keywords.append(date_desc)
814
+
815
+ # 按照垂直位置排序 (使用 min_y 和 limit_y 约束)
816
+ content_nodes = [n for n in text_nodes if min_y < n['t'] < limit_y]
817
+ content_nodes.sort(key=lambda x: x['t'])
818
+
819
+ for n in content_nodes:
820
+ t = n['text']
821
+ if len(t) < 2: continue
822
+ # if any(k in t for k in exclude_keywords): continue
823
+
824
+ # 简单的去重策略
825
+ if content_lines and t in content_lines[-1]: continue
826
+ content_lines.append(t)
827
+
828
+ content = "\n".join(content_lines)
829
+ logger.info(f"提取正文: {content}")
830
+ # 5. 图片提取 (保持原有逻辑但优化等待)
831
+ try:
832
+ # 这里还是需要交互,无法纯靠XML
833
+ share_btn = self.device(description="分享")
834
+ if share_btn.exists:
835
+ share_btn.click()
836
+ # 显式等待 "复制链接"
837
+ copy_link = self.device(text="复制链接")
838
+ if copy_link.wait(timeout=2.0):
839
+ copy_link.click()
840
+ # 等待剪贴板更新? 稍微缓一下
841
+ time.sleep(0.5)
842
+ share_link = self.device.clipboard
843
+ if "http" in str(share_link):
844
+ image_urls = self._fetch_web_images(share_link)
845
+ else:
846
+ logger.warning("未找到复制链接按钮")
847
+ self.device.press("back")
848
+ except Exception as e:
849
+ logger.warning(f"⚠ 图片提取异常: {e}")
850
+
851
+ self._time_method("_get_detail_data", start_time)
852
+ return {
853
+ "content": content,
854
+ "image_urls": image_urls,
855
+ "likes": likes,
856
+ "collects": collects,
857
+ "comments": comments,
858
+ "author_name": author_name,
859
+ "publish_time": publish_time,
860
+ "date_desc": date_desc
861
+ }
862
+
863
+ def _fetch_web_images(self, url: str) -> List[str]:
864
+ """
865
+ 从分享链接中解析图片地址
866
+
867
+ Args:
868
+ url (str): 分享链接URL
869
+
870
+ Returns:
871
+ List[str]: 图片URL列表
872
+ """
873
+ start_time = time.time()
874
+ try:
875
+ headers = {"User-Agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 14_8 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.2 Mobile/15E148 Safari/604.1"}
876
+ res = requests.get(url, headers=headers, timeout=10)
877
+ html = res.text
878
+ img_patterns = [
879
+ r'property="og:image" content="(https://[^"]+)"',
880
+ r'"url":"(https://sns-img-[^"]+)"',
881
+ r'"url":"(https://sns-img-qc\.xhscdn\.com/[^"]+)"'
882
+ ]
883
+ found = []
884
+ for pattern in img_patterns:
885
+ matches = re.findall(pattern, html)
886
+ for m in matches:
887
+ clean_url = m.replace('\\u002F', '/')
888
+ if clean_url not in found: found.append(clean_url)
889
+ self._time_method("_fetch_web_images", start_time)
890
+ return found
891
+ except:
892
+ self._time_method("_fetch_web_images", start_time)
893
+ return []
894
+
895
+ def save_note_data(self, data: Dict[str, Union[str, List[str]]],
896
+ filename: str = "last_extracted_note.txt",
897
+ note_url: str = "") -> None:
898
+ """
899
+ 保存笔记数据到文件
900
+
901
+ Args:
902
+ data (Dict[str, Union[str, List[str]]]): 笔记数据
903
+ filename (str): 保存文件名
904
+ note_url (str): 笔记URL
905
+ """
906
+ start_time = time.time()
907
+ try:
908
+ with open(filename, "w", encoding="utf-8") as f:
909
+ f.write("=" * 50 + "\n")
910
+ f.write("【小红书笔记提取结果】\n")
911
+ f.write("=" * 50 + "\n")
912
+ if note_url:
913
+ f.write(f"笔记URL: {note_url}\n")
914
+ f.write("=" * 50 + "\n")
915
+ f.write(f"作者: {data.get('author_name', 'Unknown')}\n")
916
+ f.write(f"点赞数: {data.get('likes', '0')}\n")
917
+ f.write(f"收藏数: {data.get('collects', '0')}\n")
918
+ f.write(f"评论数: {data.get('comments', '0')}\n")
919
+ f.write(f"图片数: {len(data.get('image_urls', []))}\n")
920
+ f.write(f"发布时间: {data.get('date_desc', '')} ({data.get('publish_time', 0)})\n")
921
+ f.write("=" * 50 + "\n")
922
+ f.write("【正文内容】\n")
923
+ f.write(data['content'])
924
+ f.write("\n" + "=" * 50 + "\n")
925
+ if data['image_urls']:
926
+ f.write("【图片URL】\n")
927
+ for i, url in enumerate(data['image_urls'], 1):
928
+ f.write(f"{i}. {url}\n")
929
+ f.write("=" * 50 + "\n")
930
+
931
+ logger.info(f"✓ 笔记数据已保存到: {filename}")
932
+ self._time_method("save_note_data", start_time)
933
+ except Exception as e:
934
+ logger.error(f"✗ 保存笔记数据失败: {e}")
935
+ self._time_method("save_note_data", start_time)
936
+ raise
937
+
938
+
939
+ def extract_note_from_url(url: str, device_serial: Optional[str] = None, enable_time_logging: bool = True) -> Optional[Dict[str, Union[str, List[str]]]]:
940
+ """
941
+ 便捷函数:直接从URL提取笔记数据,支持设备重试机制
942
+
943
+ Args:
944
+ url (str): 小红书笔记URL
945
+ device_serial (str, optional): 设备序列号
946
+ enable_time_logging (bool, optional): 是否启用耗时打印,默认为True
947
+
948
+ Returns:
949
+ Optional[Dict[str, Union[str, List[str]]]]: 笔记数据,如果没有成功则返回None
950
+ """
951
+ start_time = time.time()
952
+ logger.info(f"[extract_note_from_url] 开始处理URL: {url}")
953
+ try:
954
+ extractor = XHSNoteExtractor(device_serial=device_serial, enable_time_logging=enable_time_logging)
955
+ result = extractor.extract_note_data(url=url)
956
+ elapsed_time = time.time() - start_time
957
+ logger.info(f"[extract_note_from_url] 总耗时: {elapsed_time:.3f}秒")
958
+ return result
959
+ except Exception as e:
960
+ logger.error(f"[extract_note_from_url] 提取失败: {e}")
961
+ elapsed_time = time.time() - start_time
962
+ logger.info(f"[extract_note_from_url] 总耗时: {elapsed_time:.3f}秒")
963
+ return None
964
+
965
+
966
+ def convert_url_format(url: str) -> str:
967
+ """
968
+ 便捷函数:转换URL格式
969
+
970
+ Args:
971
+ url (str): 输入URL
972
+
973
+ Returns:
974
+ str: 转换后的xhsdiscover协议格式URL
975
+ """
976
+ start_time = time.time()
977
+ logger.info(f"[convert_url_format] 开始转换URL: {url}")
978
+ parsed_data = XHSNoteExtractor.parse_xhs_url(url)
979
+ result = XHSNoteExtractor.convert_to_xhsdiscover_format(
980
+ parsed_data["note_id"],
981
+ parsed_data["xsec_token"]
982
+ )
983
+ elapsed_time = time.time() - start_time
984
+ logger.info(f"[convert_url_format] 耗时: {elapsed_time:.3f}秒,结果: {result}")
985
+ return result