vibesurf 0.1.25__py3-none-any.whl → 0.1.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- vibe_surf/_version.py +2 -2
- vibe_surf/agents/vibe_surf_agent.py +4 -5
- vibe_surf/backend/api/task.py +1 -1
- vibe_surf/backend/database/queries.py +2 -2
- vibe_surf/backend/utils/llm_factory.py +1 -1
- vibe_surf/browser/agent_browser_session.py +26 -0
- vibe_surf/cli.py +1 -1
- vibe_surf/llm/openai_compatible.py +1 -1
- vibe_surf/tools/browser_use_tools.py +168 -1
- vibe_surf/tools/vibesurf_tools.py +463 -21
- vibe_surf/tools/views.py +75 -0
- {vibesurf-0.1.25.dist-info → vibesurf-0.1.27.dist-info}/METADATA +26 -5
- {vibesurf-0.1.25.dist-info → vibesurf-0.1.27.dist-info}/RECORD +17 -17
- vibesurf-0.1.27.dist-info/licenses/LICENSE +22 -0
- vibesurf-0.1.25.dist-info/licenses/LICENSE +0 -201
- {vibesurf-0.1.25.dist-info → vibesurf-0.1.27.dist-info}/WHEEL +0 -0
- {vibesurf-0.1.25.dist-info → vibesurf-0.1.27.dist-info}/entry_points.txt +0 -0
- {vibesurf-0.1.25.dist-info → vibesurf-0.1.27.dist-info}/top_level.txt +0 -0
|
@@ -31,7 +31,8 @@ from browser_use.tools.views import NoParamsAction
|
|
|
31
31
|
from vibe_surf.browser.agent_browser_session import AgentBrowserSession
|
|
32
32
|
from vibe_surf.tools.views import HoverAction, ExtractionAction, FileExtractionAction, BrowserUseAgentExecution, \
|
|
33
33
|
ReportWriterTask, TodoGenerateAction, TodoModifyAction, VibeSurfDoneAction, SkillSearchAction, SkillCrawlAction, \
|
|
34
|
-
SkillSummaryAction, SkillTakeScreenshotAction, SkillDeepResearchAction, SkillCodeAction, SkillFinanceAction
|
|
34
|
+
SkillSummaryAction, SkillTakeScreenshotAction, SkillDeepResearchAction, SkillCodeAction, SkillFinanceAction, \
|
|
35
|
+
SkillXhsAction, SkillDouyinAction, SkillYoutubeAction, SkillWeiboAction
|
|
35
36
|
from vibe_surf.tools.finance_tools import FinanceDataRetriever, FinanceMarkdownFormatter, FinanceMethod
|
|
36
37
|
from vibe_surf.tools.mcp_client import CustomMCPClient
|
|
37
38
|
from vibe_surf.tools.file_system import CustomFileSystem
|
|
@@ -231,7 +232,9 @@ Example format: ["query 1", "query 2", "query 3", "query 4", "query 5", "query 6
|
|
|
231
232
|
# Fallback to simple queries if parsing fails
|
|
232
233
|
try:
|
|
233
234
|
from json_repair import repair_json
|
|
234
|
-
|
|
235
|
+
search_queries_s = repair_json(response.completion.strip())
|
|
236
|
+
search_queries = json.loads(search_queries_s)
|
|
237
|
+
search_queries = search_queries[:query_num]
|
|
235
238
|
except Exception as e:
|
|
236
239
|
search_queries = [
|
|
237
240
|
params.query,
|
|
@@ -244,7 +247,7 @@ Example format: ["query 1", "query 2", "query 3", "query 4", "query 5", "query 6
|
|
|
244
247
|
# Step 2: Create browser sessions for parallel searching
|
|
245
248
|
register_sessions = []
|
|
246
249
|
|
|
247
|
-
for i, query in enumerate(search_queries):
|
|
250
|
+
for i, query in enumerate(search_queries[:query_num]):
|
|
248
251
|
agent_id = f"search_agent_{i + 1:03d}"
|
|
249
252
|
register_sessions.append(
|
|
250
253
|
browser_manager.register_agent(agent_id, target_id=None)
|
|
@@ -278,41 +281,59 @@ Example format: ["query 1", "query 2", "query 3", "query 4", "query 5", "query 6
|
|
|
278
281
|
# Step 5: Use LLM only for final ranking and selection (much smaller dataset now)
|
|
279
282
|
if all_results and len(all_results) > 10:
|
|
280
283
|
# Only use LLM if we have more than 10 results to rank
|
|
284
|
+
# Create indexed results for LLM prompt
|
|
285
|
+
indexed_results = []
|
|
286
|
+
for i, result in enumerate(all_results):
|
|
287
|
+
indexed_results.append({
|
|
288
|
+
"index": i,
|
|
289
|
+
"title": result.get('title', 'Unknown Title'),
|
|
290
|
+
"url": result.get('url', 'No URL'),
|
|
291
|
+
"summary": result.get('summary', 'No summary available')
|
|
292
|
+
})
|
|
293
|
+
|
|
281
294
|
ranking_prompt = f"""
|
|
282
295
|
Rank these search results for the query "{params.query}" by relevance and value.
|
|
283
296
|
Select the TOP 10 most relevant and valuable results.
|
|
284
297
|
|
|
285
|
-
Search Results ({len(
|
|
286
|
-
{json.dumps(
|
|
298
|
+
Search Results ({len(indexed_results)} total):
|
|
299
|
+
{json.dumps(indexed_results, indent=2, ensure_ascii=False)}
|
|
287
300
|
|
|
288
|
-
Return the top 10 results as a JSON array
|
|
289
|
-
|
|
290
|
-
- url: string
|
|
291
|
-
- summary: string (brief description of why this result is valuable)
|
|
301
|
+
Return ONLY the indices of the top 10 results as a JSON array of numbers.
|
|
302
|
+
For example: [0, 5, 2, 8, 1, 9, 3, 7, 4, 6]
|
|
292
303
|
|
|
293
|
-
Format: [
|
|
304
|
+
Format: [index1, index2, index3, ...]
|
|
294
305
|
"""
|
|
295
306
|
|
|
296
307
|
ranking_response = await llm.ainvoke([
|
|
297
308
|
SystemMessage(
|
|
298
|
-
content="You are an expert at ranking search results for relevance and value."),
|
|
309
|
+
content="You are an expert at ranking search results for relevance and value. Return only the indices of the top results."),
|
|
299
310
|
UserMessage(content=ranking_prompt)
|
|
300
311
|
])
|
|
301
312
|
|
|
302
313
|
try:
|
|
303
|
-
|
|
304
|
-
if not isinstance(
|
|
314
|
+
selected_indices = json.loads(ranking_response.completion.strip())
|
|
315
|
+
if not isinstance(selected_indices, list):
|
|
305
316
|
raise ValueError("Invalid ranking results format")
|
|
306
|
-
|
|
317
|
+
# Ensure indices are valid and limit to 10
|
|
318
|
+
valid_indices = [i for i in selected_indices if isinstance(i, int) and 0 <= i < len(all_results)][:10]
|
|
319
|
+
if valid_indices:
|
|
320
|
+
top_results = [all_results[i] for i in valid_indices]
|
|
321
|
+
else:
|
|
322
|
+
top_results = all_results[:10]
|
|
307
323
|
except (json.JSONDecodeError, ValueError):
|
|
308
324
|
try:
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
325
|
+
selected_indices_s = repair_json(ranking_response.completion.strip())
|
|
326
|
+
selected_indices = json.loads(selected_indices_s)
|
|
327
|
+
if isinstance(selected_indices, list):
|
|
328
|
+
valid_indices = [i for i in selected_indices if isinstance(i, int) and 0 <= i < len(all_results)][:10]
|
|
329
|
+
if valid_indices:
|
|
330
|
+
top_results = [all_results[i] for i in valid_indices]
|
|
331
|
+
else:
|
|
332
|
+
top_results = all_results[:10]
|
|
312
333
|
else:
|
|
313
334
|
top_results = all_results[:10]
|
|
314
335
|
except Exception:
|
|
315
|
-
# Fallback to first 10
|
|
336
|
+
# Fallback to first 10 results
|
|
316
337
|
top_results = all_results[:10]
|
|
317
338
|
elif all_results:
|
|
318
339
|
# If we have 10 or fewer results, skip LLM ranking
|
|
@@ -496,7 +517,7 @@ Format: [{{"title": "...", "url": "...", "summary": "..."}}, ...]
|
|
|
496
517
|
with open(filepath, "wb") as f:
|
|
497
518
|
f.write(base64.b64decode(screenshot))
|
|
498
519
|
|
|
499
|
-
msg = f'📸 Screenshot saved to path: {str(filepath.relative_to(fs_dir))}'
|
|
520
|
+
msg = f'📸 Screenshot saved to path: [{filename}]({str(filepath.relative_to(fs_dir))})'
|
|
500
521
|
logger.info(msg)
|
|
501
522
|
return ActionResult(
|
|
502
523
|
extracted_content=msg,
|
|
@@ -922,7 +943,428 @@ Please generate alternative JavaScript code that avoids this system error:"""
|
|
|
922
943
|
except Exception as e:
|
|
923
944
|
error_msg = f'❌ Failed to retrieve financial data for {params.symbol}: {str(e)}'
|
|
924
945
|
logger.error(error_msg)
|
|
925
|
-
return ActionResult(error=error_msg)
|
|
946
|
+
return ActionResult(error=error_msg, extracted_content=error_msg)
|
|
947
|
+
|
|
948
|
+
|
|
949
|
+
@self.registry.action(
|
|
950
|
+
'Skill: Xiaohongshu API - Access Xiaohongshu (Little Red Book) platform data including search, content details, comments, user profiles, and recommendations. Methods: search_content_by_keyword, fetch_content_details, fetch_all_content_comments, get_user_profile, fetch_all_user_content, get_home_recommendations.',
|
|
951
|
+
param_model=SkillXhsAction,
|
|
952
|
+
)
|
|
953
|
+
async def skill_xhs(
|
|
954
|
+
params: SkillXhsAction,
|
|
955
|
+
browser_manager: BrowserManager,
|
|
956
|
+
file_system: CustomFileSystem
|
|
957
|
+
):
|
|
958
|
+
"""
|
|
959
|
+
Skill: Xiaohongshu API integration
|
|
960
|
+
|
|
961
|
+
Available methods:
|
|
962
|
+
- search_content_by_keyword: Search content by keyword with sorting options
|
|
963
|
+
- fetch_content_details: Get detailed information about specific content
|
|
964
|
+
- fetch_all_content_comments: Get all comments for specific content
|
|
965
|
+
- get_user_profile: Get user profile information
|
|
966
|
+
- fetch_all_user_content: Get all content posted by a user
|
|
967
|
+
- get_home_recommendations: Get homepage recommended content
|
|
968
|
+
"""
|
|
969
|
+
try:
|
|
970
|
+
from vibe_surf.tools.website_api.xhs.client import XiaoHongShuApiClient
|
|
971
|
+
|
|
972
|
+
# Initialize client
|
|
973
|
+
xhs_client = XiaoHongShuApiClient(browser_session=browser_manager.main_browser_session)
|
|
974
|
+
await xhs_client.setup()
|
|
975
|
+
|
|
976
|
+
# Parse params JSON string
|
|
977
|
+
import json
|
|
978
|
+
from json_repair import repair_json
|
|
979
|
+
try:
|
|
980
|
+
method_params = json.loads(params.params)
|
|
981
|
+
except json.JSONDecodeError:
|
|
982
|
+
method_params = json.loads(repair_json(params.params))
|
|
983
|
+
|
|
984
|
+
# Execute the requested method
|
|
985
|
+
result = None
|
|
986
|
+
if params.method == "search_content_by_keyword":
|
|
987
|
+
result = await xhs_client.search_content_by_keyword(**method_params)
|
|
988
|
+
elif params.method == "fetch_content_details":
|
|
989
|
+
result = await xhs_client.fetch_content_details(**method_params)
|
|
990
|
+
elif params.method == "fetch_all_content_comments":
|
|
991
|
+
result = await xhs_client.fetch_all_content_comments(**method_params)
|
|
992
|
+
elif params.method == "get_user_profile":
|
|
993
|
+
result = await xhs_client.get_user_profile(**method_params)
|
|
994
|
+
elif params.method == "fetch_all_user_content":
|
|
995
|
+
result = await xhs_client.fetch_all_user_content(**method_params)
|
|
996
|
+
elif params.method == "get_home_recommendations":
|
|
997
|
+
result = await xhs_client.get_home_recommendations()
|
|
998
|
+
else:
|
|
999
|
+
return ActionResult(error=f"Unknown method: {params.method}")
|
|
1000
|
+
|
|
1001
|
+
# Save result to file
|
|
1002
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
1003
|
+
filename = f"xhs_{params.method}_{timestamp}.json"
|
|
1004
|
+
filepath = file_system.get_dir() / "data" / filename
|
|
1005
|
+
filepath.parent.mkdir(exist_ok=True)
|
|
1006
|
+
|
|
1007
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
1008
|
+
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
1009
|
+
|
|
1010
|
+
# Format result as markdown
|
|
1011
|
+
if isinstance(result, list):
|
|
1012
|
+
display_count = min(5, len(result))
|
|
1013
|
+
md_content = f"## Xiaohongshu {params.method.replace('_', ' ').title()}\n\n"
|
|
1014
|
+
md_content += f"Showing {display_count} of {len(result)} results:\n\n"
|
|
1015
|
+
for i, item in enumerate(result[:display_count]):
|
|
1016
|
+
md_content += f"### Result {i+1}\n"
|
|
1017
|
+
for key, value in item.items():
|
|
1018
|
+
if not value:
|
|
1019
|
+
continue
|
|
1020
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1021
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1022
|
+
else:
|
|
1023
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1024
|
+
md_content += "\n"
|
|
1025
|
+
else:
|
|
1026
|
+
md_content = f"## Xiaohongshu {params.method.replace('_', ' ').title()}\n\n"
|
|
1027
|
+
for key, value in result.items():
|
|
1028
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1029
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1030
|
+
else:
|
|
1031
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1032
|
+
md_content += "\n"
|
|
1033
|
+
|
|
1034
|
+
# Add file path to markdown
|
|
1035
|
+
relative_path = str(filepath.relative_to(file_system.get_dir()))
|
|
1036
|
+
md_content += f"\n> 📁 Full data saved to: [{filename}]({relative_path})\n"
|
|
1037
|
+
md_content += f"> 💡 Click the link above to view all results.\n"
|
|
1038
|
+
|
|
1039
|
+
logger.info(f'📕 Xiaohongshu data retrieved with method: {params.method}')
|
|
1040
|
+
|
|
1041
|
+
# Close client
|
|
1042
|
+
await xhs_client.close()
|
|
1043
|
+
|
|
1044
|
+
return ActionResult(
|
|
1045
|
+
extracted_content=md_content
|
|
1046
|
+
)
|
|
1047
|
+
|
|
1048
|
+
except Exception as e:
|
|
1049
|
+
error_msg = f'❌ Failed to retrieve Xiaohongshu data: {str(e)}'
|
|
1050
|
+
logger.error(error_msg)
|
|
1051
|
+
return ActionResult(error=error_msg, extracted_content=error_msg)
|
|
1052
|
+
|
|
1053
|
+
|
|
1054
|
+
@self.registry.action(
|
|
1055
|
+
'Skill: Weibo API - Access Weibo platform data including search, post details, comments, user profiles, hot posts, and trending lists. Methods: search_posts_by_keyword, get_post_detail, get_all_post_comments, get_user_info, get_all_user_posts, get_hot_posts(推荐榜), get_trending_posts(热搜榜).',
|
|
1056
|
+
param_model=SkillWeiboAction,
|
|
1057
|
+
)
|
|
1058
|
+
async def skill_weibo(
|
|
1059
|
+
params: SkillWeiboAction,
|
|
1060
|
+
browser_manager: BrowserManager,
|
|
1061
|
+
file_system: CustomFileSystem
|
|
1062
|
+
):
|
|
1063
|
+
"""
|
|
1064
|
+
Skill: Weibo API integration
|
|
1065
|
+
|
|
1066
|
+
Available methods:
|
|
1067
|
+
- search_posts_by_keyword: Search posts by keyword with sorting options
|
|
1068
|
+
- get_post_detail: Get detailed information about specific post
|
|
1069
|
+
- get_all_post_comments: Get all comments for specific post
|
|
1070
|
+
- get_user_info: Get user profile information
|
|
1071
|
+
- get_all_user_posts: Get all posts by a user
|
|
1072
|
+
- get_hot_posts: Get hot posts
|
|
1073
|
+
- get_trending_list: Get trending list
|
|
1074
|
+
"""
|
|
1075
|
+
try:
|
|
1076
|
+
from vibe_surf.tools.website_api.weibo.client import WeiboApiClient
|
|
1077
|
+
|
|
1078
|
+
# Initialize client
|
|
1079
|
+
wb_client = WeiboApiClient(browser_session=browser_manager.main_browser_session)
|
|
1080
|
+
await wb_client.setup()
|
|
1081
|
+
|
|
1082
|
+
# Parse params JSON string
|
|
1083
|
+
import json
|
|
1084
|
+
from json_repair import repair_json
|
|
1085
|
+
try:
|
|
1086
|
+
method_params = json.loads(params.params)
|
|
1087
|
+
except json.JSONDecodeError:
|
|
1088
|
+
method_params = json.loads(repair_json(params.params))
|
|
1089
|
+
|
|
1090
|
+
# Execute the requested method
|
|
1091
|
+
result = None
|
|
1092
|
+
if params.method == "search_posts_by_keyword":
|
|
1093
|
+
result = await wb_client.search_posts_by_keyword(**method_params)
|
|
1094
|
+
elif params.method == "get_post_detail":
|
|
1095
|
+
result = await wb_client.get_post_detail(**method_params)
|
|
1096
|
+
elif params.method == "get_all_post_comments":
|
|
1097
|
+
result = await wb_client.get_all_post_comments(**method_params)
|
|
1098
|
+
elif params.method == "get_user_info":
|
|
1099
|
+
result = await wb_client.get_user_info(**method_params)
|
|
1100
|
+
elif params.method == "get_all_user_posts":
|
|
1101
|
+
result = await wb_client.get_all_user_posts(**method_params)
|
|
1102
|
+
elif params.method == "get_hot_posts":
|
|
1103
|
+
result = await wb_client.get_hot_posts()
|
|
1104
|
+
elif params.method == "get_trending_posts":
|
|
1105
|
+
result = await wb_client.get_trending_posts()
|
|
1106
|
+
else:
|
|
1107
|
+
return ActionResult(error=f"Unknown method: {params.method}")
|
|
1108
|
+
|
|
1109
|
+
# Save result to file
|
|
1110
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
1111
|
+
filename = f"weibo_{params.method}_{timestamp}.json"
|
|
1112
|
+
filepath = file_system.get_dir() / "data" / filename
|
|
1113
|
+
filepath.parent.mkdir(exist_ok=True)
|
|
1114
|
+
|
|
1115
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
1116
|
+
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
1117
|
+
# Format result as markdown
|
|
1118
|
+
if isinstance(result, list):
|
|
1119
|
+
display_count = min(5, len(result))
|
|
1120
|
+
md_content = f"## Weibo {params.method.replace('_', ' ').title()}\n\n"
|
|
1121
|
+
md_content += f"Showing {display_count} of {len(result)} results:\n\n"
|
|
1122
|
+
for i, item in enumerate(result[:display_count]):
|
|
1123
|
+
md_content += f"### Result {i+1}\n"
|
|
1124
|
+
for key, value in item.items():
|
|
1125
|
+
if not value:
|
|
1126
|
+
continue
|
|
1127
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1128
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1129
|
+
else:
|
|
1130
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1131
|
+
md_content += "\n"
|
|
1132
|
+
else:
|
|
1133
|
+
md_content = f"## Weibo {params.method.replace('_', ' ').title()}\n\n"
|
|
1134
|
+
for key, value in result.items():
|
|
1135
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1136
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1137
|
+
else:
|
|
1138
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1139
|
+
md_content += "\n"
|
|
1140
|
+
|
|
1141
|
+
# Add file path to markdown
|
|
1142
|
+
relative_path = str(filepath.relative_to(file_system.get_dir()))
|
|
1143
|
+
md_content += f"\n> 📁 Full data saved to: [{filename}]({relative_path})\n"
|
|
1144
|
+
md_content += f"> 💡 Click the link above to view all results.\n"
|
|
1145
|
+
|
|
1146
|
+
logger.info(f'🐦 Weibo data retrieved with method: {params.method}')
|
|
1147
|
+
|
|
1148
|
+
# Close client
|
|
1149
|
+
await wb_client.close()
|
|
1150
|
+
|
|
1151
|
+
return ActionResult(
|
|
1152
|
+
extracted_content=md_content
|
|
1153
|
+
)
|
|
1154
|
+
|
|
1155
|
+
except Exception as e:
|
|
1156
|
+
import traceback
|
|
1157
|
+
traceback.print_exc()
|
|
1158
|
+
error_msg = f'❌ Failed to retrieve Weibo data: {str(e)}. \nMost likely you are not login, please go to: [Weibo login page](https://passport.weibo.com/sso/signin?entry=miniblog&source=miniblog) and login.'
|
|
1159
|
+
logger.error(error_msg)
|
|
1160
|
+
return ActionResult(error=error_msg, extracted_content=error_msg)
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
@self.registry.action(
|
|
1164
|
+
'Skill: Douyin API - Access Douyin platform data including search, video details, comments, user profiles, and videos. Methods: search_content_by_keyword, fetch_video_details, fetch_all_video_comments, fetch_user_info, fetch_all_user_videos.',
|
|
1165
|
+
param_model=SkillDouyinAction,
|
|
1166
|
+
)
|
|
1167
|
+
async def skill_douyin(
|
|
1168
|
+
params: SkillDouyinAction,
|
|
1169
|
+
browser_manager: BrowserManager,
|
|
1170
|
+
file_system: CustomFileSystem
|
|
1171
|
+
):
|
|
1172
|
+
"""
|
|
1173
|
+
Skill: Douyin API integration
|
|
1174
|
+
|
|
1175
|
+
Available methods:
|
|
1176
|
+
- search_content_by_keyword: Search content by keyword with filtering options
|
|
1177
|
+
- fetch_video_details: Get detailed information about specific video
|
|
1178
|
+
- fetch_all_video_comments: Get all comments for specific video
|
|
1179
|
+
- fetch_user_info: Get user profile information
|
|
1180
|
+
- fetch_all_user_videos: Get all videos by a user
|
|
1181
|
+
"""
|
|
1182
|
+
try:
|
|
1183
|
+
from vibe_surf.tools.website_api.douyin.client import DouyinApiClient
|
|
1184
|
+
|
|
1185
|
+
# Initialize client
|
|
1186
|
+
dy_client = DouyinApiClient(browser_session=browser_manager.main_browser_session)
|
|
1187
|
+
await dy_client.setup()
|
|
1188
|
+
|
|
1189
|
+
# Parse params JSON string
|
|
1190
|
+
import json
|
|
1191
|
+
from json_repair import repair_json
|
|
1192
|
+
try:
|
|
1193
|
+
method_params = json.loads(params.params)
|
|
1194
|
+
except json.JSONDecodeError:
|
|
1195
|
+
method_params = json.loads(repair_json(params.params))
|
|
1196
|
+
|
|
1197
|
+
# Execute the requested method
|
|
1198
|
+
result = None
|
|
1199
|
+
if params.method == "search_content_by_keyword":
|
|
1200
|
+
result = await dy_client.search_content_by_keyword(**method_params)
|
|
1201
|
+
elif params.method == "fetch_video_details":
|
|
1202
|
+
result = await dy_client.fetch_video_details(**method_params)
|
|
1203
|
+
elif params.method == "fetch_all_video_comments":
|
|
1204
|
+
result = await dy_client.fetch_all_video_comments(**method_params)
|
|
1205
|
+
elif params.method == "fetch_user_info":
|
|
1206
|
+
result = await dy_client.fetch_user_info(**method_params)
|
|
1207
|
+
elif params.method == "fetch_all_user_videos":
|
|
1208
|
+
result = await dy_client.fetch_all_user_videos(**method_params)
|
|
1209
|
+
else:
|
|
1210
|
+
return ActionResult(error=f"Unknown method: {params.method}")
|
|
1211
|
+
|
|
1212
|
+
# Save result to file
|
|
1213
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
1214
|
+
filename = f"douyin_{params.method}_{timestamp}.json"
|
|
1215
|
+
filepath = file_system.get_dir() / "data" / filename
|
|
1216
|
+
filepath.parent.mkdir(exist_ok=True)
|
|
1217
|
+
|
|
1218
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
1219
|
+
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
1220
|
+
|
|
1221
|
+
# Format result as markdown
|
|
1222
|
+
if isinstance(result, list):
|
|
1223
|
+
display_count = min(5, len(result))
|
|
1224
|
+
md_content = f"## Douyin {params.method.replace('_', ' ').title()}\n\n"
|
|
1225
|
+
md_content += f"Showing {display_count} of {len(result)} results:\n\n"
|
|
1226
|
+
for i, item in enumerate(result[:display_count]):
|
|
1227
|
+
md_content += f"### Result {i+1}\n"
|
|
1228
|
+
for key, value in item.items():
|
|
1229
|
+
if not value:
|
|
1230
|
+
continue
|
|
1231
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1232
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1233
|
+
else:
|
|
1234
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1235
|
+
md_content += "\n"
|
|
1236
|
+
else:
|
|
1237
|
+
md_content = f"## Douyin {params.method.replace('_', ' ').title()}\n\n"
|
|
1238
|
+
for key, value in result.items():
|
|
1239
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1240
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1241
|
+
else:
|
|
1242
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1243
|
+
md_content += "\n"
|
|
1244
|
+
|
|
1245
|
+
# Add file path to markdown
|
|
1246
|
+
relative_path = str(filepath.relative_to(file_system.get_dir()))
|
|
1247
|
+
md_content += f"\n> 📁 Full data saved to: [{filename}]({relative_path})\n"
|
|
1248
|
+
md_content += f"> 💡 Click the link above to view all results.\n"
|
|
1249
|
+
|
|
1250
|
+
logger.info(f'🎵 Douyin data retrieved with method: {params.method}')
|
|
1251
|
+
|
|
1252
|
+
# Close client
|
|
1253
|
+
await dy_client.close()
|
|
1254
|
+
|
|
1255
|
+
return ActionResult(
|
|
1256
|
+
extracted_content=md_content
|
|
1257
|
+
)
|
|
1258
|
+
|
|
1259
|
+
except Exception as e:
|
|
1260
|
+
error_msg = f'❌ Failed to retrieve Douyin data: {str(e)}'
|
|
1261
|
+
logger.error(error_msg)
|
|
1262
|
+
return ActionResult(error=error_msg, extracted_content=error_msg)
|
|
1263
|
+
|
|
1264
|
+
|
|
1265
|
+
@self.registry.action(
|
|
1266
|
+
'Skill: YouTube API - Access YouTube platform data including search, video details, comments, channel info, and trending videos. Methods: search_videos, get_video_details, get_video_comments, get_channel_info, get_channel_videos, get_trending_videos.',
|
|
1267
|
+
param_model=SkillYoutubeAction,
|
|
1268
|
+
)
|
|
1269
|
+
async def skill_youtube(
|
|
1270
|
+
params: SkillYoutubeAction,
|
|
1271
|
+
browser_manager: BrowserManager,
|
|
1272
|
+
file_system: CustomFileSystem
|
|
1273
|
+
):
|
|
1274
|
+
"""
|
|
1275
|
+
Skill: YouTube API integration
|
|
1276
|
+
|
|
1277
|
+
Available methods:
|
|
1278
|
+
- search_videos: Search videos by keyword
|
|
1279
|
+
- get_video_details: Get detailed information about specific video
|
|
1280
|
+
- get_video_comments: Get comments for specific video
|
|
1281
|
+
- get_channel_info: Get channel information
|
|
1282
|
+
- get_channel_videos: Get videos from specific channel
|
|
1283
|
+
- get_trending_videos: Get trending videos
|
|
1284
|
+
"""
|
|
1285
|
+
try:
|
|
1286
|
+
from vibe_surf.tools.website_api.youtube.client import YouTubeApiClient
|
|
1287
|
+
|
|
1288
|
+
# Initialize client
|
|
1289
|
+
yt_client = YouTubeApiClient(browser_session=browser_manager.main_browser_session)
|
|
1290
|
+
await yt_client.setup()
|
|
1291
|
+
|
|
1292
|
+
# Parse params JSON string
|
|
1293
|
+
import json
|
|
1294
|
+
from json_repair import repair_json
|
|
1295
|
+
try:
|
|
1296
|
+
method_params = json.loads(params.params)
|
|
1297
|
+
except json.JSONDecodeError:
|
|
1298
|
+
method_params = json.loads(repair_json(params.params))
|
|
1299
|
+
|
|
1300
|
+
# Execute the requested method
|
|
1301
|
+
result = None
|
|
1302
|
+
if params.method == "search_videos":
|
|
1303
|
+
result = await yt_client.search_videos(**method_params)
|
|
1304
|
+
elif params.method == "get_video_details":
|
|
1305
|
+
result = await yt_client.get_video_details(**method_params)
|
|
1306
|
+
elif params.method == "get_video_comments":
|
|
1307
|
+
result = await yt_client.get_video_comments(**method_params)
|
|
1308
|
+
elif params.method == "get_channel_info":
|
|
1309
|
+
result = await yt_client.get_channel_info(**method_params)
|
|
1310
|
+
elif params.method == "get_channel_videos":
|
|
1311
|
+
result = await yt_client.get_channel_videos(**method_params)
|
|
1312
|
+
elif params.method == "get_trending_videos":
|
|
1313
|
+
result = await yt_client.get_trending_videos()
|
|
1314
|
+
else:
|
|
1315
|
+
return ActionResult(error=f"Unknown method: {params.method}")
|
|
1316
|
+
|
|
1317
|
+
# Save result to file
|
|
1318
|
+
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
|
1319
|
+
filename = f"youtube_{params.method}_{timestamp}.json"
|
|
1320
|
+
filepath = file_system.get_dir() / "data" / filename
|
|
1321
|
+
filepath.parent.mkdir(exist_ok=True)
|
|
1322
|
+
|
|
1323
|
+
with open(filepath, "w", encoding="utf-8") as f:
|
|
1324
|
+
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
1325
|
+
|
|
1326
|
+
# Format result as markdown
|
|
1327
|
+
if isinstance(result, list):
|
|
1328
|
+
display_count = min(5, len(result))
|
|
1329
|
+
md_content = f"## YouTube {params.method.replace('_', ' ').title()}\n\n"
|
|
1330
|
+
md_content += f"Showing {display_count} of {len(result)} results:\n\n"
|
|
1331
|
+
for i, item in enumerate(result[:display_count]):
|
|
1332
|
+
md_content += f"### Result {i+1}\n"
|
|
1333
|
+
for key, value in item.items():
|
|
1334
|
+
if not value:
|
|
1335
|
+
continue
|
|
1336
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1337
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1338
|
+
else:
|
|
1339
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1340
|
+
md_content += "\n"
|
|
1341
|
+
else:
|
|
1342
|
+
md_content = f"## YouTube {params.method.replace('_', ' ').title()}\n\n"
|
|
1343
|
+
for key, value in result.items():
|
|
1344
|
+
if isinstance(value, str) and len(value) > 200:
|
|
1345
|
+
md_content += f"- **{key}**: {value[:200]}...\n"
|
|
1346
|
+
else:
|
|
1347
|
+
md_content += f"- **{key}**: {value}\n"
|
|
1348
|
+
md_content += "\n"
|
|
1349
|
+
|
|
1350
|
+
# Add file path to markdown
|
|
1351
|
+
relative_path = str(filepath.relative_to(file_system.get_dir()))
|
|
1352
|
+
md_content += f"\n> 📁 Full data saved to: [{filename}]({relative_path})\n"
|
|
1353
|
+
md_content += f"> 💡 Click the link above to view all results.\n"
|
|
1354
|
+
|
|
1355
|
+
logger.info(f'🎬 YouTube data retrieved with method: {params.method}')
|
|
1356
|
+
|
|
1357
|
+
# Close client
|
|
1358
|
+
await yt_client.close()
|
|
1359
|
+
|
|
1360
|
+
return ActionResult(
|
|
1361
|
+
extracted_content=md_content
|
|
1362
|
+
)
|
|
1363
|
+
|
|
1364
|
+
except Exception as e:
|
|
1365
|
+
error_msg = f'❌ Failed to retrieve YouTube data: {str(e)}'
|
|
1366
|
+
logger.error(error_msg)
|
|
1367
|
+
return ActionResult(error=error_msg, extracted_content=error_msg)
|
|
926
1368
|
|
|
927
1369
|
|
|
928
1370
|
async def _extract_google_results_rule_based(self, browser_session):
|
|
@@ -1075,7 +1517,7 @@ Please generate alternative JavaScript code that avoids this system error:"""
|
|
|
1075
1517
|
results = await self._extract_google_results_rule_based(browser_session)
|
|
1076
1518
|
if results and len(results) > 0:
|
|
1077
1519
|
# Rule-based extraction succeeded
|
|
1078
|
-
logger.
|
|
1520
|
+
logger.debug(f"Rule-based extraction found {len(results)} results for query: {query}")
|
|
1079
1521
|
return results[:search_ret_len] # Return top 6 results
|
|
1080
1522
|
|
|
1081
1523
|
# Fallback to LLM extraction if rule-based fails
|
vibe_surf/tools/views.py
CHANGED
|
@@ -211,3 +211,78 @@ class SkillFinanceAction(BaseModel):
|
|
|
211
211
|
ge=1,
|
|
212
212
|
le=20,
|
|
213
213
|
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
class DownloadMediaAction(BaseModel):
|
|
217
|
+
"""Parameters for downloading media from URL"""
|
|
218
|
+
url: str = Field(
|
|
219
|
+
description='URL of the media to download',
|
|
220
|
+
)
|
|
221
|
+
filename: str | None = Field(
|
|
222
|
+
default=None,
|
|
223
|
+
description='Optional custom filename. If not provided, will auto-detect from URL or Content-Disposition header',
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class SkillXhsAction(BaseModel):
|
|
228
|
+
"""Parameters for skill_xhs action - Xiaohongshu API skill"""
|
|
229
|
+
method: str = Field(
|
|
230
|
+
description='''Xiaohongshu API method name. Available methods:
|
|
231
|
+
- search_content_by_keyword: Search content by keyword, params required: {"keyword": "search keyword", "page": 1, "page_size": 20}
|
|
232
|
+
- fetch_content_details: Get content details, params required: {"content_id": "content ID", "xsec_token": "security token"}
|
|
233
|
+
- fetch_all_content_comments: Get all comments for content, params required: {"content_id": "content ID", "xsec_token": "security token", "max_comments": 100}
|
|
234
|
+
- get_user_profile: Get user profile, params required: {"user_id": "user ID"}
|
|
235
|
+
- fetch_all_user_content: Get all content by user, params required: {"user_id": "user ID", "max_content": 100}
|
|
236
|
+
- get_home_recommendations: Get home page recommendations, params: {}'''
|
|
237
|
+
)
|
|
238
|
+
params: str = Field(
|
|
239
|
+
description='JSON string of method parameters, provide corresponding parameters according to the method parameter. Example: {"keyword": "food"}'
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class SkillWeiboAction(BaseModel):
|
|
244
|
+
"""Parameters for skill_weibo action - Weibo API skill"""
|
|
245
|
+
method: str = Field(
|
|
246
|
+
description='''Weibo API method name. Available methods:
|
|
247
|
+
- search_posts_by_keyword: Search posts by keyword, params required: {"keyword": "search keyword", "page": 1}
|
|
248
|
+
- get_post_detail: Get post details, params required: {"mid": "post ID"}
|
|
249
|
+
- get_all_post_comments: Get all comments for post, params required: {"mid": "post ID", "max_comments": 100}
|
|
250
|
+
- get_user_info: Get user information, params required: {"user_id": "user ID"}
|
|
251
|
+
- get_all_user_posts: Get all posts by user, params required: {"user_id": "user ID", "max_posts": 100}
|
|
252
|
+
- get_hot_posts: Get hot posts(推荐榜), params: {}
|
|
253
|
+
- get_trending_posts: Get trending posts(热搜榜), params: {}'''
|
|
254
|
+
)
|
|
255
|
+
params: str = Field(
|
|
256
|
+
description='JSON string of method parameters, provide corresponding parameters according to the method parameter. Example: {"keyword": "AI trending"}'
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
class SkillDouyinAction(BaseModel):
|
|
261
|
+
"""Parameters for skill_douyin action - Douyin API skill"""
|
|
262
|
+
method: str = Field(
|
|
263
|
+
description='''Douyin API method name. Available methods:
|
|
264
|
+
- search_content_by_keyword: Search videos by keyword, params required: {"keyword": "search keyword", "offset": 0}
|
|
265
|
+
- fetch_video_details: Get video details, params required: {"aweme_id": "video ID"}
|
|
266
|
+
- fetch_all_video_comments: Get all comments for video, params required: {"aweme_id": "video ID", "max_comments": 100}
|
|
267
|
+
- fetch_user_info: Get user information, params required: {"sec_user_id": "user security ID"}
|
|
268
|
+
- fetch_all_user_videos: Get all videos by user, params required: {"sec_user_id": "user security ID", "max_videos": 100}'''
|
|
269
|
+
)
|
|
270
|
+
params: str = Field(
|
|
271
|
+
description='JSON string of method parameters, provide corresponding parameters according to the method parameter. Example: {"keyword": "music"}'
|
|
272
|
+
)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
class SkillYoutubeAction(BaseModel):
|
|
276
|
+
"""Parameters for skill_youtube action - YouTube API skill"""
|
|
277
|
+
method: str = Field(
|
|
278
|
+
description='''YouTube API method name. Available methods:
|
|
279
|
+
- search_videos: Search videos, params required: {"query": "search keyword", "max_results": 20}
|
|
280
|
+
- get_video_details: Get video details, params required: {"video_id": "video ID"}
|
|
281
|
+
- get_video_comments: Get video comments, params required: {"video_id": "video ID", "max_comments": 200}
|
|
282
|
+
- get_channel_info: Get channel information, params required: {"channel_id": "channel ID"}
|
|
283
|
+
- get_channel_videos: Get channel videos, params required: {"channel_id": "channel ID", "max_videos": 20}
|
|
284
|
+
- get_trending_videos: Get trending videos, params: {}'''
|
|
285
|
+
)
|
|
286
|
+
params: str = Field(
|
|
287
|
+
description='JSON string of method parameters, provide corresponding parameters according to the method parameter. Example: {"query": "tech tutorial", "max_results": 30}'
|
|
288
|
+
)
|