botrun-flow-lang 5.12.263__py3-none-any.whl → 6.2.21__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- botrun_flow_lang/api/auth_api.py +39 -39
- botrun_flow_lang/api/auth_utils.py +183 -183
- botrun_flow_lang/api/botrun_back_api.py +65 -65
- botrun_flow_lang/api/flow_api.py +3 -3
- botrun_flow_lang/api/hatch_api.py +508 -508
- botrun_flow_lang/api/langgraph_api.py +816 -811
- botrun_flow_lang/api/langgraph_constants.py +11 -0
- botrun_flow_lang/api/line_bot_api.py +1484 -1484
- botrun_flow_lang/api/model_api.py +300 -300
- botrun_flow_lang/api/rate_limit_api.py +32 -32
- botrun_flow_lang/api/routes.py +79 -79
- botrun_flow_lang/api/search_api.py +53 -53
- botrun_flow_lang/api/storage_api.py +395 -395
- botrun_flow_lang/api/subsidy_api.py +290 -290
- botrun_flow_lang/api/subsidy_api_system_prompt.txt +109 -109
- botrun_flow_lang/api/user_setting_api.py +70 -70
- botrun_flow_lang/api/version_api.py +31 -31
- botrun_flow_lang/api/youtube_api.py +26 -26
- botrun_flow_lang/constants.py +13 -13
- botrun_flow_lang/langgraph_agents/agents/agent_runner.py +178 -178
- botrun_flow_lang/langgraph_agents/agents/agent_tools/step_planner.py +77 -77
- botrun_flow_lang/langgraph_agents/agents/checkpointer/firestore_checkpointer.py +666 -666
- botrun_flow_lang/langgraph_agents/agents/gov_researcher/GOV_RESEARCHER_PRD.md +192 -192
- botrun_flow_lang/langgraph_agents/agents/gov_researcher/gemini_subsidy_graph.py +460 -460
- botrun_flow_lang/langgraph_agents/agents/gov_researcher/gov_researcher_2_graph.py +1002 -1002
- botrun_flow_lang/langgraph_agents/agents/gov_researcher/gov_researcher_graph.py +822 -822
- botrun_flow_lang/langgraph_agents/agents/langgraph_react_agent.py +730 -723
- botrun_flow_lang/langgraph_agents/agents/search_agent_graph.py +864 -864
- botrun_flow_lang/langgraph_agents/agents/tools/__init__.py +4 -4
- botrun_flow_lang/langgraph_agents/agents/tools/gemini_code_execution.py +376 -376
- botrun_flow_lang/langgraph_agents/agents/util/gemini_grounding.py +66 -66
- botrun_flow_lang/langgraph_agents/agents/util/html_util.py +316 -316
- botrun_flow_lang/langgraph_agents/agents/util/img_util.py +336 -294
- botrun_flow_lang/langgraph_agents/agents/util/local_files.py +419 -419
- botrun_flow_lang/langgraph_agents/agents/util/mermaid_util.py +86 -86
- botrun_flow_lang/langgraph_agents/agents/util/model_utils.py +143 -143
- botrun_flow_lang/langgraph_agents/agents/util/pdf_analyzer.py +562 -486
- botrun_flow_lang/langgraph_agents/agents/util/pdf_cache.py +250 -250
- botrun_flow_lang/langgraph_agents/agents/util/pdf_processor.py +204 -204
- botrun_flow_lang/langgraph_agents/agents/util/perplexity_search.py +464 -464
- botrun_flow_lang/langgraph_agents/agents/util/plotly_util.py +59 -59
- botrun_flow_lang/langgraph_agents/agents/util/tavily_search.py +199 -199
- botrun_flow_lang/langgraph_agents/agents/util/usage_metadata.py +34 -0
- botrun_flow_lang/langgraph_agents/agents/util/youtube_util.py +90 -90
- botrun_flow_lang/langgraph_agents/cache/langgraph_botrun_cache.py +197 -197
- botrun_flow_lang/llm_agent/llm_agent.py +19 -19
- botrun_flow_lang/llm_agent/llm_agent_util.py +83 -83
- botrun_flow_lang/log/.gitignore +2 -2
- botrun_flow_lang/main.py +61 -61
- botrun_flow_lang/main_fast.py +51 -51
- botrun_flow_lang/mcp_server/__init__.py +10 -10
- botrun_flow_lang/mcp_server/default_mcp.py +854 -744
- botrun_flow_lang/models/nodes/utils.py +205 -205
- botrun_flow_lang/models/token_usage.py +34 -34
- botrun_flow_lang/requirements.txt +21 -21
- botrun_flow_lang/services/base/firestore_base.py +30 -30
- botrun_flow_lang/services/hatch/hatch_factory.py +11 -11
- botrun_flow_lang/services/hatch/hatch_fs_store.py +419 -419
- botrun_flow_lang/services/storage/storage_cs_store.py +206 -206
- botrun_flow_lang/services/storage/storage_factory.py +12 -12
- botrun_flow_lang/services/storage/storage_store.py +65 -65
- botrun_flow_lang/services/user_setting/user_setting_factory.py +9 -9
- botrun_flow_lang/services/user_setting/user_setting_fs_store.py +66 -66
- botrun_flow_lang/static/docs/tools/index.html +926 -926
- botrun_flow_lang/tests/api_functional_tests.py +1525 -1525
- botrun_flow_lang/tests/api_stress_test.py +357 -357
- botrun_flow_lang/tests/shared_hatch_tests.py +333 -333
- botrun_flow_lang/tests/test_botrun_app.py +46 -46
- botrun_flow_lang/tests/test_html_util.py +31 -31
- botrun_flow_lang/tests/test_img_analyzer.py +190 -190
- botrun_flow_lang/tests/test_img_util.py +39 -39
- botrun_flow_lang/tests/test_local_files.py +114 -114
- botrun_flow_lang/tests/test_mermaid_util.py +103 -103
- botrun_flow_lang/tests/test_pdf_analyzer.py +104 -104
- botrun_flow_lang/tests/test_plotly_util.py +151 -151
- botrun_flow_lang/tests/test_run_workflow_engine.py +65 -65
- botrun_flow_lang/tools/generate_docs.py +133 -133
- botrun_flow_lang/tools/templates/tools.html +153 -153
- botrun_flow_lang/utils/__init__.py +7 -7
- botrun_flow_lang/utils/botrun_logger.py +344 -344
- botrun_flow_lang/utils/clients/rate_limit_client.py +209 -209
- botrun_flow_lang/utils/clients/token_verify_client.py +153 -153
- botrun_flow_lang/utils/google_drive_utils.py +654 -654
- botrun_flow_lang/utils/langchain_utils.py +324 -324
- botrun_flow_lang/utils/yaml_utils.py +9 -9
- {botrun_flow_lang-5.12.263.dist-info → botrun_flow_lang-6.2.21.dist-info}/METADATA +6 -6
- botrun_flow_lang-6.2.21.dist-info/RECORD +104 -0
- botrun_flow_lang-5.12.263.dist-info/RECORD +0 -102
- {botrun_flow_lang-5.12.263.dist-info → botrun_flow_lang-6.2.21.dist-info}/WHEEL +0 -0
|
@@ -1,1525 +1,1525 @@
|
|
|
1
|
-
from datetime import datetime
|
|
2
|
-
import unittest
|
|
3
|
-
from langchain_anthropic import ChatAnthropic
|
|
4
|
-
import pytz
|
|
5
|
-
import requests
|
|
6
|
-
from typing import Dict, Any, List
|
|
7
|
-
from pydantic import BaseModel, Field
|
|
8
|
-
from langchain_openai import ChatOpenAI
|
|
9
|
-
from trustcall import create_extractor
|
|
10
|
-
from botrun_flow_lang.langgraph_agents.agents.util.local_files import (
|
|
11
|
-
upload_and_get_tmp_public_url,
|
|
12
|
-
)
|
|
13
|
-
from pathlib import Path
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
class ValidationResult(BaseModel):
|
|
17
|
-
"""Pydantic model for the validation result"""
|
|
18
|
-
|
|
19
|
-
pass_: bool = Field(
|
|
20
|
-
description="Whether the validation passes (true) or fails (false), If all conditions are met, return true, otherwise return false"
|
|
21
|
-
)
|
|
22
|
-
reason: str = Field(
|
|
23
|
-
description="Detailed explanation of why validation passed or failed"
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
class TestAPIFunctionality(unittest.TestCase):
|
|
28
|
-
"""Test class for REST API functionality tests"""
|
|
29
|
-
|
|
30
|
-
def setUp(self):
|
|
31
|
-
"""Setup method that runs before each test"""
|
|
32
|
-
# Default base URL, can be overridden by setting the class attribute
|
|
33
|
-
if not hasattr(self, "base_url"):
|
|
34
|
-
self.base_url = "http://localhost:8080"
|
|
35
|
-
# self.base_url = (
|
|
36
|
-
# "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app"
|
|
37
|
-
# )
|
|
38
|
-
|
|
39
|
-
# Common headers
|
|
40
|
-
self.headers = {"Content-Type": "application/json"}
|
|
41
|
-
|
|
42
|
-
# Initialize LLM and extractor
|
|
43
|
-
# self.llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
|
|
44
|
-
self.llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0)
|
|
45
|
-
self.validator = create_extractor(
|
|
46
|
-
self.llm, tools=[ValidationResult], tool_choice="ValidationResult"
|
|
47
|
-
)
|
|
48
|
-
local_tz = pytz.timezone("Asia/Taipei")
|
|
49
|
-
self.local_time = datetime.now(local_tz)
|
|
50
|
-
|
|
51
|
-
def api_post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
52
|
-
"""Helper method to make POST requests to the API
|
|
53
|
-
|
|
54
|
-
Args:
|
|
55
|
-
endpoint: The API endpoint path (without base URL)
|
|
56
|
-
data: The request payload as a dictionary
|
|
57
|
-
|
|
58
|
-
Returns:
|
|
59
|
-
The JSON response as a dictionary
|
|
60
|
-
"""
|
|
61
|
-
url = f"{self.base_url}{endpoint}"
|
|
62
|
-
response = requests.post(url, headers=self.headers, json=data)
|
|
63
|
-
|
|
64
|
-
# Raise an exception if the response was unsuccessful
|
|
65
|
-
response.raise_for_status()
|
|
66
|
-
|
|
67
|
-
return response.json()
|
|
68
|
-
|
|
69
|
-
def validate_with_llm(
|
|
70
|
-
self, response_content: str, validation_criteria: str
|
|
71
|
-
) -> Dict[str, Any]:
|
|
72
|
-
"""Use trustcall with GPT-4o-mini to validate the response content
|
|
73
|
-
|
|
74
|
-
Args:
|
|
75
|
-
response_content: The content to validate
|
|
76
|
-
validation_criteria: Validation criteria description
|
|
77
|
-
|
|
78
|
-
Returns:
|
|
79
|
-
Dictionary with 'pass' (boolean) and 'reason' (string)
|
|
80
|
-
"""
|
|
81
|
-
prompt = f"""
|
|
82
|
-
你是一個專業的API回應驗證員。請評估以下API回應是否符合所有指定條件。
|
|
83
|
-
|
|
84
|
-
=== 驗證條件 ===
|
|
85
|
-
{validation_criteria}
|
|
86
|
-
|
|
87
|
-
=== API回應內容 ===
|
|
88
|
-
{response_content}
|
|
89
|
-
|
|
90
|
-
請評估API回應是否符合所有驗證條件。詳細說明評估原因,若不符合條件,請明確指出哪些條件未達成。
|
|
91
|
-
"""
|
|
92
|
-
|
|
93
|
-
try:
|
|
94
|
-
# Use trustcall extractor to validate
|
|
95
|
-
result = self.validator.invoke(
|
|
96
|
-
{"messages": [{"role": "user", "content": prompt}]}
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
# Extract the validated response
|
|
100
|
-
validation_result = result["responses"][0]
|
|
101
|
-
|
|
102
|
-
# Convert to the expected format
|
|
103
|
-
return {"pass": validation_result.pass_, "reason": validation_result.reason}
|
|
104
|
-
|
|
105
|
-
except Exception as e:
|
|
106
|
-
return {"pass": False, "reason": f"Error during validation: {str(e)}"}
|
|
107
|
-
|
|
108
|
-
def test_langgraph_news_joke_emoji(self):
|
|
109
|
-
"""測試是否會抓到今天的新聞,檢查重點:
|
|
110
|
-
1. 是否會抓到今天的新聞
|
|
111
|
-
2. 是否會列出來源網址
|
|
112
|
-
3. 是否會講個笑話,並加上 emoji
|
|
113
|
-
"""
|
|
114
|
-
# Test payload
|
|
115
|
-
payload = {
|
|
116
|
-
"graph_name": "langgraph_react_agent",
|
|
117
|
-
"messages": [
|
|
118
|
-
{
|
|
119
|
-
"role": "user",
|
|
120
|
-
"content": "幫我搜尋今天的新聞是什麼?一一列出來,並給我參考來源網址。",
|
|
121
|
-
}
|
|
122
|
-
],
|
|
123
|
-
"config": {
|
|
124
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
125
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
126
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
127
|
-
},
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
# Make the request
|
|
131
|
-
endpoint = "/api/langgraph/invoke"
|
|
132
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
133
|
-
print("-" * 50)
|
|
134
|
-
|
|
135
|
-
try:
|
|
136
|
-
response = self.api_post(endpoint, payload)
|
|
137
|
-
|
|
138
|
-
# Basic assertions to verify the response
|
|
139
|
-
self.assertIsNotNone(response)
|
|
140
|
-
|
|
141
|
-
# Extract the content field from the response
|
|
142
|
-
if "content" in response:
|
|
143
|
-
response_content = response["content"]
|
|
144
|
-
else:
|
|
145
|
-
self.fail("Response does not contain 'content' field")
|
|
146
|
-
|
|
147
|
-
validation_criteria = f"""
|
|
148
|
-
1. 是否包含今天 {self.local_time.strftime("%Y-%m-%d")} 日期的新聞資訊,沒有列出日期測試算失敗
|
|
149
|
-
2. 是否列出每則新聞的來源網址
|
|
150
|
-
3. 是否在回答結尾包含一個笑話
|
|
151
|
-
"""
|
|
152
|
-
|
|
153
|
-
# Validate with LLM
|
|
154
|
-
validation_result = self.validate_with_llm(
|
|
155
|
-
response_content, validation_criteria
|
|
156
|
-
)
|
|
157
|
-
|
|
158
|
-
# Assert that the validation passed
|
|
159
|
-
self.assertTrue(
|
|
160
|
-
validation_result["pass"],
|
|
161
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
162
|
-
)
|
|
163
|
-
|
|
164
|
-
except Exception as e:
|
|
165
|
-
print(f"test_langgraph_news_joke_emoji: Test failed with error: {str(e)}")
|
|
166
|
-
raise
|
|
167
|
-
|
|
168
|
-
def test_langgraph_multinode_news_dall_e(self):
|
|
169
|
-
"""測試多節點處理流程,檢查重點:
|
|
170
|
-
1. 是否會抓到今天的新聞
|
|
171
|
-
2. 是否有評分新聞 (1-10分)
|
|
172
|
-
3. 是否有產出一張圖片,並帶有 URL
|
|
173
|
-
"""
|
|
174
|
-
# Test payload
|
|
175
|
-
payload = {
|
|
176
|
-
"graph_name": "langgraph_react_agent",
|
|
177
|
-
"messages": [
|
|
178
|
-
{
|
|
179
|
-
"role": "user",
|
|
180
|
-
"content": "好。 我們現在就是跟那個Bert人講多個節點,那它裡面它就會用多個節點的方式直接去工作,比如說第一個節點就是請Bert人上網去搜尋今天的新聞。 然後第二個節點呢,請你把這個新聞,打分,一分到十分,哪個新聞最可愛。 然後第三個節點呢,請你根據分數最高的那一個新聞,你幫我呼叫達利畫一張跟那個新聞相關的圖片,那我們就用這個來示範一下,來。",
|
|
181
|
-
}
|
|
182
|
-
],
|
|
183
|
-
"config": {
|
|
184
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
185
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
186
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
187
|
-
},
|
|
188
|
-
}
|
|
189
|
-
|
|
190
|
-
# Make the request
|
|
191
|
-
endpoint = "/api/langgraph/invoke"
|
|
192
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
193
|
-
print("-" * 50)
|
|
194
|
-
|
|
195
|
-
try:
|
|
196
|
-
response = self.api_post(endpoint, payload)
|
|
197
|
-
|
|
198
|
-
# Basic assertions to verify the response
|
|
199
|
-
self.assertIsNotNone(response)
|
|
200
|
-
|
|
201
|
-
# Extract the content field from the response
|
|
202
|
-
if "content" in response:
|
|
203
|
-
response_content = response["content"]
|
|
204
|
-
else:
|
|
205
|
-
self.fail("Response does not contain 'content' field")
|
|
206
|
-
|
|
207
|
-
# Define validation criteria based on the test requirements
|
|
208
|
-
validation_criteria = f"""
|
|
209
|
-
1. 是否會抓到今天 {self.local_time.strftime("%Y-%m-%d")} 日期的新聞
|
|
210
|
-
2. 是否有評分新聞 (1-10分)
|
|
211
|
-
3. 是否有產出一張圖片,並帶有 URL
|
|
212
|
-
4. 是否在回答結尾包含一個笑話
|
|
213
|
-
"""
|
|
214
|
-
|
|
215
|
-
# Validate with LLM
|
|
216
|
-
validation_result = self.validate_with_llm(
|
|
217
|
-
response_content, validation_criteria
|
|
218
|
-
)
|
|
219
|
-
|
|
220
|
-
# Assert that the validation passed
|
|
221
|
-
self.assertTrue(
|
|
222
|
-
validation_result["pass"],
|
|
223
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
224
|
-
)
|
|
225
|
-
|
|
226
|
-
except Exception as e:
|
|
227
|
-
print(
|
|
228
|
-
f"test_langgraph_multinode_news_dall_e: Test failed with error: {str(e)}"
|
|
229
|
-
)
|
|
230
|
-
raise
|
|
231
|
-
|
|
232
|
-
def test_langgraph_future_date_news(self):
|
|
233
|
-
"""測試未來日期的新聞搜尋,檢查重點:
|
|
234
|
-
1. 是否是指定時間的 2025/2/10 的新聞,回覆內容要有這個時間
|
|
235
|
-
2. 不能回應說這個時間在未來,所以無法回答,可以說 "截至2025年2月10日,相關的新聞如下:"。
|
|
236
|
-
"""
|
|
237
|
-
# Test payload
|
|
238
|
-
payload = {
|
|
239
|
-
"graph_name": "langgraph_react_agent",
|
|
240
|
-
"messages": [
|
|
241
|
-
{
|
|
242
|
-
"role": "user",
|
|
243
|
-
"content": "請你幫我找2025/2/10全球災難新聞",
|
|
244
|
-
}
|
|
245
|
-
],
|
|
246
|
-
"config": {
|
|
247
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
248
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
249
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
250
|
-
},
|
|
251
|
-
}
|
|
252
|
-
|
|
253
|
-
# Make the request
|
|
254
|
-
endpoint = "/api/langgraph/invoke"
|
|
255
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
256
|
-
print("-" * 50)
|
|
257
|
-
|
|
258
|
-
try:
|
|
259
|
-
response = self.api_post(endpoint, payload)
|
|
260
|
-
|
|
261
|
-
# Basic assertions to verify the response
|
|
262
|
-
self.assertIsNotNone(response)
|
|
263
|
-
|
|
264
|
-
# Extract the content field from the response
|
|
265
|
-
if "content" in response:
|
|
266
|
-
response_content = response["content"]
|
|
267
|
-
else:
|
|
268
|
-
self.fail("Response does not contain 'content' field")
|
|
269
|
-
|
|
270
|
-
# Define validation criteria based on the test requirements
|
|
271
|
-
validation_criteria = """
|
|
272
|
-
1. 是否包含指定時間 2025/2/10 的新聞資訊,回覆內容中必須有出現「2025/2/10」或類似的日期格式
|
|
273
|
-
2. 不能包含任何提到該日期在未來、無法預測未來、尚未發生等類似的說明
|
|
274
|
-
3. 是否在回答結尾包含一個笑話
|
|
275
|
-
"""
|
|
276
|
-
|
|
277
|
-
# Validate with LLM
|
|
278
|
-
validation_result = self.validate_with_llm(
|
|
279
|
-
response_content, validation_criteria
|
|
280
|
-
)
|
|
281
|
-
|
|
282
|
-
# Assert that the validation passed
|
|
283
|
-
self.assertTrue(
|
|
284
|
-
validation_result["pass"],
|
|
285
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
286
|
-
)
|
|
287
|
-
|
|
288
|
-
except Exception as e:
|
|
289
|
-
print(f"test_langgraph_future_date_news: Test failed with error: {str(e)}")
|
|
290
|
-
raise
|
|
291
|
-
|
|
292
|
-
def test_langgraph_pdf_analysis(self):
|
|
293
|
-
"""測試PDF分析功能,檢查重點:
|
|
294
|
-
1. 是否能正確解析PDF檔案中的「表 4.3-1 環境敏感地區調查表-第一級環境敏感地區」
|
|
295
|
-
2. 是否能列出所有項目的「查詢結果及限制內容」(是或否)
|
|
296
|
-
3. 回傳結果是否符合預期的敏感區域結果
|
|
297
|
-
"""
|
|
298
|
-
# 使用pathlib構建正確的檔案路徑
|
|
299
|
-
|
|
300
|
-
current_dir = Path(__file__).parent
|
|
301
|
-
pdf_path = (
|
|
302
|
-
current_dir
|
|
303
|
-
/ "test_files"
|
|
304
|
-
/ "1120701A海廣離岸風力發電計畫環境影響說明書-C04.PDF"
|
|
305
|
-
)
|
|
306
|
-
|
|
307
|
-
# 確保檔案存在
|
|
308
|
-
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
309
|
-
# 將絕對路徑轉為字串
|
|
310
|
-
pdf_path_str = str(pdf_path)
|
|
311
|
-
# 上傳檔案到 tmp_public_url
|
|
312
|
-
tmp_public_url = upload_and_get_tmp_public_url(
|
|
313
|
-
pdf_path_str,
|
|
314
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
315
|
-
"sebastian.hsu@gmail.com",
|
|
316
|
-
)
|
|
317
|
-
|
|
318
|
-
# Test payload
|
|
319
|
-
payload = {
|
|
320
|
-
"graph_name": "langgraph_react_agent",
|
|
321
|
-
"messages": [
|
|
322
|
-
{
|
|
323
|
-
"role": "user",
|
|
324
|
-
"content": f"幫我分析 {tmp_public_url} 這個檔案,請你幫我找出在報告書中的「表 4.3-1 環境敏感地區調查表-第一級環境敏感地區」表格中的所有項目的「查詢結果及限制內容」幫我列出是或否?請全部列出來,不要遺漏",
|
|
325
|
-
}
|
|
326
|
-
],
|
|
327
|
-
"config": {
|
|
328
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
329
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
330
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
331
|
-
},
|
|
332
|
-
}
|
|
333
|
-
|
|
334
|
-
# Make the request
|
|
335
|
-
endpoint = "/api/langgraph/invoke"
|
|
336
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
337
|
-
print("-" * 50)
|
|
338
|
-
|
|
339
|
-
try:
|
|
340
|
-
response = self.api_post(endpoint, payload)
|
|
341
|
-
|
|
342
|
-
# Basic assertions to verify the response
|
|
343
|
-
self.assertIsNotNone(response)
|
|
344
|
-
|
|
345
|
-
# Extract the content field from the response
|
|
346
|
-
if "content" in response:
|
|
347
|
-
response_content = response["content"]
|
|
348
|
-
else:
|
|
349
|
-
self.fail("Response does not contain 'content' field")
|
|
350
|
-
|
|
351
|
-
# Define validation criteria based on existing test_pdf_analyzer.py
|
|
352
|
-
validation_criteria = """
|
|
353
|
-
請確認回應是否包含以下項目的查詢結果(是或否),所有項目都必須存在:
|
|
354
|
-
1. 活動斷層兩側一定範圍: 否
|
|
355
|
-
2. 特定水土保持區: 否
|
|
356
|
-
3. 河川區域: 否
|
|
357
|
-
4. 洪氾區一級管制區及洪水平原一級管制區: 否
|
|
358
|
-
5. 區域排水設施範圍: 是
|
|
359
|
-
6. 國家公園區內之特別景觀區、生態保護區: 否
|
|
360
|
-
7. 自然保留區: 否
|
|
361
|
-
8. 野生動物保護區: 否
|
|
362
|
-
9. 野生動物重要棲息環境: 是
|
|
363
|
-
10. 自然保護區: 否
|
|
364
|
-
11. 一級海岸保護區: 是
|
|
365
|
-
12. 國際級重要濕地、國家級重要濕地之核心保育區及生態復育區: 否
|
|
366
|
-
13. 古蹟保存區: 否
|
|
367
|
-
14. 考古遺址: 否
|
|
368
|
-
15. 重要聚落建築群: 否
|
|
369
|
-
|
|
370
|
-
所有項目都必須正確列出,且其中:
|
|
371
|
-
- 區域排水設施範圍應為「是」
|
|
372
|
-
- 野生動物重要棲息環境應為「是」
|
|
373
|
-
- 一級海岸保護區應為「是」
|
|
374
|
-
|
|
375
|
-
如果有遺漏任何一項或者結果不符合預期,則視為測試失敗。
|
|
376
|
-
如果結果有超過,沒有關係。
|
|
377
|
-
"""
|
|
378
|
-
|
|
379
|
-
# Validate with LLM
|
|
380
|
-
validation_result = self.validate_with_llm(
|
|
381
|
-
response_content, validation_criteria
|
|
382
|
-
)
|
|
383
|
-
|
|
384
|
-
# Assert that the validation passed
|
|
385
|
-
self.assertTrue(
|
|
386
|
-
validation_result["pass"],
|
|
387
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
388
|
-
)
|
|
389
|
-
|
|
390
|
-
except Exception as e:
|
|
391
|
-
print(f"test_langgraph_pdf_analysis: Test failed with error: {str(e)}")
|
|
392
|
-
raise
|
|
393
|
-
|
|
394
|
-
def test_langgraph_pdf_attendance_analysis(self):
|
|
395
|
-
"""測試PDF分析功能,檢查重點:
|
|
396
|
-
1. 是否能正確解析PDF檔案中的「目錄4」的出席名單
|
|
397
|
-
2. 回答中是否有包含「德懷師父」、「德宸師父」、「德倫師父」
|
|
398
|
-
"""
|
|
399
|
-
|
|
400
|
-
current_dir = Path(__file__).parent
|
|
401
|
-
pdf_path = (
|
|
402
|
-
current_dir
|
|
403
|
-
/ "test_files"
|
|
404
|
-
/ "(溫馨成果 行政請示匯總)20250210向 上人報告簡報 (1).pdf"
|
|
405
|
-
)
|
|
406
|
-
|
|
407
|
-
# 確保檔案存在
|
|
408
|
-
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
409
|
-
|
|
410
|
-
# 將絕對路徑轉為字串
|
|
411
|
-
pdf_path_str = str(pdf_path)
|
|
412
|
-
|
|
413
|
-
# 上傳檔案到 tmp_public_url
|
|
414
|
-
tmp_public_url = upload_and_get_tmp_public_url(
|
|
415
|
-
pdf_path_str,
|
|
416
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
417
|
-
"sebastian.hsu@gmail.com",
|
|
418
|
-
)
|
|
419
|
-
|
|
420
|
-
# Test payload
|
|
421
|
-
payload = {
|
|
422
|
-
"graph_name": "langgraph_react_agent",
|
|
423
|
-
"messages": [
|
|
424
|
-
{
|
|
425
|
-
"role": "user",
|
|
426
|
-
"content": f"幫我分析 {tmp_public_url} 這個檔案,你幫我看「目錄4」,告訴我有哪些師父和講者、執辦、主管有出席",
|
|
427
|
-
}
|
|
428
|
-
],
|
|
429
|
-
"config": {
|
|
430
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
431
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
432
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
433
|
-
},
|
|
434
|
-
}
|
|
435
|
-
|
|
436
|
-
# Make the request
|
|
437
|
-
endpoint = "/api/langgraph/invoke"
|
|
438
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
439
|
-
print("-" * 50)
|
|
440
|
-
|
|
441
|
-
try:
|
|
442
|
-
response = self.api_post(endpoint, payload)
|
|
443
|
-
|
|
444
|
-
# Basic assertions to verify the response
|
|
445
|
-
self.assertIsNotNone(response)
|
|
446
|
-
|
|
447
|
-
# Extract the content field from the response
|
|
448
|
-
if "content" in response:
|
|
449
|
-
response_content = response["content"]
|
|
450
|
-
else:
|
|
451
|
-
self.fail("Response does not contain 'content' field")
|
|
452
|
-
|
|
453
|
-
# Define validation criteria
|
|
454
|
-
validation_criteria = """
|
|
455
|
-
請確認回應是否包含以下師父的名字,所有名字都必須存在:
|
|
456
|
-
1. 德懷師父
|
|
457
|
-
2. 德宸師父
|
|
458
|
-
3. 德倫師父
|
|
459
|
-
|
|
460
|
-
此外,回應應該提供「目錄4」中出席的師父、講者、執辦和主管的完整列表。
|
|
461
|
-
如果缺少上述任一師父的名字,則視為測試失敗。
|
|
462
|
-
"""
|
|
463
|
-
|
|
464
|
-
# Validate with LLM
|
|
465
|
-
validation_result = self.validate_with_llm(
|
|
466
|
-
response_content, validation_criteria
|
|
467
|
-
)
|
|
468
|
-
|
|
469
|
-
# Assert that the validation passed
|
|
470
|
-
self.assertTrue(
|
|
471
|
-
validation_result["pass"],
|
|
472
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
473
|
-
)
|
|
474
|
-
|
|
475
|
-
except Exception as e:
|
|
476
|
-
print(
|
|
477
|
-
f"test_langgraph_pdf_attendance_analysis: Test failed with error: {str(e)}"
|
|
478
|
-
)
|
|
479
|
-
raise
|
|
480
|
-
|
|
481
|
-
def test_langgraph_image_analysis_generation(self):
|
|
482
|
-
"""測試圖片分析與生成功能,檢查重點:
|
|
483
|
-
1. 是否能正確分析圖片並識別出「佛教」相關元素
|
|
484
|
-
2. 是否產生一張相同意境的圖片並提供URL
|
|
485
|
-
"""
|
|
486
|
-
|
|
487
|
-
current_dir = Path(__file__).parent
|
|
488
|
-
image_path = current_dir / "test_files" / "d5712343.jpg"
|
|
489
|
-
|
|
490
|
-
# 確保檔案存在
|
|
491
|
-
self.assertTrue(image_path.exists(), f"Test file not found at {image_path}")
|
|
492
|
-
|
|
493
|
-
# 將絕對路徑轉為字串
|
|
494
|
-
image_path_str = str(image_path)
|
|
495
|
-
|
|
496
|
-
# 上傳檔案到 tmp_public_url
|
|
497
|
-
tmp_public_url = upload_and_get_tmp_public_url(
|
|
498
|
-
image_path_str,
|
|
499
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
500
|
-
"sebastian.hsu@gmail.com",
|
|
501
|
-
)
|
|
502
|
-
|
|
503
|
-
# Test payload
|
|
504
|
-
payload = {
|
|
505
|
-
"graph_name": "langgraph_react_agent",
|
|
506
|
-
"messages": [
|
|
507
|
-
{
|
|
508
|
-
"role": "user",
|
|
509
|
-
"content": f"{tmp_public_url} 幫我分析這張圖裡的元素,然後幫我創作一張相同意境的圖片",
|
|
510
|
-
}
|
|
511
|
-
],
|
|
512
|
-
"config": {
|
|
513
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
514
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
515
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
516
|
-
},
|
|
517
|
-
}
|
|
518
|
-
|
|
519
|
-
# Make the request
|
|
520
|
-
endpoint = "/api/langgraph/invoke"
|
|
521
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
522
|
-
print("-" * 50)
|
|
523
|
-
|
|
524
|
-
try:
|
|
525
|
-
response = self.api_post(endpoint, payload)
|
|
526
|
-
|
|
527
|
-
# Basic assertions to verify the response
|
|
528
|
-
self.assertIsNotNone(response)
|
|
529
|
-
|
|
530
|
-
# Extract the content field from the response
|
|
531
|
-
if "content" in response:
|
|
532
|
-
response_content = response["content"]
|
|
533
|
-
else:
|
|
534
|
-
self.fail("Response does not contain 'content' field")
|
|
535
|
-
|
|
536
|
-
# Define validation criteria
|
|
537
|
-
validation_criteria = """
|
|
538
|
-
請確認回應是否符合以下條件:
|
|
539
|
-
1. 分析結果中有提到「佛教」相關的元素(如佛像、和尚、寺廟、佛教符號等)
|
|
540
|
-
2. 回應中包含一個圖片的URL(通常是以http或https開頭的網址,並包含在圖片的描述旁)
|
|
541
|
-
3. 是否在回答結尾包含一個笑話
|
|
542
|
-
|
|
543
|
-
所有條件都必須滿足,尤其是必須確認分析中有提到佛教元素,並且有生成一張新的圖片和提供其URL。
|
|
544
|
-
"""
|
|
545
|
-
|
|
546
|
-
# Validate with LLM
|
|
547
|
-
validation_result = self.validate_with_llm(
|
|
548
|
-
response_content, validation_criteria
|
|
549
|
-
)
|
|
550
|
-
|
|
551
|
-
# Assert that the validation passed
|
|
552
|
-
self.assertTrue(
|
|
553
|
-
validation_result["pass"],
|
|
554
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
555
|
-
)
|
|
556
|
-
|
|
557
|
-
except Exception as e:
|
|
558
|
-
print(
|
|
559
|
-
f"test_langgraph_image_analysis_generation: Test failed with error: {str(e)}"
|
|
560
|
-
)
|
|
561
|
-
raise
|
|
562
|
-
|
|
563
|
-
def test_langgraph_spot_difference(self):
|
|
564
|
-
"""測試圖片比對功能,檢查重點:
|
|
565
|
-
1. 是否能正確分析兩張找不同遊戲的圖片
|
|
566
|
-
2. 是否能找出並明確描述出兩張圖片的不同之處
|
|
567
|
-
"""
|
|
568
|
-
|
|
569
|
-
current_dir = Path(__file__).parent
|
|
570
|
-
image1_path = current_dir / "test_files" / "spot_difference_1.png"
|
|
571
|
-
image2_path = current_dir / "test_files" / "spot_difference_2.png"
|
|
572
|
-
|
|
573
|
-
# 確保檔案存在
|
|
574
|
-
self.assertTrue(image1_path.exists(), f"Test file not found at {image1_path}")
|
|
575
|
-
self.assertTrue(image2_path.exists(), f"Test file not found at {image2_path}")
|
|
576
|
-
|
|
577
|
-
# 將絕對路徑轉為字串
|
|
578
|
-
image1_path_str = str(image1_path)
|
|
579
|
-
image2_path_str = str(image2_path)
|
|
580
|
-
|
|
581
|
-
# 上傳檔案到 tmp_public_url
|
|
582
|
-
tmp_public_url_1 = upload_and_get_tmp_public_url(
|
|
583
|
-
image1_path_str,
|
|
584
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
585
|
-
"sebastian.hsu@gmail.com",
|
|
586
|
-
)
|
|
587
|
-
tmp_public_url_2 = upload_and_get_tmp_public_url(
|
|
588
|
-
image2_path_str,
|
|
589
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
590
|
-
"sebastian.hsu@gmail.com",
|
|
591
|
-
)
|
|
592
|
-
|
|
593
|
-
# Test payload
|
|
594
|
-
payload = {
|
|
595
|
-
"graph_name": "langgraph_react_agent",
|
|
596
|
-
"messages": [
|
|
597
|
-
{
|
|
598
|
-
"role": "user",
|
|
599
|
-
"content": f"這是一個找不同的遊戲,幫我分析兩張圖有幾處不同? {tmp_public_url_1},{tmp_public_url_2}",
|
|
600
|
-
}
|
|
601
|
-
],
|
|
602
|
-
"config": {
|
|
603
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
604
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
605
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
606
|
-
},
|
|
607
|
-
}
|
|
608
|
-
|
|
609
|
-
# Make the request
|
|
610
|
-
endpoint = "/api/langgraph/invoke"
|
|
611
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
612
|
-
print("-" * 50)
|
|
613
|
-
|
|
614
|
-
try:
|
|
615
|
-
response = self.api_post(endpoint, payload)
|
|
616
|
-
|
|
617
|
-
# Basic assertions to verify the response
|
|
618
|
-
self.assertIsNotNone(response)
|
|
619
|
-
|
|
620
|
-
# Extract the content field from the response
|
|
621
|
-
if "content" in response:
|
|
622
|
-
response_content = response["content"]
|
|
623
|
-
else:
|
|
624
|
-
self.fail("Response does not contain 'content' field")
|
|
625
|
-
|
|
626
|
-
# Define validation criteria
|
|
627
|
-
validation_criteria = """
|
|
628
|
-
請確認回應是否符合以下條件:
|
|
629
|
-
1. 回應中有具體指出並描述兩張圖片之間的不同之處
|
|
630
|
-
2. 必須明確描述出不同的位置、形狀、顏色或其他特徵差異
|
|
631
|
-
3. 不能只回應「無法處理」、「無法比較」或類似的無能力陳述
|
|
632
|
-
4. 是否在回答結尾包含一個笑話
|
|
633
|
-
|
|
634
|
-
關鍵是要確保系統能夠實際找出差異並清楚描述,而不是迴避任務或宣稱無法完成。
|
|
635
|
-
如果回應只是說明系統不支援圖片比較功能,則測試視為失敗。
|
|
636
|
-
"""
|
|
637
|
-
|
|
638
|
-
# Validate with LLM
|
|
639
|
-
validation_result = self.validate_with_llm(
|
|
640
|
-
response_content, validation_criteria
|
|
641
|
-
)
|
|
642
|
-
|
|
643
|
-
# Assert that the validation passed
|
|
644
|
-
self.assertTrue(
|
|
645
|
-
validation_result["pass"],
|
|
646
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
647
|
-
)
|
|
648
|
-
|
|
649
|
-
except Exception as e:
|
|
650
|
-
print(f"test_langgraph_spot_difference: Test failed with error: {str(e)}")
|
|
651
|
-
raise
|
|
652
|
-
|
|
653
|
-
def test_langgraph_platform_images_analysis(self):
|
|
654
|
-
"""測試分析多張車站月台圖片功能,檢查重點:
|
|
655
|
-
1. 是否能正確分析多張車站月台圖片
|
|
656
|
-
2. 是否能根據圖片提供清晰的月台指引和理由
|
|
657
|
-
3. 回覆中是否包含「月台」相關的特定字眼
|
|
658
|
-
"""
|
|
659
|
-
|
|
660
|
-
current_dir = Path(__file__).parent
|
|
661
|
-
image1_path = current_dir / "test_files" / "ImportedPhoto.760363950.029251.jpeg"
|
|
662
|
-
image2_path = current_dir / "test_files" / "ImportedPhoto.760363950.031127.jpeg"
|
|
663
|
-
image3_path = current_dir / "test_files" / "ImportedPhoto.760363950.030446.jpeg"
|
|
664
|
-
|
|
665
|
-
# 確保檔案存在
|
|
666
|
-
self.assertTrue(image1_path.exists(), f"Test file not found at {image1_path}")
|
|
667
|
-
self.assertTrue(image2_path.exists(), f"Test file not found at {image2_path}")
|
|
668
|
-
self.assertTrue(image3_path.exists(), f"Test file not found at {image3_path}")
|
|
669
|
-
|
|
670
|
-
# 將絕對路徑轉為字串
|
|
671
|
-
image1_path_str = str(image1_path)
|
|
672
|
-
image2_path_str = str(image2_path)
|
|
673
|
-
image3_path_str = str(image3_path)
|
|
674
|
-
|
|
675
|
-
# 上傳檔案到 tmp_public_url
|
|
676
|
-
tmp_public_url_1 = upload_and_get_tmp_public_url(
|
|
677
|
-
image1_path_str,
|
|
678
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
679
|
-
"sebastian.hsu@gmail.com",
|
|
680
|
-
)
|
|
681
|
-
tmp_public_url_2 = upload_and_get_tmp_public_url(
|
|
682
|
-
image2_path_str,
|
|
683
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
684
|
-
"sebastian.hsu@gmail.com",
|
|
685
|
-
)
|
|
686
|
-
tmp_public_url_3 = upload_and_get_tmp_public_url(
|
|
687
|
-
image3_path_str,
|
|
688
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
689
|
-
"sebastian.hsu@gmail.com",
|
|
690
|
-
)
|
|
691
|
-
|
|
692
|
-
# Test payload
|
|
693
|
-
payload = {
|
|
694
|
-
"graph_name": "langgraph_react_agent",
|
|
695
|
-
"messages": [
|
|
696
|
-
{
|
|
697
|
-
"role": "system",
|
|
698
|
-
"content": "妳是臺灣人,回答要用臺灣繁體中文正式用語,需要的時候也可以用英文,可以親切、俏皮、幽默,但不能隨便輕浮。在使用者合理的要求下請盡量配合他的需求,不要隨便拒絕。",
|
|
699
|
-
},
|
|
700
|
-
{
|
|
701
|
-
"role": "user",
|
|
702
|
-
"content": [
|
|
703
|
-
{
|
|
704
|
-
"type": "text",
|
|
705
|
-
"text": f"以下為使用者上傳的圖片,請參考這些圖片回答使用者的問題:\n\n{tmp_public_url_1}\n{tmp_public_url_2}\n{tmp_public_url_3}\n\n使用者問題:\n\n我要去哪個月台,為什麼?",
|
|
706
|
-
}
|
|
707
|
-
],
|
|
708
|
-
},
|
|
709
|
-
],
|
|
710
|
-
"config": {
|
|
711
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
712
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
713
|
-
},
|
|
714
|
-
}
|
|
715
|
-
|
|
716
|
-
# Make the request
|
|
717
|
-
endpoint = "/api/langgraph/invoke"
|
|
718
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
719
|
-
print("-" * 50)
|
|
720
|
-
|
|
721
|
-
try:
|
|
722
|
-
response = self.api_post(endpoint, payload)
|
|
723
|
-
|
|
724
|
-
# Basic assertions to verify the response
|
|
725
|
-
self.assertIsNotNone(response)
|
|
726
|
-
|
|
727
|
-
# Extract the content field from the response
|
|
728
|
-
if "content" in response:
|
|
729
|
-
response_content = response["content"]
|
|
730
|
-
else:
|
|
731
|
-
self.fail("Response does not contain 'content' field")
|
|
732
|
-
|
|
733
|
-
# Define validation criteria
|
|
734
|
-
validation_criteria = """
|
|
735
|
-
請確認回應是否符合以下條件:
|
|
736
|
-
1. 回應中有明確提及「月台」、「站台」或「platform」等相關詞彙
|
|
737
|
-
2. 回應中有具體指出一個明確的月台方向或號碼,尤其應該包含以下月台號碼其中之一:
|
|
738
|
-
- 5 A-C
|
|
739
|
-
- 5 D-F
|
|
740
|
-
- 5-A-C
|
|
741
|
-
- 5-A/C
|
|
742
|
-
- 5-D-F
|
|
743
|
-
- 5-D/F
|
|
744
|
-
3. 回應中提供了選擇該月台的理由或依據(例如目的地、車次、方向等)
|
|
745
|
-
4. 回應使用了臺灣繁體中文正式用語
|
|
746
|
-
|
|
747
|
-
回應必須能清楚指引使用者應該前往哪個月台,以及為什麼要去那個月台。如果回應中缺少明確的月台指引或理由,則視為測試失敗。
|
|
748
|
-
"""
|
|
749
|
-
|
|
750
|
-
# Validate with LLM
|
|
751
|
-
validation_result = self.validate_with_llm(
|
|
752
|
-
response_content, validation_criteria
|
|
753
|
-
)
|
|
754
|
-
|
|
755
|
-
# Assert that the validation passed
|
|
756
|
-
self.assertTrue(
|
|
757
|
-
validation_result["pass"],
|
|
758
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
759
|
-
)
|
|
760
|
-
|
|
761
|
-
# Additional check for specific platform numbers
|
|
762
|
-
possible_platforms = ["5 A-C", "5 D-F", "5-A-C", "5-A/C", "5-D-F", "5-D/F"]
|
|
763
|
-
platform_found = False
|
|
764
|
-
|
|
765
|
-
for platform in possible_platforms:
|
|
766
|
-
if platform in response_content:
|
|
767
|
-
platform_found = True
|
|
768
|
-
print(f"Found expected platform: {platform}")
|
|
769
|
-
break
|
|
770
|
-
|
|
771
|
-
self.assertTrue(
|
|
772
|
-
platform_found,
|
|
773
|
-
f"Response does not contain any of the expected platform numbers: {possible_platforms}, but get {response_content}",
|
|
774
|
-
)
|
|
775
|
-
|
|
776
|
-
except Exception as e:
|
|
777
|
-
print(
|
|
778
|
-
f"test_langgraph_platform_images_analysis: Test failed with error: {str(e)}"
|
|
779
|
-
)
|
|
780
|
-
raise
|
|
781
|
-
|
|
782
|
-
def test_langgraph_population_analysis(self):
|
|
783
|
-
"""測試PDF人口分析與圖表生成功能,檢查重點:
|
|
784
|
-
1. 是否能正確分析PDF中各縣市的人口數據
|
|
785
|
-
2. 是否生成相關的比較圖表並提供Google Storage URL
|
|
786
|
-
"""
|
|
787
|
-
|
|
788
|
-
current_dir = Path(__file__).parent
|
|
789
|
-
pdf_path = (
|
|
790
|
-
current_dir
|
|
791
|
-
/ "test_files"
|
|
792
|
-
/ "11206_10808人口數(3段年齡組+比率)天下雜誌1.pdf"
|
|
793
|
-
)
|
|
794
|
-
|
|
795
|
-
# 確保檔案存在
|
|
796
|
-
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
797
|
-
|
|
798
|
-
# 將絕對路徑轉為字串
|
|
799
|
-
pdf_path_str = str(pdf_path)
|
|
800
|
-
tmp_public_url = upload_and_get_tmp_public_url(
|
|
801
|
-
pdf_path_str,
|
|
802
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
803
|
-
"sebastian.hsu@gmail.com",
|
|
804
|
-
)
|
|
805
|
-
|
|
806
|
-
# Test payload
|
|
807
|
-
payload = {
|
|
808
|
-
"graph_name": "langgraph_react_agent",
|
|
809
|
-
"messages": [
|
|
810
|
-
{
|
|
811
|
-
"role": "user",
|
|
812
|
-
"content": f"{tmp_public_url} 幫我分析這個檔案,做深度的人口狀況分析,然後產出一個相關的比較圖表給我看。",
|
|
813
|
-
}
|
|
814
|
-
],
|
|
815
|
-
"config": {
|
|
816
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
817
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
818
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
819
|
-
},
|
|
820
|
-
}
|
|
821
|
-
|
|
822
|
-
# Make the request
|
|
823
|
-
endpoint = "/api/langgraph/invoke"
|
|
824
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
825
|
-
print("-" * 50)
|
|
826
|
-
|
|
827
|
-
try:
|
|
828
|
-
response = self.api_post(endpoint, payload)
|
|
829
|
-
|
|
830
|
-
# Basic assertions to verify the response
|
|
831
|
-
self.assertIsNotNone(response)
|
|
832
|
-
|
|
833
|
-
# Extract the content field from the response
|
|
834
|
-
if "content" in response:
|
|
835
|
-
response_content = response["content"]
|
|
836
|
-
else:
|
|
837
|
-
self.fail("Response does not contain 'content' field")
|
|
838
|
-
|
|
839
|
-
# Define validation criteria
|
|
840
|
-
validation_criteria = """
|
|
841
|
-
請確認回應是否符合以下條件:
|
|
842
|
-
1. 回應中有包含各縣市的人口數據分析,至少提及三個以上的縣市名稱及其人口狀況
|
|
843
|
-
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的圖表
|
|
844
|
-
3. 分析內容應該涵蓋人口結構的深度分析,例如年齡分布、老化指數、人口增減等
|
|
845
|
-
4. 是否在回答結尾包含一個笑話
|
|
846
|
-
|
|
847
|
-
所有條件都必須滿足,特別是必須有各縣市的人口分析並含有Google Storage的圖表URL。
|
|
848
|
-
"""
|
|
849
|
-
|
|
850
|
-
# Validate with LLM
|
|
851
|
-
validation_result = self.validate_with_llm(
|
|
852
|
-
response_content, validation_criteria
|
|
853
|
-
)
|
|
854
|
-
|
|
855
|
-
# Assert that the validation passed
|
|
856
|
-
self.assertTrue(
|
|
857
|
-
validation_result["pass"],
|
|
858
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
859
|
-
)
|
|
860
|
-
|
|
861
|
-
# Additional check for Google Storage URL
|
|
862
|
-
self.assertTrue(
|
|
863
|
-
"https://storage.googleapis.com" in response_content,
|
|
864
|
-
"Response does not contain a Google Storage URL",
|
|
865
|
-
)
|
|
866
|
-
|
|
867
|
-
except Exception as e:
|
|
868
|
-
print(
|
|
869
|
-
f"test_langgraph_population_analysis: Test failed with error: {str(e)}"
|
|
870
|
-
)
|
|
871
|
-
raise
|
|
872
|
-
|
|
873
|
-
def test_langgraph_wind_power_flowchart(self):
|
|
874
|
-
"""測試風力發電計畫PDF分析與流程圖生成功能,檢查重點:
|
|
875
|
-
1. 是否能正確分析PDF中的風力發電計畫內容
|
|
876
|
-
2. 是否生成相關的流程圖並提供Google Storage URL
|
|
877
|
-
"""
|
|
878
|
-
|
|
879
|
-
current_dir = Path(__file__).parent
|
|
880
|
-
pdf_path = (
|
|
881
|
-
current_dir
|
|
882
|
-
/ "test_files"
|
|
883
|
-
/ "1120701A海廣離岸風力發電計畫環境影響說明書-C04.PDF"
|
|
884
|
-
)
|
|
885
|
-
|
|
886
|
-
# 確保檔案存在
|
|
887
|
-
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
888
|
-
|
|
889
|
-
# 將絕對路徑轉為字串
|
|
890
|
-
pdf_path_str = str(pdf_path)
|
|
891
|
-
tmp_public_url = upload_and_get_tmp_public_url(
|
|
892
|
-
pdf_path_str,
|
|
893
|
-
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
894
|
-
"sebastian.hsu@gmail.com",
|
|
895
|
-
)
|
|
896
|
-
|
|
897
|
-
# Test payload
|
|
898
|
-
payload = {
|
|
899
|
-
"graph_name": "langgraph_react_agent",
|
|
900
|
-
"messages": [
|
|
901
|
-
{
|
|
902
|
-
"role": "user",
|
|
903
|
-
"content": f"{tmp_public_url} 幫我分析這個檔案,針對風力發電計畫,生成一張流程圖給我。",
|
|
904
|
-
}
|
|
905
|
-
],
|
|
906
|
-
"config": {
|
|
907
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
908
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
909
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
910
|
-
},
|
|
911
|
-
}
|
|
912
|
-
|
|
913
|
-
# Make the request
|
|
914
|
-
endpoint = "/api/langgraph/invoke"
|
|
915
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
916
|
-
print("-" * 50)
|
|
917
|
-
|
|
918
|
-
try:
|
|
919
|
-
response = self.api_post(endpoint, payload)
|
|
920
|
-
|
|
921
|
-
# Basic assertions to verify the response
|
|
922
|
-
self.assertIsNotNone(response)
|
|
923
|
-
|
|
924
|
-
# Extract the content field from the response
|
|
925
|
-
if "content" in response:
|
|
926
|
-
response_content = response["content"]
|
|
927
|
-
else:
|
|
928
|
-
self.fail("Response does not contain 'content' field")
|
|
929
|
-
|
|
930
|
-
# Define validation criteria
|
|
931
|
-
validation_criteria = """
|
|
932
|
-
請確認回應是否符合以下條件:
|
|
933
|
-
1. 回應中有包含風力發電計畫的分析內容,例如計畫目標、執行步驟、環境影響等
|
|
934
|
-
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的流程圖
|
|
935
|
-
3. 分析內容應該專注於風力發電計畫的程序或流程,而非僅是一般性描述
|
|
936
|
-
4. 是否在回答結尾包含一個笑話
|
|
937
|
-
|
|
938
|
-
所有條件都必須滿足,特別是必須有風力發電計畫的分析並含有Google Storage的流程圖URL。
|
|
939
|
-
"""
|
|
940
|
-
|
|
941
|
-
# Validate with LLM
|
|
942
|
-
validation_result = self.validate_with_llm(
|
|
943
|
-
response_content, validation_criteria
|
|
944
|
-
)
|
|
945
|
-
|
|
946
|
-
# Assert that the validation passed
|
|
947
|
-
self.assertTrue(
|
|
948
|
-
validation_result["pass"],
|
|
949
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
950
|
-
)
|
|
951
|
-
|
|
952
|
-
# Additional check for Google Storage URL
|
|
953
|
-
self.assertTrue(
|
|
954
|
-
"https://storage.googleapis.com" in response_content,
|
|
955
|
-
"Response does not contain a Google Storage URL",
|
|
956
|
-
)
|
|
957
|
-
|
|
958
|
-
except Exception as e:
|
|
959
|
-
print(
|
|
960
|
-
f"test_langgraph_wind_power_flowchart: Test failed with error: {str(e)}"
|
|
961
|
-
)
|
|
962
|
-
raise
|
|
963
|
-
|
|
964
|
-
def test_langgraph_oauth_flow_diagram(self):
|
|
965
|
-
"""測試OAuth流程圖生成功能,檢查重點:
|
|
966
|
-
1. 是否能正確生成OAuth認證流程圖
|
|
967
|
-
2. 是否提供Google Storage URL連結到生成的圖表
|
|
968
|
-
"""
|
|
969
|
-
# Test payload based on the curl command
|
|
970
|
-
payload = {
|
|
971
|
-
"graph_name": "langgraph_react_agent",
|
|
972
|
-
"messages": [
|
|
973
|
-
{
|
|
974
|
-
"role": "user",
|
|
975
|
-
"content": "我想做一個 oauth 的流程,幫我生出一個流程表",
|
|
976
|
-
}
|
|
977
|
-
],
|
|
978
|
-
"config": {
|
|
979
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
980
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
981
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
982
|
-
},
|
|
983
|
-
}
|
|
984
|
-
|
|
985
|
-
# Make the request
|
|
986
|
-
endpoint = "/api/langgraph/invoke"
|
|
987
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
988
|
-
print("-" * 50)
|
|
989
|
-
|
|
990
|
-
try:
|
|
991
|
-
response = self.api_post(endpoint, payload)
|
|
992
|
-
|
|
993
|
-
# Basic assertions to verify the response
|
|
994
|
-
self.assertIsNotNone(response)
|
|
995
|
-
|
|
996
|
-
# Extract the content field from the response
|
|
997
|
-
if "content" in response:
|
|
998
|
-
response_content = response["content"]
|
|
999
|
-
else:
|
|
1000
|
-
self.fail("Response does not contain 'content' field")
|
|
1001
|
-
|
|
1002
|
-
# Define validation criteria
|
|
1003
|
-
validation_criteria = """
|
|
1004
|
-
請確認回應是否符合以下條件:
|
|
1005
|
-
1. 回應中有詳細描述OAuth認證流程的步驟,必須包含關鍵步驟如授權請求、令牌交換等
|
|
1006
|
-
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的流程圖
|
|
1007
|
-
3. 回應應該提供清晰的OAuth流程解釋,包括不同參與者(如用戶、客戶端應用、授權伺服器等)之間的互動
|
|
1008
|
-
4. 是否在回答結尾包含一個笑話
|
|
1009
|
-
|
|
1010
|
-
所有條件都必須滿足,特別是必須有OAuth流程的詳細描述,並含有Google Storage的流程圖URL。
|
|
1011
|
-
"""
|
|
1012
|
-
|
|
1013
|
-
# Validate with LLM
|
|
1014
|
-
validation_result = self.validate_with_llm(
|
|
1015
|
-
response_content, validation_criteria
|
|
1016
|
-
)
|
|
1017
|
-
|
|
1018
|
-
# Assert that the validation passed
|
|
1019
|
-
self.assertTrue(
|
|
1020
|
-
validation_result["pass"],
|
|
1021
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1022
|
-
)
|
|
1023
|
-
|
|
1024
|
-
# Additional check for Google Storage URL
|
|
1025
|
-
self.assertTrue(
|
|
1026
|
-
"https://storage.googleapis.com" in response_content,
|
|
1027
|
-
"Response does not contain a Google Storage URL",
|
|
1028
|
-
)
|
|
1029
|
-
|
|
1030
|
-
except Exception as e:
|
|
1031
|
-
print(
|
|
1032
|
-
f"test_langgraph_oauth_flow_diagram: Test failed with error: {str(e)}"
|
|
1033
|
-
)
|
|
1034
|
-
raise
|
|
1035
|
-
|
|
1036
|
-
def test_langgraph_moda_news_dall_e(self):
|
|
1037
|
-
"""測試多節點處理數位發展部新聞流程,檢查重點:
|
|
1038
|
-
1. 是否會抓到今天日期,或截至今天日期的數位發展部相關新聞
|
|
1039
|
-
2. 是否有評分新聞 (1-10分),並標示出「最可愛」的新聞
|
|
1040
|
-
3. 是否有產出一張圖片,並帶有 URL
|
|
1041
|
-
"""
|
|
1042
|
-
# Test payload based on the curl command
|
|
1043
|
-
payload = {
|
|
1044
|
-
"graph_name": "langgraph_react_agent",
|
|
1045
|
-
"messages": [
|
|
1046
|
-
{
|
|
1047
|
-
"role": "user",
|
|
1048
|
-
"content": "好,那個你幫我那個啟動幾個多個節點,然後第一個節點請你幫我上網搜尋。 上網搜尋今天那個我們那個數位發展部的夥伴,或者最近一個禮拜數位發展部的那個夥伴有沒有什麼新聞好。 然後第二個節點,你幫我做一件事,你幫我做幫我把這些新聞評分數,評分一到十分。 那哪個新聞你覺得最可愛。 然後第三個節點,你幫我做一件事。 就是你把這個分數最可愛的那一個新聞挑出來以後,你幫我生成一個prompt,這個prompt是我要把你丟進打理畫圖用的prompt。那第四個節點你才真的呼叫打理把那個圖給畫出來,你幫我依序執行這個這個工作流程好不好,謝謝。",
|
|
1049
|
-
}
|
|
1050
|
-
],
|
|
1051
|
-
"config": {
|
|
1052
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1053
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1054
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
1055
|
-
},
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
# Make the request
|
|
1059
|
-
endpoint = "/api/langgraph/invoke"
|
|
1060
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1061
|
-
print("-" * 50)
|
|
1062
|
-
|
|
1063
|
-
try:
|
|
1064
|
-
response = self.api_post(endpoint, payload)
|
|
1065
|
-
|
|
1066
|
-
# Basic assertions to verify the response
|
|
1067
|
-
self.assertIsNotNone(response)
|
|
1068
|
-
|
|
1069
|
-
# Extract the content field from the response
|
|
1070
|
-
if "content" in response:
|
|
1071
|
-
response_content = response["content"]
|
|
1072
|
-
else:
|
|
1073
|
-
self.fail("Response does not contain 'content' field")
|
|
1074
|
-
|
|
1075
|
-
# Define validation criteria based on the test requirements
|
|
1076
|
-
validation_criteria = f"""
|
|
1077
|
-
請確認回應是否符合以下條件:
|
|
1078
|
-
1. 回應中有包含今天({self.local_time.strftime("%Y-%m-%d")})或最近一週內的數位發展部相關新聞資訊
|
|
1079
|
-
2. 回應中有對新聞進行1-10分的評分,並明確指出哪則新聞「最可愛」或分數最高
|
|
1080
|
-
3. 回應中包含至少一個圖片URL(通常是以http或https開頭的網址)
|
|
1081
|
-
4. 是否在回答結尾包含一個笑話
|
|
1082
|
-
|
|
1083
|
-
所有條件都必須滿足,特別是必須有數位發展部相關新聞、評分結果以及最終生成的圖片URL。
|
|
1084
|
-
"""
|
|
1085
|
-
|
|
1086
|
-
# Validate with LLM
|
|
1087
|
-
validation_result = self.validate_with_llm(
|
|
1088
|
-
response_content, validation_criteria
|
|
1089
|
-
)
|
|
1090
|
-
|
|
1091
|
-
# Assert that the validation passed
|
|
1092
|
-
self.assertTrue(
|
|
1093
|
-
validation_result["pass"],
|
|
1094
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1095
|
-
)
|
|
1096
|
-
|
|
1097
|
-
# Additional check for an image URL
|
|
1098
|
-
self.assertTrue(
|
|
1099
|
-
"http" in response_content.lower()
|
|
1100
|
-
and (
|
|
1101
|
-
"jpg" in response_content.lower()
|
|
1102
|
-
or "png" in response_content.lower()
|
|
1103
|
-
or "https://storage.googleapis.com" in response_content
|
|
1104
|
-
),
|
|
1105
|
-
"Response does not contain a valid image URL",
|
|
1106
|
-
)
|
|
1107
|
-
|
|
1108
|
-
except Exception as e:
|
|
1109
|
-
print(f"test_langgraph_moda_news_dall_e: Test failed with error: {str(e)}")
|
|
1110
|
-
raise
|
|
1111
|
-
|
|
1112
|
-
def test_langgraph_global_disaster_news(self):
|
|
1113
|
-
"""測試深度研究災難新聞流程,檢查重點:
|
|
1114
|
-
1. 是否有收集全球災難新聞
|
|
1115
|
-
2. 是否以表格方式呈現災難資料
|
|
1116
|
-
3. 是否提供新聞來源
|
|
1117
|
-
"""
|
|
1118
|
-
# Test payload based on the curl command
|
|
1119
|
-
payload = {
|
|
1120
|
-
"graph_name": "langgraph_react_agent",
|
|
1121
|
-
"messages": [
|
|
1122
|
-
{
|
|
1123
|
-
"role": "user",
|
|
1124
|
-
"content": "請幫我進行深度研究,深度研究時請遵循以下三個步驟\n第一步驟:\n身為一個專業的全球新聞蒐集分析人員,請透過網路幫我收集Google News、路透社、美聯社、CNN、BBC、法新社、歐洲傳媒應急中心、公共透視網路、台灣聯合報及東森新聞,不使用不可信賴媒體及模擬資料,盡力確保可收集到全球的災難新聞。產生的研究報告文件名稱請以「xxxx年xx月xx日 全球災難新聞收集與追蹤」這樣的格式生成。收集時間請以UTC+8時區為基準,收集從2025年2月24日15:00到2025年2月25日15:00 的24 小時內,全球在時區內發生的災難事件及發生時間,包括大型自然災害或人為災難的人數統計,包括「傷亡」、「失蹤」、「受影響」、「流離失所」、「避難」等。自然災難類型包括但不限於地震、風災、火山爆發、寒流、大雪、冰雹、雪崩、土石流、野火、山火之類的極端氣候災難,人為災害包括 空難、戰爭、大型交通事故、海難、建物倒塌、疫情、中毒等並整理成表格,以繁體中文輸出,表格名請加上當天的年月日,格式為「xxxx年xx月xx日」並按照亞洲、歐洲、美洲、大洋洲、非洲等五大洲排列,後面要加上國家、省市別做完整地點呈現\n第二個表格請搜集以2025年2月24日為基準過去 96 小時的全球新聞中的災難報導。請確認事件發生時間在區間內,若是非區間內發生,請在說明欄清楚說明原因。並再三確認報導更新時間是否在區間內,也就是從前三天到當天,四天中發生的災難後續報導。\n第三個表格請就第一和第二個表格中收集到的災難事件中,逐條就每個災難進行250字的災難摘要及資料來源連結。並收集有關房屋(棟)的損壞統計,包括「受損」、「毁損」等。請務必以表格呈現,不要逐條展示。\n第二步驟:\n我要復盤上述資料都來自於可信賴的國際或台灣新聞媒體即時新聞報導,並且要找到三個不同的資訊來源,交差比對確認災難真實發生的時間點。第一個表格要有詳細的災難發生地點,每條災難收集的時間條件是指災難發生時間而非新聞發布時間在時間區間內,若災難發生時間不在時間區間內,請移到第二個表格。第二個表格是在過去96個小時不重複第一個表格的時間區間中新聞媒體對於災難的後續報導,第三個部份也請用表格呈現,而不是條列式。請檢視每個表格的災難資料,並合併相同的災難事件,確保每條事件只有一筆, 我很怕你使用到不可信的網路媒體資料,如:維基百科或是災難預言、天氣預警、模擬訊息或是你預訓練的資料Youtube及專題報導等。 若無傷亡或防屋毀損實際統計數據,請勿收集。\n第三步驟:\n請將下列附加檔案的表格一和表格二匯的每條災難事件透過網路可信賴媒體交叉比對時間和真實性後,匯整到原有的表格一和二中並合併相同的災難資訊,重新整理完整的表格三。再重新檢查每筆資料都符合各表格的時間區間及規範,重新盤點100遍,幫我整理出最完整的表格一到三。每筆資料請幫我再三搜尋可信賴媒體進行交差比對,務求每條事件都在真實世界中發生,地點明確,發生時間可驗證,若無傷亡資料就不收集。要確認表格一加上表格二的條目,能完整在表格三中呈現,不可多也不可少。",
|
|
1125
|
-
}
|
|
1126
|
-
],
|
|
1127
|
-
"config": {
|
|
1128
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1129
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1130
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
1131
|
-
},
|
|
1132
|
-
}
|
|
1133
|
-
|
|
1134
|
-
# Make the request
|
|
1135
|
-
endpoint = "/api/langgraph/invoke"
|
|
1136
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1137
|
-
print("-" * 50)
|
|
1138
|
-
|
|
1139
|
-
try:
|
|
1140
|
-
response = self.api_post(endpoint, payload)
|
|
1141
|
-
|
|
1142
|
-
# Basic assertions to verify the response
|
|
1143
|
-
self.assertIsNotNone(response)
|
|
1144
|
-
|
|
1145
|
-
# Extract the content field from the response
|
|
1146
|
-
if "content" in response:
|
|
1147
|
-
response_content = response["content"]
|
|
1148
|
-
else:
|
|
1149
|
-
self.fail("Response does not contain 'content' field")
|
|
1150
|
-
|
|
1151
|
-
# Define validation criteria based on the test requirements
|
|
1152
|
-
validation_criteria = """
|
|
1153
|
-
請確認回應是否符合以下條件:
|
|
1154
|
-
1. 回應中包含災難新聞信息(至少提到了一些具體災難事件)
|
|
1155
|
-
2. 回應中有表格呈現(HTML表格標籤或是文字表格形式呈現災難數據)
|
|
1156
|
-
3. 回應中提供了新聞來源(至少包含一個可識別的媒體來源名稱如CNN、BBC、路透社等)
|
|
1157
|
-
4. 回應中有提及災難事件的類型(自然災害或人為災害)
|
|
1158
|
-
5. 回應中有提及災難事件的地理位置(國家、城市等)
|
|
1159
|
-
6. 回應中是否在結尾包含一個笑話
|
|
1160
|
-
|
|
1161
|
-
所有條件都必須滿足,特別是必須有災難新聞、表格呈現方式及新聞來源引用。
|
|
1162
|
-
"""
|
|
1163
|
-
|
|
1164
|
-
# Validate with LLM
|
|
1165
|
-
validation_result = self.validate_with_llm(
|
|
1166
|
-
response_content, validation_criteria
|
|
1167
|
-
)
|
|
1168
|
-
|
|
1169
|
-
# Assert that the validation passed
|
|
1170
|
-
self.assertTrue(
|
|
1171
|
-
validation_result["pass"],
|
|
1172
|
-
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1173
|
-
)
|
|
1174
|
-
|
|
1175
|
-
# Additional checks for tables and sources
|
|
1176
|
-
self.assertTrue(
|
|
1177
|
-
"|" in response_content
|
|
1178
|
-
or "<table" in response_content.lower()
|
|
1179
|
-
or "表格" in response_content,
|
|
1180
|
-
"Response does not appear to contain any tables",
|
|
1181
|
-
)
|
|
1182
|
-
|
|
1183
|
-
# # Check for news sources
|
|
1184
|
-
# news_sources = [
|
|
1185
|
-
# "CNN",
|
|
1186
|
-
# "BBC",
|
|
1187
|
-
# "路透社",
|
|
1188
|
-
# "美聯社",
|
|
1189
|
-
# "法新社",
|
|
1190
|
-
# "聯合報",
|
|
1191
|
-
# "東森",
|
|
1192
|
-
# ]
|
|
1193
|
-
# sources_found = any(source in response_content for source in news_sources)
|
|
1194
|
-
# self.assertTrue(
|
|
1195
|
-
# sources_found,
|
|
1196
|
-
# "Response does not reference any recognizable news sources",
|
|
1197
|
-
# )
|
|
1198
|
-
|
|
1199
|
-
except Exception as e:
|
|
1200
|
-
print(
|
|
1201
|
-
f"test_langgraph_global_disaster_news: Test failed with error: {str(e)}"
|
|
1202
|
-
)
|
|
1203
|
-
raise
|
|
1204
|
-
|
|
1205
|
-
def test_langgraph_date_time_comparison(self):
|
|
1206
|
-
"""測試日期時間比較功能,檢查重點:
|
|
1207
|
-
1. 當使用者僅指定日期時間而未明確要求比較時,agent 是否能自動使用 current_date_time 工具獲取當前時間
|
|
1208
|
-
2. agent 是否能自動使用 compare_date_time 工具比較指定時間與當前時間
|
|
1209
|
-
3. agent 是否能正確判斷指定時間是過去還是未來並提供解釋
|
|
1210
|
-
"""
|
|
1211
|
-
# 共用的系統提示
|
|
1212
|
-
system_prompt = "如果使用者的問題中有指定日期時間,不要預設它是未來或過去,一定要先使用 current_date_time 和 compare_date_time 這兩個工具,以取得現在的日期時間並判斷使用者指定的日期時間是過去或未來,然後再進行後續的動作。"
|
|
1213
|
-
|
|
1214
|
-
# 共用的配置
|
|
1215
|
-
base_config = {
|
|
1216
|
-
"system_prompt": system_prompt,
|
|
1217
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1218
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
# 測試過去時間
|
|
1222
|
-
past_payload = {
|
|
1223
|
-
"graph_name": "langgraph_react_agent",
|
|
1224
|
-
"messages": [
|
|
1225
|
-
{
|
|
1226
|
-
"role": "user",
|
|
1227
|
-
"content": "2020年1月1日發生了什麼重要事件?",
|
|
1228
|
-
}
|
|
1229
|
-
],
|
|
1230
|
-
"config": base_config,
|
|
1231
|
-
}
|
|
1232
|
-
|
|
1233
|
-
# 測試未來時間
|
|
1234
|
-
future_payload = {
|
|
1235
|
-
"graph_name": "langgraph_react_agent",
|
|
1236
|
-
"messages": [
|
|
1237
|
-
{
|
|
1238
|
-
"role": "user",
|
|
1239
|
-
"content": "2030年12月31日會有什麼重要活動?",
|
|
1240
|
-
}
|
|
1241
|
-
],
|
|
1242
|
-
"config": base_config,
|
|
1243
|
-
}
|
|
1244
|
-
|
|
1245
|
-
# 測試端點
|
|
1246
|
-
endpoint = "/api/langgraph/invoke"
|
|
1247
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1248
|
-
print("-" * 50)
|
|
1249
|
-
|
|
1250
|
-
try:
|
|
1251
|
-
# 測試過去時間
|
|
1252
|
-
print("Testing past date comparison...")
|
|
1253
|
-
past_response = self.api_post(endpoint, past_payload)
|
|
1254
|
-
self.assertIsNotNone(past_response)
|
|
1255
|
-
|
|
1256
|
-
if "content" in past_response:
|
|
1257
|
-
past_response_content = past_response["content"]
|
|
1258
|
-
else:
|
|
1259
|
-
self.fail("Response does not contain 'content' field")
|
|
1260
|
-
|
|
1261
|
-
# 定義過去時間的驗證標準
|
|
1262
|
-
past_validation_criteria = """
|
|
1263
|
-
請確認回應是否符合以下條件:
|
|
1264
|
-
1. 回應中是否提到或暗示 2020年1月1日 是過去的時間
|
|
1265
|
-
2. 回應中是否有跡象表明 agent 使用了 current_date_time 工具獲取當前時間(例如提到「根據當前時間」、「現在是...」等)
|
|
1266
|
-
3. 回應中是否有跡象表明 agent 使用了 compare_date_time 工具比較時間(例如提到「比較結果」、「早於當前時間」等)
|
|
1267
|
-
4. 回應中是否包含關於 2020年1月1日 發生的重要事件的資訊
|
|
1268
|
-
"""
|
|
1269
|
-
|
|
1270
|
-
# 使用 LLM 驗證
|
|
1271
|
-
past_validation_result = self.validate_with_llm(
|
|
1272
|
-
past_response_content, past_validation_criteria
|
|
1273
|
-
)
|
|
1274
|
-
|
|
1275
|
-
# 驗證結果
|
|
1276
|
-
self.assertTrue(
|
|
1277
|
-
past_validation_result["pass"],
|
|
1278
|
-
f"LLM validation failed for past date in {self._testMethodName}: {past_validation_result['reason']}, LLM response: {past_response_content}",
|
|
1279
|
-
)
|
|
1280
|
-
|
|
1281
|
-
# 測試未來時間
|
|
1282
|
-
print("Testing future date comparison...")
|
|
1283
|
-
future_response = self.api_post(endpoint, future_payload)
|
|
1284
|
-
self.assertIsNotNone(future_response)
|
|
1285
|
-
|
|
1286
|
-
if "content" in future_response:
|
|
1287
|
-
future_response_content = future_response["content"]
|
|
1288
|
-
else:
|
|
1289
|
-
self.fail("Response does not contain 'content' field")
|
|
1290
|
-
|
|
1291
|
-
# 定義未來時間的驗證標準
|
|
1292
|
-
future_validation_criteria = """
|
|
1293
|
-
請確認回應是否符合以下條件:
|
|
1294
|
-
1. 回應中是否提到或暗示 2030年12月31日 是未來的時間
|
|
1295
|
-
2. 回應中是否有跡象表明 agent 使用了 current_date_time 工具獲取當前時間(例如提到「根據當前時間」、「現在是...」等)
|
|
1296
|
-
3. 回應中是否有跡象表明 agent 使用了 compare_date_time 工具比較時間(例如提到「比較結果」、「晚於當前時間」等)
|
|
1297
|
-
4. 回應中是否適當地處理了關於未來日期的問題(例如表明無法預測未來具體事件,但可能提供一些合理的推測或建議)
|
|
1298
|
-
"""
|
|
1299
|
-
|
|
1300
|
-
# 使用 LLM 驗證
|
|
1301
|
-
future_validation_result = self.validate_with_llm(
|
|
1302
|
-
future_response_content, future_validation_criteria
|
|
1303
|
-
)
|
|
1304
|
-
|
|
1305
|
-
# 驗證結果
|
|
1306
|
-
self.assertTrue(
|
|
1307
|
-
future_validation_result["pass"],
|
|
1308
|
-
f"LLM validation failed for future date in {self._testMethodName}: {future_validation_result['reason']}, LLM response: {future_response_content}",
|
|
1309
|
-
)
|
|
1310
|
-
|
|
1311
|
-
except Exception as e:
|
|
1312
|
-
print(
|
|
1313
|
-
f"test_langgraph_date_time_comparison: Test failed with error: {str(e)}"
|
|
1314
|
-
)
|
|
1315
|
-
raise
|
|
1316
|
-
|
|
1317
|
-
def test_langgraph_react_agent_business_flow(self):
|
|
1318
|
-
"""Test the langgraph_react_agent with a business flow example."""
|
|
1319
|
-
# Test payload
|
|
1320
|
-
payload = {
|
|
1321
|
-
"graph_name": "langgraph_react_agent",
|
|
1322
|
-
"messages": [
|
|
1323
|
-
{
|
|
1324
|
-
"role": "user",
|
|
1325
|
-
"content": "我想要請你 給我 一個業務流的範例,我只要三個節點,然後我還要針對這個業務流範例當中的其中中間的一個工作流程,然後來 進行示意,然後我那個工作流程也只需要三個節點,那原因是因為我要做成簡報,這樣子比較清晰容易看懂,那業務流 他是人如何工作的,的,的一個重點,記錄人員互動、決策跟情緒體驗,那也因此在業務流的時候呢,我需要請你幫我畫成使用者旅程地圖,裡面有酷酷點跟笑笑點,對不起,那請你節點幫我增加為五個好,然後再來工作流程的地方,它則是聚焦在任務跟文件,還有系統間的流動,它是一個操作的程序跟規則,那我,我也需要,我要修改前面的說法,要改成五個節點,那業務流是使用使用者旅程庫庫地圖,那工作流程的話,我希望它是一個,也許是一個工作流程程圖,或者是時序圖,然後看你覺得哪一個的表現比較明確,那這個內容呢,我需要以內政部的任何一個轄下的任何一個業務來進行舉例,然後我我想要請你幫我現在畫出來。",
|
|
1326
|
-
},
|
|
1327
|
-
{
|
|
1328
|
-
"role": "assistant",
|
|
1329
|
-
"content": "**我瞭解您需要一個業務流程的範例,特別是以內政部的業務為例,包含:**\n1. **一個使用者旅程地圖(業務流)- 5個節點,記錄人員互動、決策和情緒體驗**\n2. **一個工作流程程圖或時序圖 - 5個節點,聚焦在任務、文件和系統間的流動**\n**我會先幫您創建這兩個圖表。讓我們以內政部戶政司的「結婚登記」業務為例。**\n**首先,讓我建立使用者旅程地圖(業務流):**",
|
|
1330
|
-
},
|
|
1331
|
-
{
|
|
1332
|
-
"role": "user",
|
|
1333
|
-
"content": "等一下,你在第二個跟第三個給我的連結啊,裡面都是亂買,你到底在幹什麼?你重新確認一下。",
|
|
1334
|
-
},
|
|
1335
|
-
{"role": "assistant", "content": ""},
|
|
1336
|
-
{"role": "user", "content": "你還好嗎?你有沒有在動作?"},
|
|
1337
|
-
],
|
|
1338
|
-
"config": {
|
|
1339
|
-
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1340
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1341
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
1342
|
-
},
|
|
1343
|
-
}
|
|
1344
|
-
|
|
1345
|
-
# Make the request
|
|
1346
|
-
endpoint = "/api/langgraph/invoke"
|
|
1347
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1348
|
-
print("-" * 50)
|
|
1349
|
-
|
|
1350
|
-
try:
|
|
1351
|
-
response = self.api_post(endpoint, payload)
|
|
1352
|
-
|
|
1353
|
-
# Basic assertions to verify the response
|
|
1354
|
-
self.assertIsNotNone(response)
|
|
1355
|
-
|
|
1356
|
-
# Check for error response format
|
|
1357
|
-
if "detail" in response:
|
|
1358
|
-
error_message = response["detail"]
|
|
1359
|
-
# Fail if the error is about empty message content
|
|
1360
|
-
self.assertNotIn(
|
|
1361
|
-
"messages.3: all messages must have non-empty content except for the optional final assistant message",
|
|
1362
|
-
error_message,
|
|
1363
|
-
f"API returned expected error about empty content: {error_message}",
|
|
1364
|
-
)
|
|
1365
|
-
print(
|
|
1366
|
-
f"API returned an error, but not the empty content error: {error_message}"
|
|
1367
|
-
)
|
|
1368
|
-
# Check for successful response format
|
|
1369
|
-
else:
|
|
1370
|
-
# For successful responses, verify content key exists at the top level
|
|
1371
|
-
self.assertIn("content", response, "Response missing 'content' field")
|
|
1372
|
-
print(f"Response received successfully with content!")
|
|
1373
|
-
|
|
1374
|
-
except Exception as e:
|
|
1375
|
-
self.fail(f"Error testing API: {str(e)}")
|
|
1376
|
-
|
|
1377
|
-
def test_auth_token_verify_api(self):
|
|
1378
|
-
"""測試 auth token verification API,檢查重點:
|
|
1379
|
-
1. 有效 token 的驗證
|
|
1380
|
-
2. 無效 token 的錯誤處理
|
|
1381
|
-
3. 缺少 token 參數的錯誤處理
|
|
1382
|
-
"""
|
|
1383
|
-
# Test with valid-looking token
|
|
1384
|
-
endpoint = "/api/auth/token_verify"
|
|
1385
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1386
|
-
print("-" * 50)
|
|
1387
|
-
|
|
1388
|
-
try:
|
|
1389
|
-
# Test case 1: Valid token format (though may not be valid in backend)
|
|
1390
|
-
valid_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.token"
|
|
1391
|
-
|
|
1392
|
-
# Use form data for POST request
|
|
1393
|
-
import requests
|
|
1394
|
-
url = f"{self.base_url}{endpoint}"
|
|
1395
|
-
response = requests.post(url, data={"access_token": valid_token})
|
|
1396
|
-
|
|
1397
|
-
# Expect either 200 (valid) or 401 (invalid token) or 500 (service unavailable)
|
|
1398
|
-
self.assertIn(response.status_code, [200, 401, 500],
|
|
1399
|
-
f"Unexpected status code: {response.status_code}")
|
|
1400
|
-
|
|
1401
|
-
if response.status_code == 200:
|
|
1402
|
-
json_response = response.json()
|
|
1403
|
-
self.assertIn("is_success", json_response)
|
|
1404
|
-
print(f"Token verification succeeded: {json_response}")
|
|
1405
|
-
elif response.status_code == 401:
|
|
1406
|
-
json_response = response.json()
|
|
1407
|
-
self.assertIn("detail", json_response)
|
|
1408
|
-
self.assertIn("Invalid", json_response["detail"])
|
|
1409
|
-
print(f"Token verification failed as expected: {json_response}")
|
|
1410
|
-
elif response.status_code == 500:
|
|
1411
|
-
# Service might not be configured
|
|
1412
|
-
json_response = response.json()
|
|
1413
|
-
print(f"Service unavailable (expected in dev): {json_response}")
|
|
1414
|
-
|
|
1415
|
-
# Test case 2: Missing token parameter
|
|
1416
|
-
response_missing = requests.post(url, data={})
|
|
1417
|
-
self.assertEqual(response_missing.status_code, 422,
|
|
1418
|
-
"Missing token should return 422 validation error")
|
|
1419
|
-
|
|
1420
|
-
json_response_missing = response_missing.json()
|
|
1421
|
-
self.assertIn("detail", json_response_missing)
|
|
1422
|
-
print(f"Missing token handled correctly: {json_response_missing}")
|
|
1423
|
-
|
|
1424
|
-
# Test case 3: Empty token
|
|
1425
|
-
response_empty = requests.post(url, data={"access_token": ""})
|
|
1426
|
-
self.assertIn(response_empty.status_code, [400, 401, 500],
|
|
1427
|
-
"Empty token should return 400, 401, or 500")
|
|
1428
|
-
print(f"Empty token handled with status: {response_empty.status_code}")
|
|
1429
|
-
|
|
1430
|
-
except Exception as e:
|
|
1431
|
-
print(f"test_auth_token_verify_api: Test failed with error: {str(e)}")
|
|
1432
|
-
raise
|
|
1433
|
-
|
|
1434
|
-
def test_langgraph_react_agent_social_housing(self):
|
|
1435
|
-
"""Test the langgraph_react_agent with social housing application analysis."""
|
|
1436
|
-
# Test payload
|
|
1437
|
-
payload = {
|
|
1438
|
-
"graph_name": "langgraph_react_agent",
|
|
1439
|
-
"messages": [
|
|
1440
|
-
{
|
|
1441
|
-
"role": "user",
|
|
1442
|
-
"content": "幫我分析這分申請資料,我的檔案已經在你的system prompt裡面了,不需要再做其他的處理,直接使用system prompt裡的資料進行分析",
|
|
1443
|
-
}
|
|
1444
|
-
],
|
|
1445
|
-
"config": {
|
|
1446
|
-
"system_prompt": "<Context>\n您是一位非常細心且專業的中華民國內政部審查社宅入住資格的資深審查員\n</Context>\n\n<Objective>\n您的目標是在審查使用者是否符合社宅入住資格\n</Objective>\n\n<Style>\n請保持中立及客觀,以精煉的方式分析資料,確保過程簡潔明瞭。\n</Style>\n\n<Tone>\n專業且耐心的語氣。\n</Tone>\n\n<Audience>\n此流程設計專門供機關內的相關同仁使用,他們需清楚知道哪些是合格的資料,哪些不是,以便後續決策。\n<Audience>\n\n<第一步驟>\n<審查規則>\n1. 年滿18歲(含)以上之中華民國國民\n2. 有於北北基桃設籍、就學、就業任一需求者\n3. 於北北基桃無自有住宅者或個別持有小於40平方公尺\n4. 家庭成員每人每月平均所得不超過新臺幣59,150元(舉例:新臺幣60,000元就是超過59,150元)\n</審查規則>\n\n<Response>\n1. 請幫我列出完整的審查結果,通過打V,不通過打X,並且列出不通過的原因\n2. 幫我把上面的內容產出一個表格回傳給我\n</Response>\n</第一步驟>\n\n<第二步驟>\n幫我依照<第一步驟>產生的結果,生產一個html的頁面報告,html頁面裡面要顯示以下資訊,你不要直接給我html的程式\n1. 請幫我畫一張圓餅圖分析年齡分佈\n2. 請幫我把原始審查資料跟<第一步驟>的產出做合併\n3. 請幫我做一個搜尋工具快速搜尋審查資料\n</第二步驟>\n\n\n\n\n\n以下為附加檔案內容:\n\n檔名:\n社宅申請模擬資料.csv\n檔案內容:\n申請人姓名,性別,出生年月日,婚姻狀況,身分證字號,電話,電子郵件,職業,戶籍地址,戶籍地址是否承租,通訊地址,通訊地址是否承租,緊急聯絡人,稱謂,緊急聯絡人電話,申請類別,申請戶類型,家具承租方案,配偶姓名,配偶身分證字號,家庭成員數量,持有住宅平方公尺數,平均家庭成員月收入\n林志明,男,1985-06-12,已婚,A123456789,0912-345-678,zhiming.lin@example.com,軟體工程師,新北市板橋區文化路一段100號5樓,是,新北市板橋區文化路一段100號5樓,是,林志豪,兄弟,0922-123-456,設籍,一般戶,同步租,王美玲,B234567890,3,0,45000\n陳雅婷,女,2001-05-06,未婚,B287654321,0933-876-543,yating.chen@example.com,學生,台北市信義區松仁路50號12樓,否,新北市新莊區中正路200號3樓,是,陳大明,父親,0955-987-654,就學,一般戶,買斷,,,1,0,38000\n張家豪,男,1978-11-30,已婚,C198765432,0977-654-321,jiahao.zhang@example.com,公務員,台北市大安區和平東路二段106號7樓,是,台北市大安區和平東路二段106號7樓,是,張明德,父親,0910-234-567,設籍,現職警消人員,同步租,李佩珊,D123456789,4,0,52000\n黃麗華,女,1965-08-15,喪偶,E234567890,0988-765-432,lihua.huang@example.com,退休教師,新北市三重區重新路一段88號4樓,否,新北市三重區重新路一段88號4樓,否,黃志成,兒子,0923-456-789,設籍,65歲以上老人,同步租,,,1,0,25000\n吳建志,男,1982-04-20,已婚,F123456789,0932-123-456,jianzhih.wu@example.com,銀行經理,台北市中山區南京東路三段25號9樓,是,台北市中山區南京東路三段25號9樓,是,吳大維,父親,0912-876-543,就業,一般戶,買斷,林美琪,G234567890,5,15,60000\n李小芳,女,1992-12-05,已婚,H123456789,0956-789-123,xiaofang.li@example.com,設計師,基隆市中正區中正路100號3樓,是,台北市松山區民生東路四段133號6樓,是,李大中,父親,0933-222-111,就業,未成年子女三人以上,同步租,王大明,I234567890,5,0,42000\n王俊傑,男,1988-07-15,未婚,J123456789,0978-456-123,junjie.wang@example.com,工程師,桃園市中壢區中央西路二段30號5樓,否,新北市新店區北新路三段100號7樓,是,王大華,父親,0910-876-543,就業,身心障礙,買斷,,,1,0,35000\n蔡美玲,女,1975-09-28,離婚,K123456789,0933-789-456,meiling.tsai@example.com,會計師,新北市永和區永和路一段50號4樓,是,新北市永和區永和路一段50號4樓,是,蔡明哲,兄弟,0922-333-444,就業,特殊境遇家庭,同步租,,,2,0,40000\n鄭志偉,男,1980-02-14,已婚,L123456789,0955-123-789,zhiwei.zheng@example.com,教師,台北市文山區木柵路一段100號3樓,否,台北市文山區木柵路一段100號3樓,否,鄭大勇,父親,0912-345-678,就業,一般戶,買斷,陳美美,M234567890,3,20,48000\n林美華,女,2003-01-03,未婚,N123456789,0978-789-123,meihua.lin@example.com,學生,新北市汐止區大同路一段150號5樓,是,新北市汐止區大同路一段150號5樓,是,林大明,父親,0933-456-789,就學,一般戶,同步租,,,1,0,36000\n張大為,男,1972-06-30,已婚,O123456789,0910-234-567,dawei.zhang@example.com,建築師,台北市大安區復興南路一段200號7樓,否,台北市大安區復興南路一段200號7樓,否,張小明,兒子,0922-123-456,設籍,一般戶,買斷,王麗麗,P234567890,3,25,65000\n陳俊宏,男,1990-08-12,未婚,Q123456789,0933-222-111,junhong.chen@example.com,軍職人員,桃園市龜山區文化一路100號5樓,是,桃園市龜山區文化一路100號5樓,是,陳大勇,父親,0955-123-456,就業,軍職人員,同步租,,,1,0,42000\n楊雅琪,女,1987-04-15,已婚,R123456789,0978-456-789,yaqi.yang@example.com,行銷經理,新北市中和區中和路100號6樓,是,新北市中和區中和路100號6樓,是,楊大明,父親,0912-345-678,設籍,一般戶,買斷,李志明,S234567890,2,0,55000\n劉大偉,男,1968-12-25,已婚,T123456789,0955-789-123,dawei.liu@example.com,計程車司機,基隆市安樂區安樂路二段50號3樓,否,基隆市安樂區安樂路二段50號3樓,否,劉小明,兒子,0933-456-789,設籍,低收入戶,同步租,張美美,U234567890,4,0,28000\n高美玲,女,1993-03-08,未婚,V123456789,0910-123-456,meiling.gao@example.com,設計師,台北市內湖區內湖路一段300號8樓,是,台北市內湖區內湖路一段300號8樓,是,高大明,父親,0922-789-123,就業,一般戶,買斷,,,1,0,38000\n鄭美美,女,1983-09-18,離婚,W123456789,0933-789-123,meimei.zheng@example.com,餐廳經理,新北市新莊區新莊路100號4樓,是,新北市新莊區新莊路100號4樓,是,鄭大勇,父親,0955-456-789,設籍,特殊境遇家庭,同步租,,,2,0,32000\n林志豪,男,1991-05-20,未婚,X123456789,0978-123-456,zhihao.lin@example.com,工程師,台北市士林區士林路100號5樓,否,台北市士林區士林路100號5樓,否,林大明,父親,0912-789-123,就業,一般戶,買斷,,,1,0,45000\n王美玲,女,1979-11-12,已婚,Y123456789,0955-123-456,meiling.wang@example.com,會計師,新北市板橋區民生路100號6樓,是,新北市板橋區民生路100號6樓,是,王大明,父親,0933-123-456,設籍,一般戶,同步租,李大偉,Z234567890,3,0,50000\n陳志明,男,1960-02-28,已婚,A234567891,0910-789-456,zhiming.chen@example.com,退休公務員,台北市北投區石牌路一段100號3樓,否,台北市北投區石牌路一段100號3樓,否,陳小明,兒子,0922-456-789,設籍,65歲以上老人,同步租,林美美,B234567891,2,0,30000\n莊雅婷,女,2002-08-08,未婚,C234567891,0933-456-123,yating.zhuang@example.com,學生,桃園市桃園區中正路100號5樓,是,桃園市桃園區中正路100號5樓,是,莊大明,父親,0955-789-456,就學,原住民,買斷,,,1,0,37000",
|
|
1447
|
-
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1448
|
-
"user_id": "sebastian.hsu@gmail.com",
|
|
1449
|
-
},
|
|
1450
|
-
}
|
|
1451
|
-
|
|
1452
|
-
# Make the request
|
|
1453
|
-
endpoint = "/api/langgraph/invoke"
|
|
1454
|
-
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1455
|
-
print("-" * 50)
|
|
1456
|
-
|
|
1457
|
-
try:
|
|
1458
|
-
response = self.api_post(endpoint, payload)
|
|
1459
|
-
|
|
1460
|
-
# Basic assertions to verify the response
|
|
1461
|
-
self.assertIsNotNone(response)
|
|
1462
|
-
|
|
1463
|
-
# Validation criteria for social housing application analysis
|
|
1464
|
-
validation_criteria = """
|
|
1465
|
-
請驗證回應是否包含以下內容:
|
|
1466
|
-
1. 完整的社宅申請審查結果,包括通過/不通過標記
|
|
1467
|
-
2. 以Markdown格式顯示的審查結果表格(應包含 | 字符作為表格格式)
|
|
1468
|
-
3. 一個以"https://storage.googleapis.com"開頭的HTML頁面URL
|
|
1469
|
-
4. 基於提供的人口統計數據的分析
|
|
1470
|
-
"""
|
|
1471
|
-
|
|
1472
|
-
# Validate the response with LLM
|
|
1473
|
-
if "content" in response:
|
|
1474
|
-
validation_result = self.validate_with_llm(
|
|
1475
|
-
response["content"], validation_criteria
|
|
1476
|
-
)
|
|
1477
|
-
|
|
1478
|
-
# Additionally, directly check for markdown table and storage URL
|
|
1479
|
-
content = response["content"]
|
|
1480
|
-
has_markdown_table = "|" in content and "-|-" in content
|
|
1481
|
-
has_storage_url = "https://storage.googleapis.com" in content
|
|
1482
|
-
|
|
1483
|
-
# Custom assertions for specific requirements
|
|
1484
|
-
self.assertTrue(has_markdown_table, "回應中不包含Markdown表格")
|
|
1485
|
-
self.assertTrue(
|
|
1486
|
-
has_storage_url,
|
|
1487
|
-
"回應中不包含Google Cloud Storage網址",
|
|
1488
|
-
)
|
|
1489
|
-
|
|
1490
|
-
# Assert that validation passed
|
|
1491
|
-
self.assertTrue(
|
|
1492
|
-
validation_result.get("pass", False),
|
|
1493
|
-
f"驗證失敗:{validation_result.get('reason', '未提供原因')}",
|
|
1494
|
-
)
|
|
1495
|
-
|
|
1496
|
-
print(f"回應驗證成功:{validation_result.get('reason', '')}")
|
|
1497
|
-
else:
|
|
1498
|
-
self.fail("Response does not contain 'content' field")
|
|
1499
|
-
|
|
1500
|
-
except Exception as e:
|
|
1501
|
-
self.fail(f"Error testing API: {str(e)}")
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
def run_with_base_url(base_url=None):
|
|
1505
|
-
"""Run the tests with an optional custom base URL
|
|
1506
|
-
|
|
1507
|
-
Args:
|
|
1508
|
-
base_url: Optional base URL to override the default
|
|
1509
|
-
"""
|
|
1510
|
-
# If a base URL is provided, set it as a class attribute
|
|
1511
|
-
if base_url:
|
|
1512
|
-
TestAPIFunctionality.base_url = base_url
|
|
1513
|
-
|
|
1514
|
-
# Run the tests
|
|
1515
|
-
unittest.main(argv=["first-arg-is-ignored"], exit=False)
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
if __name__ == "__main__":
|
|
1519
|
-
unittest.main()
|
|
1520
|
-
# single test test_langgraph_multinode_news_dall_e
|
|
1521
|
-
# run_with_base_url("https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app")
|
|
1522
|
-
# test_suite = unittest.TestSuite()
|
|
1523
|
-
# test_suite.addTest(TestAPIFunctionality("test_langgraph_multinode_news_dall_e"))
|
|
1524
|
-
# runner = unittest.TextTestRunner()
|
|
1525
|
-
# runner.run(test_suite)
|
|
1
|
+
from datetime import datetime
|
|
2
|
+
import unittest
|
|
3
|
+
from langchain_anthropic import ChatAnthropic
|
|
4
|
+
import pytz
|
|
5
|
+
import requests
|
|
6
|
+
from typing import Dict, Any, List
|
|
7
|
+
from pydantic import BaseModel, Field
|
|
8
|
+
from langchain_openai import ChatOpenAI
|
|
9
|
+
from trustcall import create_extractor
|
|
10
|
+
from botrun_flow_lang.langgraph_agents.agents.util.local_files import (
|
|
11
|
+
upload_and_get_tmp_public_url,
|
|
12
|
+
)
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class ValidationResult(BaseModel):
|
|
17
|
+
"""Pydantic model for the validation result"""
|
|
18
|
+
|
|
19
|
+
pass_: bool = Field(
|
|
20
|
+
description="Whether the validation passes (true) or fails (false), If all conditions are met, return true, otherwise return false"
|
|
21
|
+
)
|
|
22
|
+
reason: str = Field(
|
|
23
|
+
description="Detailed explanation of why validation passed or failed"
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class TestAPIFunctionality(unittest.TestCase):
|
|
28
|
+
"""Test class for REST API functionality tests"""
|
|
29
|
+
|
|
30
|
+
def setUp(self):
|
|
31
|
+
"""Setup method that runs before each test"""
|
|
32
|
+
# Default base URL, can be overridden by setting the class attribute
|
|
33
|
+
if not hasattr(self, "base_url"):
|
|
34
|
+
self.base_url = "http://localhost:8080"
|
|
35
|
+
# self.base_url = (
|
|
36
|
+
# "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app"
|
|
37
|
+
# )
|
|
38
|
+
|
|
39
|
+
# Common headers
|
|
40
|
+
self.headers = {"Content-Type": "application/json"}
|
|
41
|
+
|
|
42
|
+
# Initialize LLM and extractor
|
|
43
|
+
# self.llm = ChatOpenAI(model="gpt-4o-mini", temperature=0)
|
|
44
|
+
self.llm = ChatAnthropic(model="claude-3-5-haiku-latest", temperature=0)
|
|
45
|
+
self.validator = create_extractor(
|
|
46
|
+
self.llm, tools=[ValidationResult], tool_choice="ValidationResult"
|
|
47
|
+
)
|
|
48
|
+
local_tz = pytz.timezone("Asia/Taipei")
|
|
49
|
+
self.local_time = datetime.now(local_tz)
|
|
50
|
+
|
|
51
|
+
def api_post(self, endpoint: str, data: Dict[str, Any]) -> Dict[str, Any]:
|
|
52
|
+
"""Helper method to make POST requests to the API
|
|
53
|
+
|
|
54
|
+
Args:
|
|
55
|
+
endpoint: The API endpoint path (without base URL)
|
|
56
|
+
data: The request payload as a dictionary
|
|
57
|
+
|
|
58
|
+
Returns:
|
|
59
|
+
The JSON response as a dictionary
|
|
60
|
+
"""
|
|
61
|
+
url = f"{self.base_url}{endpoint}"
|
|
62
|
+
response = requests.post(url, headers=self.headers, json=data)
|
|
63
|
+
|
|
64
|
+
# Raise an exception if the response was unsuccessful
|
|
65
|
+
response.raise_for_status()
|
|
66
|
+
|
|
67
|
+
return response.json()
|
|
68
|
+
|
|
69
|
+
def validate_with_llm(
|
|
70
|
+
self, response_content: str, validation_criteria: str
|
|
71
|
+
) -> Dict[str, Any]:
|
|
72
|
+
"""Use trustcall with GPT-4o-mini to validate the response content
|
|
73
|
+
|
|
74
|
+
Args:
|
|
75
|
+
response_content: The content to validate
|
|
76
|
+
validation_criteria: Validation criteria description
|
|
77
|
+
|
|
78
|
+
Returns:
|
|
79
|
+
Dictionary with 'pass' (boolean) and 'reason' (string)
|
|
80
|
+
"""
|
|
81
|
+
prompt = f"""
|
|
82
|
+
你是一個專業的API回應驗證員。請評估以下API回應是否符合所有指定條件。
|
|
83
|
+
|
|
84
|
+
=== 驗證條件 ===
|
|
85
|
+
{validation_criteria}
|
|
86
|
+
|
|
87
|
+
=== API回應內容 ===
|
|
88
|
+
{response_content}
|
|
89
|
+
|
|
90
|
+
請評估API回應是否符合所有驗證條件。詳細說明評估原因,若不符合條件,請明確指出哪些條件未達成。
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
try:
|
|
94
|
+
# Use trustcall extractor to validate
|
|
95
|
+
result = self.validator.invoke(
|
|
96
|
+
{"messages": [{"role": "user", "content": prompt}]}
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
# Extract the validated response
|
|
100
|
+
validation_result = result["responses"][0]
|
|
101
|
+
|
|
102
|
+
# Convert to the expected format
|
|
103
|
+
return {"pass": validation_result.pass_, "reason": validation_result.reason}
|
|
104
|
+
|
|
105
|
+
except Exception as e:
|
|
106
|
+
return {"pass": False, "reason": f"Error during validation: {str(e)}"}
|
|
107
|
+
|
|
108
|
+
def test_langgraph_news_joke_emoji(self):
|
|
109
|
+
"""測試是否會抓到今天的新聞,檢查重點:
|
|
110
|
+
1. 是否會抓到今天的新聞
|
|
111
|
+
2. 是否會列出來源網址
|
|
112
|
+
3. 是否會講個笑話,並加上 emoji
|
|
113
|
+
"""
|
|
114
|
+
# Test payload
|
|
115
|
+
payload = {
|
|
116
|
+
"graph_name": "langgraph_react_agent",
|
|
117
|
+
"messages": [
|
|
118
|
+
{
|
|
119
|
+
"role": "user",
|
|
120
|
+
"content": "幫我搜尋今天的新聞是什麼?一一列出來,並給我參考來源網址。",
|
|
121
|
+
}
|
|
122
|
+
],
|
|
123
|
+
"config": {
|
|
124
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
125
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
126
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
127
|
+
},
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
# Make the request
|
|
131
|
+
endpoint = "/api/langgraph/invoke"
|
|
132
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
133
|
+
print("-" * 50)
|
|
134
|
+
|
|
135
|
+
try:
|
|
136
|
+
response = self.api_post(endpoint, payload)
|
|
137
|
+
|
|
138
|
+
# Basic assertions to verify the response
|
|
139
|
+
self.assertIsNotNone(response)
|
|
140
|
+
|
|
141
|
+
# Extract the content field from the response
|
|
142
|
+
if "content" in response:
|
|
143
|
+
response_content = response["content"]
|
|
144
|
+
else:
|
|
145
|
+
self.fail("Response does not contain 'content' field")
|
|
146
|
+
|
|
147
|
+
validation_criteria = f"""
|
|
148
|
+
1. 是否包含今天 {self.local_time.strftime("%Y-%m-%d")} 日期的新聞資訊,沒有列出日期測試算失敗
|
|
149
|
+
2. 是否列出每則新聞的來源網址
|
|
150
|
+
3. 是否在回答結尾包含一個笑話
|
|
151
|
+
"""
|
|
152
|
+
|
|
153
|
+
# Validate with LLM
|
|
154
|
+
validation_result = self.validate_with_llm(
|
|
155
|
+
response_content, validation_criteria
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
# Assert that the validation passed
|
|
159
|
+
self.assertTrue(
|
|
160
|
+
validation_result["pass"],
|
|
161
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
except Exception as e:
|
|
165
|
+
print(f"test_langgraph_news_joke_emoji: Test failed with error: {str(e)}")
|
|
166
|
+
raise
|
|
167
|
+
|
|
168
|
+
def test_langgraph_multinode_news_dall_e(self):
|
|
169
|
+
"""測試多節點處理流程,檢查重點:
|
|
170
|
+
1. 是否會抓到今天的新聞
|
|
171
|
+
2. 是否有評分新聞 (1-10分)
|
|
172
|
+
3. 是否有產出一張圖片,並帶有 URL
|
|
173
|
+
"""
|
|
174
|
+
# Test payload
|
|
175
|
+
payload = {
|
|
176
|
+
"graph_name": "langgraph_react_agent",
|
|
177
|
+
"messages": [
|
|
178
|
+
{
|
|
179
|
+
"role": "user",
|
|
180
|
+
"content": "好。 我們現在就是跟那個Bert人講多個節點,那它裡面它就會用多個節點的方式直接去工作,比如說第一個節點就是請Bert人上網去搜尋今天的新聞。 然後第二個節點呢,請你把這個新聞,打分,一分到十分,哪個新聞最可愛。 然後第三個節點呢,請你根據分數最高的那一個新聞,你幫我呼叫達利畫一張跟那個新聞相關的圖片,那我們就用這個來示範一下,來。",
|
|
181
|
+
}
|
|
182
|
+
],
|
|
183
|
+
"config": {
|
|
184
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
185
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
186
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
187
|
+
},
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
# Make the request
|
|
191
|
+
endpoint = "/api/langgraph/invoke"
|
|
192
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
193
|
+
print("-" * 50)
|
|
194
|
+
|
|
195
|
+
try:
|
|
196
|
+
response = self.api_post(endpoint, payload)
|
|
197
|
+
|
|
198
|
+
# Basic assertions to verify the response
|
|
199
|
+
self.assertIsNotNone(response)
|
|
200
|
+
|
|
201
|
+
# Extract the content field from the response
|
|
202
|
+
if "content" in response:
|
|
203
|
+
response_content = response["content"]
|
|
204
|
+
else:
|
|
205
|
+
self.fail("Response does not contain 'content' field")
|
|
206
|
+
|
|
207
|
+
# Define validation criteria based on the test requirements
|
|
208
|
+
validation_criteria = f"""
|
|
209
|
+
1. 是否會抓到今天 {self.local_time.strftime("%Y-%m-%d")} 日期的新聞
|
|
210
|
+
2. 是否有評分新聞 (1-10分)
|
|
211
|
+
3. 是否有產出一張圖片,並帶有 URL
|
|
212
|
+
4. 是否在回答結尾包含一個笑話
|
|
213
|
+
"""
|
|
214
|
+
|
|
215
|
+
# Validate with LLM
|
|
216
|
+
validation_result = self.validate_with_llm(
|
|
217
|
+
response_content, validation_criteria
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
# Assert that the validation passed
|
|
221
|
+
self.assertTrue(
|
|
222
|
+
validation_result["pass"],
|
|
223
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
except Exception as e:
|
|
227
|
+
print(
|
|
228
|
+
f"test_langgraph_multinode_news_dall_e: Test failed with error: {str(e)}"
|
|
229
|
+
)
|
|
230
|
+
raise
|
|
231
|
+
|
|
232
|
+
def test_langgraph_future_date_news(self):
|
|
233
|
+
"""測試未來日期的新聞搜尋,檢查重點:
|
|
234
|
+
1. 是否是指定時間的 2025/2/10 的新聞,回覆內容要有這個時間
|
|
235
|
+
2. 不能回應說這個時間在未來,所以無法回答,可以說 "截至2025年2月10日,相關的新聞如下:"。
|
|
236
|
+
"""
|
|
237
|
+
# Test payload
|
|
238
|
+
payload = {
|
|
239
|
+
"graph_name": "langgraph_react_agent",
|
|
240
|
+
"messages": [
|
|
241
|
+
{
|
|
242
|
+
"role": "user",
|
|
243
|
+
"content": "請你幫我找2025/2/10全球災難新聞",
|
|
244
|
+
}
|
|
245
|
+
],
|
|
246
|
+
"config": {
|
|
247
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
248
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
249
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
250
|
+
},
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
# Make the request
|
|
254
|
+
endpoint = "/api/langgraph/invoke"
|
|
255
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
256
|
+
print("-" * 50)
|
|
257
|
+
|
|
258
|
+
try:
|
|
259
|
+
response = self.api_post(endpoint, payload)
|
|
260
|
+
|
|
261
|
+
# Basic assertions to verify the response
|
|
262
|
+
self.assertIsNotNone(response)
|
|
263
|
+
|
|
264
|
+
# Extract the content field from the response
|
|
265
|
+
if "content" in response:
|
|
266
|
+
response_content = response["content"]
|
|
267
|
+
else:
|
|
268
|
+
self.fail("Response does not contain 'content' field")
|
|
269
|
+
|
|
270
|
+
# Define validation criteria based on the test requirements
|
|
271
|
+
validation_criteria = """
|
|
272
|
+
1. 是否包含指定時間 2025/2/10 的新聞資訊,回覆內容中必須有出現「2025/2/10」或類似的日期格式
|
|
273
|
+
2. 不能包含任何提到該日期在未來、無法預測未來、尚未發生等類似的說明
|
|
274
|
+
3. 是否在回答結尾包含一個笑話
|
|
275
|
+
"""
|
|
276
|
+
|
|
277
|
+
# Validate with LLM
|
|
278
|
+
validation_result = self.validate_with_llm(
|
|
279
|
+
response_content, validation_criteria
|
|
280
|
+
)
|
|
281
|
+
|
|
282
|
+
# Assert that the validation passed
|
|
283
|
+
self.assertTrue(
|
|
284
|
+
validation_result["pass"],
|
|
285
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
except Exception as e:
|
|
289
|
+
print(f"test_langgraph_future_date_news: Test failed with error: {str(e)}")
|
|
290
|
+
raise
|
|
291
|
+
|
|
292
|
+
def test_langgraph_pdf_analysis(self):
|
|
293
|
+
"""測試PDF分析功能,檢查重點:
|
|
294
|
+
1. 是否能正確解析PDF檔案中的「表 4.3-1 環境敏感地區調查表-第一級環境敏感地區」
|
|
295
|
+
2. 是否能列出所有項目的「查詢結果及限制內容」(是或否)
|
|
296
|
+
3. 回傳結果是否符合預期的敏感區域結果
|
|
297
|
+
"""
|
|
298
|
+
# 使用pathlib構建正確的檔案路徑
|
|
299
|
+
|
|
300
|
+
current_dir = Path(__file__).parent
|
|
301
|
+
pdf_path = (
|
|
302
|
+
current_dir
|
|
303
|
+
/ "test_files"
|
|
304
|
+
/ "1120701A海廣離岸風力發電計畫環境影響說明書-C04.PDF"
|
|
305
|
+
)
|
|
306
|
+
|
|
307
|
+
# 確保檔案存在
|
|
308
|
+
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
309
|
+
# 將絕對路徑轉為字串
|
|
310
|
+
pdf_path_str = str(pdf_path)
|
|
311
|
+
# 上傳檔案到 tmp_public_url
|
|
312
|
+
tmp_public_url = upload_and_get_tmp_public_url(
|
|
313
|
+
pdf_path_str,
|
|
314
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
315
|
+
"sebastian.hsu@gmail.com",
|
|
316
|
+
)
|
|
317
|
+
|
|
318
|
+
# Test payload
|
|
319
|
+
payload = {
|
|
320
|
+
"graph_name": "langgraph_react_agent",
|
|
321
|
+
"messages": [
|
|
322
|
+
{
|
|
323
|
+
"role": "user",
|
|
324
|
+
"content": f"幫我分析 {tmp_public_url} 這個檔案,請你幫我找出在報告書中的「表 4.3-1 環境敏感地區調查表-第一級環境敏感地區」表格中的所有項目的「查詢結果及限制內容」幫我列出是或否?請全部列出來,不要遺漏",
|
|
325
|
+
}
|
|
326
|
+
],
|
|
327
|
+
"config": {
|
|
328
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
329
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
330
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
331
|
+
},
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
# Make the request
|
|
335
|
+
endpoint = "/api/langgraph/invoke"
|
|
336
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
337
|
+
print("-" * 50)
|
|
338
|
+
|
|
339
|
+
try:
|
|
340
|
+
response = self.api_post(endpoint, payload)
|
|
341
|
+
|
|
342
|
+
# Basic assertions to verify the response
|
|
343
|
+
self.assertIsNotNone(response)
|
|
344
|
+
|
|
345
|
+
# Extract the content field from the response
|
|
346
|
+
if "content" in response:
|
|
347
|
+
response_content = response["content"]
|
|
348
|
+
else:
|
|
349
|
+
self.fail("Response does not contain 'content' field")
|
|
350
|
+
|
|
351
|
+
# Define validation criteria based on existing test_pdf_analyzer.py
|
|
352
|
+
validation_criteria = """
|
|
353
|
+
請確認回應是否包含以下項目的查詢結果(是或否),所有項目都必須存在:
|
|
354
|
+
1. 活動斷層兩側一定範圍: 否
|
|
355
|
+
2. 特定水土保持區: 否
|
|
356
|
+
3. 河川區域: 否
|
|
357
|
+
4. 洪氾區一級管制區及洪水平原一級管制區: 否
|
|
358
|
+
5. 區域排水設施範圍: 是
|
|
359
|
+
6. 國家公園區內之特別景觀區、生態保護區: 否
|
|
360
|
+
7. 自然保留區: 否
|
|
361
|
+
8. 野生動物保護區: 否
|
|
362
|
+
9. 野生動物重要棲息環境: 是
|
|
363
|
+
10. 自然保護區: 否
|
|
364
|
+
11. 一級海岸保護區: 是
|
|
365
|
+
12. 國際級重要濕地、國家級重要濕地之核心保育區及生態復育區: 否
|
|
366
|
+
13. 古蹟保存區: 否
|
|
367
|
+
14. 考古遺址: 否
|
|
368
|
+
15. 重要聚落建築群: 否
|
|
369
|
+
|
|
370
|
+
所有項目都必須正確列出,且其中:
|
|
371
|
+
- 區域排水設施範圍應為「是」
|
|
372
|
+
- 野生動物重要棲息環境應為「是」
|
|
373
|
+
- 一級海岸保護區應為「是」
|
|
374
|
+
|
|
375
|
+
如果有遺漏任何一項或者結果不符合預期,則視為測試失敗。
|
|
376
|
+
如果結果有超過,沒有關係。
|
|
377
|
+
"""
|
|
378
|
+
|
|
379
|
+
# Validate with LLM
|
|
380
|
+
validation_result = self.validate_with_llm(
|
|
381
|
+
response_content, validation_criteria
|
|
382
|
+
)
|
|
383
|
+
|
|
384
|
+
# Assert that the validation passed
|
|
385
|
+
self.assertTrue(
|
|
386
|
+
validation_result["pass"],
|
|
387
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
388
|
+
)
|
|
389
|
+
|
|
390
|
+
except Exception as e:
|
|
391
|
+
print(f"test_langgraph_pdf_analysis: Test failed with error: {str(e)}")
|
|
392
|
+
raise
|
|
393
|
+
|
|
394
|
+
def test_langgraph_pdf_attendance_analysis(self):
|
|
395
|
+
"""測試PDF分析功能,檢查重點:
|
|
396
|
+
1. 是否能正確解析PDF檔案中的「目錄4」的出席名單
|
|
397
|
+
2. 回答中是否有包含「德懷師父」、「德宸師父」、「德倫師父」
|
|
398
|
+
"""
|
|
399
|
+
|
|
400
|
+
current_dir = Path(__file__).parent
|
|
401
|
+
pdf_path = (
|
|
402
|
+
current_dir
|
|
403
|
+
/ "test_files"
|
|
404
|
+
/ "(溫馨成果 行政請示匯總)20250210向 上人報告簡報 (1).pdf"
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
# 確保檔案存在
|
|
408
|
+
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
409
|
+
|
|
410
|
+
# 將絕對路徑轉為字串
|
|
411
|
+
pdf_path_str = str(pdf_path)
|
|
412
|
+
|
|
413
|
+
# 上傳檔案到 tmp_public_url
|
|
414
|
+
tmp_public_url = upload_and_get_tmp_public_url(
|
|
415
|
+
pdf_path_str,
|
|
416
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
417
|
+
"sebastian.hsu@gmail.com",
|
|
418
|
+
)
|
|
419
|
+
|
|
420
|
+
# Test payload
|
|
421
|
+
payload = {
|
|
422
|
+
"graph_name": "langgraph_react_agent",
|
|
423
|
+
"messages": [
|
|
424
|
+
{
|
|
425
|
+
"role": "user",
|
|
426
|
+
"content": f"幫我分析 {tmp_public_url} 這個檔案,你幫我看「目錄4」,告訴我有哪些師父和講者、執辦、主管有出席",
|
|
427
|
+
}
|
|
428
|
+
],
|
|
429
|
+
"config": {
|
|
430
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
431
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
432
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
433
|
+
},
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
# Make the request
|
|
437
|
+
endpoint = "/api/langgraph/invoke"
|
|
438
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
439
|
+
print("-" * 50)
|
|
440
|
+
|
|
441
|
+
try:
|
|
442
|
+
response = self.api_post(endpoint, payload)
|
|
443
|
+
|
|
444
|
+
# Basic assertions to verify the response
|
|
445
|
+
self.assertIsNotNone(response)
|
|
446
|
+
|
|
447
|
+
# Extract the content field from the response
|
|
448
|
+
if "content" in response:
|
|
449
|
+
response_content = response["content"]
|
|
450
|
+
else:
|
|
451
|
+
self.fail("Response does not contain 'content' field")
|
|
452
|
+
|
|
453
|
+
# Define validation criteria
|
|
454
|
+
validation_criteria = """
|
|
455
|
+
請確認回應是否包含以下師父的名字,所有名字都必須存在:
|
|
456
|
+
1. 德懷師父
|
|
457
|
+
2. 德宸師父
|
|
458
|
+
3. 德倫師父
|
|
459
|
+
|
|
460
|
+
此外,回應應該提供「目錄4」中出席的師父、講者、執辦和主管的完整列表。
|
|
461
|
+
如果缺少上述任一師父的名字,則視為測試失敗。
|
|
462
|
+
"""
|
|
463
|
+
|
|
464
|
+
# Validate with LLM
|
|
465
|
+
validation_result = self.validate_with_llm(
|
|
466
|
+
response_content, validation_criteria
|
|
467
|
+
)
|
|
468
|
+
|
|
469
|
+
# Assert that the validation passed
|
|
470
|
+
self.assertTrue(
|
|
471
|
+
validation_result["pass"],
|
|
472
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
except Exception as e:
|
|
476
|
+
print(
|
|
477
|
+
f"test_langgraph_pdf_attendance_analysis: Test failed with error: {str(e)}"
|
|
478
|
+
)
|
|
479
|
+
raise
|
|
480
|
+
|
|
481
|
+
def test_langgraph_image_analysis_generation(self):
|
|
482
|
+
"""測試圖片分析與生成功能,檢查重點:
|
|
483
|
+
1. 是否能正確分析圖片並識別出「佛教」相關元素
|
|
484
|
+
2. 是否產生一張相同意境的圖片並提供URL
|
|
485
|
+
"""
|
|
486
|
+
|
|
487
|
+
current_dir = Path(__file__).parent
|
|
488
|
+
image_path = current_dir / "test_files" / "d5712343.jpg"
|
|
489
|
+
|
|
490
|
+
# 確保檔案存在
|
|
491
|
+
self.assertTrue(image_path.exists(), f"Test file not found at {image_path}")
|
|
492
|
+
|
|
493
|
+
# 將絕對路徑轉為字串
|
|
494
|
+
image_path_str = str(image_path)
|
|
495
|
+
|
|
496
|
+
# 上傳檔案到 tmp_public_url
|
|
497
|
+
tmp_public_url = upload_and_get_tmp_public_url(
|
|
498
|
+
image_path_str,
|
|
499
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
500
|
+
"sebastian.hsu@gmail.com",
|
|
501
|
+
)
|
|
502
|
+
|
|
503
|
+
# Test payload
|
|
504
|
+
payload = {
|
|
505
|
+
"graph_name": "langgraph_react_agent",
|
|
506
|
+
"messages": [
|
|
507
|
+
{
|
|
508
|
+
"role": "user",
|
|
509
|
+
"content": f"{tmp_public_url} 幫我分析這張圖裡的元素,然後幫我創作一張相同意境的圖片",
|
|
510
|
+
}
|
|
511
|
+
],
|
|
512
|
+
"config": {
|
|
513
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
514
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
515
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
516
|
+
},
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
# Make the request
|
|
520
|
+
endpoint = "/api/langgraph/invoke"
|
|
521
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
522
|
+
print("-" * 50)
|
|
523
|
+
|
|
524
|
+
try:
|
|
525
|
+
response = self.api_post(endpoint, payload)
|
|
526
|
+
|
|
527
|
+
# Basic assertions to verify the response
|
|
528
|
+
self.assertIsNotNone(response)
|
|
529
|
+
|
|
530
|
+
# Extract the content field from the response
|
|
531
|
+
if "content" in response:
|
|
532
|
+
response_content = response["content"]
|
|
533
|
+
else:
|
|
534
|
+
self.fail("Response does not contain 'content' field")
|
|
535
|
+
|
|
536
|
+
# Define validation criteria
|
|
537
|
+
validation_criteria = """
|
|
538
|
+
請確認回應是否符合以下條件:
|
|
539
|
+
1. 分析結果中有提到「佛教」相關的元素(如佛像、和尚、寺廟、佛教符號等)
|
|
540
|
+
2. 回應中包含一個圖片的URL(通常是以http或https開頭的網址,並包含在圖片的描述旁)
|
|
541
|
+
3. 是否在回答結尾包含一個笑話
|
|
542
|
+
|
|
543
|
+
所有條件都必須滿足,尤其是必須確認分析中有提到佛教元素,並且有生成一張新的圖片和提供其URL。
|
|
544
|
+
"""
|
|
545
|
+
|
|
546
|
+
# Validate with LLM
|
|
547
|
+
validation_result = self.validate_with_llm(
|
|
548
|
+
response_content, validation_criteria
|
|
549
|
+
)
|
|
550
|
+
|
|
551
|
+
# Assert that the validation passed
|
|
552
|
+
self.assertTrue(
|
|
553
|
+
validation_result["pass"],
|
|
554
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
555
|
+
)
|
|
556
|
+
|
|
557
|
+
except Exception as e:
|
|
558
|
+
print(
|
|
559
|
+
f"test_langgraph_image_analysis_generation: Test failed with error: {str(e)}"
|
|
560
|
+
)
|
|
561
|
+
raise
|
|
562
|
+
|
|
563
|
+
def test_langgraph_spot_difference(self):
|
|
564
|
+
"""測試圖片比對功能,檢查重點:
|
|
565
|
+
1. 是否能正確分析兩張找不同遊戲的圖片
|
|
566
|
+
2. 是否能找出並明確描述出兩張圖片的不同之處
|
|
567
|
+
"""
|
|
568
|
+
|
|
569
|
+
current_dir = Path(__file__).parent
|
|
570
|
+
image1_path = current_dir / "test_files" / "spot_difference_1.png"
|
|
571
|
+
image2_path = current_dir / "test_files" / "spot_difference_2.png"
|
|
572
|
+
|
|
573
|
+
# 確保檔案存在
|
|
574
|
+
self.assertTrue(image1_path.exists(), f"Test file not found at {image1_path}")
|
|
575
|
+
self.assertTrue(image2_path.exists(), f"Test file not found at {image2_path}")
|
|
576
|
+
|
|
577
|
+
# 將絕對路徑轉為字串
|
|
578
|
+
image1_path_str = str(image1_path)
|
|
579
|
+
image2_path_str = str(image2_path)
|
|
580
|
+
|
|
581
|
+
# 上傳檔案到 tmp_public_url
|
|
582
|
+
tmp_public_url_1 = upload_and_get_tmp_public_url(
|
|
583
|
+
image1_path_str,
|
|
584
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
585
|
+
"sebastian.hsu@gmail.com",
|
|
586
|
+
)
|
|
587
|
+
tmp_public_url_2 = upload_and_get_tmp_public_url(
|
|
588
|
+
image2_path_str,
|
|
589
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
590
|
+
"sebastian.hsu@gmail.com",
|
|
591
|
+
)
|
|
592
|
+
|
|
593
|
+
# Test payload
|
|
594
|
+
payload = {
|
|
595
|
+
"graph_name": "langgraph_react_agent",
|
|
596
|
+
"messages": [
|
|
597
|
+
{
|
|
598
|
+
"role": "user",
|
|
599
|
+
"content": f"這是一個找不同的遊戲,幫我分析兩張圖有幾處不同? {tmp_public_url_1},{tmp_public_url_2}",
|
|
600
|
+
}
|
|
601
|
+
],
|
|
602
|
+
"config": {
|
|
603
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
604
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
605
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
606
|
+
},
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
# Make the request
|
|
610
|
+
endpoint = "/api/langgraph/invoke"
|
|
611
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
612
|
+
print("-" * 50)
|
|
613
|
+
|
|
614
|
+
try:
|
|
615
|
+
response = self.api_post(endpoint, payload)
|
|
616
|
+
|
|
617
|
+
# Basic assertions to verify the response
|
|
618
|
+
self.assertIsNotNone(response)
|
|
619
|
+
|
|
620
|
+
# Extract the content field from the response
|
|
621
|
+
if "content" in response:
|
|
622
|
+
response_content = response["content"]
|
|
623
|
+
else:
|
|
624
|
+
self.fail("Response does not contain 'content' field")
|
|
625
|
+
|
|
626
|
+
# Define validation criteria
|
|
627
|
+
validation_criteria = """
|
|
628
|
+
請確認回應是否符合以下條件:
|
|
629
|
+
1. 回應中有具體指出並描述兩張圖片之間的不同之處
|
|
630
|
+
2. 必須明確描述出不同的位置、形狀、顏色或其他特徵差異
|
|
631
|
+
3. 不能只回應「無法處理」、「無法比較」或類似的無能力陳述
|
|
632
|
+
4. 是否在回答結尾包含一個笑話
|
|
633
|
+
|
|
634
|
+
關鍵是要確保系統能夠實際找出差異並清楚描述,而不是迴避任務或宣稱無法完成。
|
|
635
|
+
如果回應只是說明系統不支援圖片比較功能,則測試視為失敗。
|
|
636
|
+
"""
|
|
637
|
+
|
|
638
|
+
# Validate with LLM
|
|
639
|
+
validation_result = self.validate_with_llm(
|
|
640
|
+
response_content, validation_criteria
|
|
641
|
+
)
|
|
642
|
+
|
|
643
|
+
# Assert that the validation passed
|
|
644
|
+
self.assertTrue(
|
|
645
|
+
validation_result["pass"],
|
|
646
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
647
|
+
)
|
|
648
|
+
|
|
649
|
+
except Exception as e:
|
|
650
|
+
print(f"test_langgraph_spot_difference: Test failed with error: {str(e)}")
|
|
651
|
+
raise
|
|
652
|
+
|
|
653
|
+
def test_langgraph_platform_images_analysis(self):
|
|
654
|
+
"""測試分析多張車站月台圖片功能,檢查重點:
|
|
655
|
+
1. 是否能正確分析多張車站月台圖片
|
|
656
|
+
2. 是否能根據圖片提供清晰的月台指引和理由
|
|
657
|
+
3. 回覆中是否包含「月台」相關的特定字眼
|
|
658
|
+
"""
|
|
659
|
+
|
|
660
|
+
current_dir = Path(__file__).parent
|
|
661
|
+
image1_path = current_dir / "test_files" / "ImportedPhoto.760363950.029251.jpeg"
|
|
662
|
+
image2_path = current_dir / "test_files" / "ImportedPhoto.760363950.031127.jpeg"
|
|
663
|
+
image3_path = current_dir / "test_files" / "ImportedPhoto.760363950.030446.jpeg"
|
|
664
|
+
|
|
665
|
+
# 確保檔案存在
|
|
666
|
+
self.assertTrue(image1_path.exists(), f"Test file not found at {image1_path}")
|
|
667
|
+
self.assertTrue(image2_path.exists(), f"Test file not found at {image2_path}")
|
|
668
|
+
self.assertTrue(image3_path.exists(), f"Test file not found at {image3_path}")
|
|
669
|
+
|
|
670
|
+
# 將絕對路徑轉為字串
|
|
671
|
+
image1_path_str = str(image1_path)
|
|
672
|
+
image2_path_str = str(image2_path)
|
|
673
|
+
image3_path_str = str(image3_path)
|
|
674
|
+
|
|
675
|
+
# 上傳檔案到 tmp_public_url
|
|
676
|
+
tmp_public_url_1 = upload_and_get_tmp_public_url(
|
|
677
|
+
image1_path_str,
|
|
678
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
679
|
+
"sebastian.hsu@gmail.com",
|
|
680
|
+
)
|
|
681
|
+
tmp_public_url_2 = upload_and_get_tmp_public_url(
|
|
682
|
+
image2_path_str,
|
|
683
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
684
|
+
"sebastian.hsu@gmail.com",
|
|
685
|
+
)
|
|
686
|
+
tmp_public_url_3 = upload_and_get_tmp_public_url(
|
|
687
|
+
image3_path_str,
|
|
688
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
689
|
+
"sebastian.hsu@gmail.com",
|
|
690
|
+
)
|
|
691
|
+
|
|
692
|
+
# Test payload
|
|
693
|
+
payload = {
|
|
694
|
+
"graph_name": "langgraph_react_agent",
|
|
695
|
+
"messages": [
|
|
696
|
+
{
|
|
697
|
+
"role": "system",
|
|
698
|
+
"content": "妳是臺灣人,回答要用臺灣繁體中文正式用語,需要的時候也可以用英文,可以親切、俏皮、幽默,但不能隨便輕浮。在使用者合理的要求下請盡量配合他的需求,不要隨便拒絕。",
|
|
699
|
+
},
|
|
700
|
+
{
|
|
701
|
+
"role": "user",
|
|
702
|
+
"content": [
|
|
703
|
+
{
|
|
704
|
+
"type": "text",
|
|
705
|
+
"text": f"以下為使用者上傳的圖片,請參考這些圖片回答使用者的問題:\n\n{tmp_public_url_1}\n{tmp_public_url_2}\n{tmp_public_url_3}\n\n使用者問題:\n\n我要去哪個月台,為什麼?",
|
|
706
|
+
}
|
|
707
|
+
],
|
|
708
|
+
},
|
|
709
|
+
],
|
|
710
|
+
"config": {
|
|
711
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
712
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
713
|
+
},
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
# Make the request
|
|
717
|
+
endpoint = "/api/langgraph/invoke"
|
|
718
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
719
|
+
print("-" * 50)
|
|
720
|
+
|
|
721
|
+
try:
|
|
722
|
+
response = self.api_post(endpoint, payload)
|
|
723
|
+
|
|
724
|
+
# Basic assertions to verify the response
|
|
725
|
+
self.assertIsNotNone(response)
|
|
726
|
+
|
|
727
|
+
# Extract the content field from the response
|
|
728
|
+
if "content" in response:
|
|
729
|
+
response_content = response["content"]
|
|
730
|
+
else:
|
|
731
|
+
self.fail("Response does not contain 'content' field")
|
|
732
|
+
|
|
733
|
+
# Define validation criteria
|
|
734
|
+
validation_criteria = """
|
|
735
|
+
請確認回應是否符合以下條件:
|
|
736
|
+
1. 回應中有明確提及「月台」、「站台」或「platform」等相關詞彙
|
|
737
|
+
2. 回應中有具體指出一個明確的月台方向或號碼,尤其應該包含以下月台號碼其中之一:
|
|
738
|
+
- 5 A-C
|
|
739
|
+
- 5 D-F
|
|
740
|
+
- 5-A-C
|
|
741
|
+
- 5-A/C
|
|
742
|
+
- 5-D-F
|
|
743
|
+
- 5-D/F
|
|
744
|
+
3. 回應中提供了選擇該月台的理由或依據(例如目的地、車次、方向等)
|
|
745
|
+
4. 回應使用了臺灣繁體中文正式用語
|
|
746
|
+
|
|
747
|
+
回應必須能清楚指引使用者應該前往哪個月台,以及為什麼要去那個月台。如果回應中缺少明確的月台指引或理由,則視為測試失敗。
|
|
748
|
+
"""
|
|
749
|
+
|
|
750
|
+
# Validate with LLM
|
|
751
|
+
validation_result = self.validate_with_llm(
|
|
752
|
+
response_content, validation_criteria
|
|
753
|
+
)
|
|
754
|
+
|
|
755
|
+
# Assert that the validation passed
|
|
756
|
+
self.assertTrue(
|
|
757
|
+
validation_result["pass"],
|
|
758
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
759
|
+
)
|
|
760
|
+
|
|
761
|
+
# Additional check for specific platform numbers
|
|
762
|
+
possible_platforms = ["5 A-C", "5 D-F", "5-A-C", "5-A/C", "5-D-F", "5-D/F"]
|
|
763
|
+
platform_found = False
|
|
764
|
+
|
|
765
|
+
for platform in possible_platforms:
|
|
766
|
+
if platform in response_content:
|
|
767
|
+
platform_found = True
|
|
768
|
+
print(f"Found expected platform: {platform}")
|
|
769
|
+
break
|
|
770
|
+
|
|
771
|
+
self.assertTrue(
|
|
772
|
+
platform_found,
|
|
773
|
+
f"Response does not contain any of the expected platform numbers: {possible_platforms}, but get {response_content}",
|
|
774
|
+
)
|
|
775
|
+
|
|
776
|
+
except Exception as e:
|
|
777
|
+
print(
|
|
778
|
+
f"test_langgraph_platform_images_analysis: Test failed with error: {str(e)}"
|
|
779
|
+
)
|
|
780
|
+
raise
|
|
781
|
+
|
|
782
|
+
def test_langgraph_population_analysis(self):
|
|
783
|
+
"""測試PDF人口分析與圖表生成功能,檢查重點:
|
|
784
|
+
1. 是否能正確分析PDF中各縣市的人口數據
|
|
785
|
+
2. 是否生成相關的比較圖表並提供Google Storage URL
|
|
786
|
+
"""
|
|
787
|
+
|
|
788
|
+
current_dir = Path(__file__).parent
|
|
789
|
+
pdf_path = (
|
|
790
|
+
current_dir
|
|
791
|
+
/ "test_files"
|
|
792
|
+
/ "11206_10808人口數(3段年齡組+比率)天下雜誌1.pdf"
|
|
793
|
+
)
|
|
794
|
+
|
|
795
|
+
# 確保檔案存在
|
|
796
|
+
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
797
|
+
|
|
798
|
+
# 將絕對路徑轉為字串
|
|
799
|
+
pdf_path_str = str(pdf_path)
|
|
800
|
+
tmp_public_url = upload_and_get_tmp_public_url(
|
|
801
|
+
pdf_path_str,
|
|
802
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
803
|
+
"sebastian.hsu@gmail.com",
|
|
804
|
+
)
|
|
805
|
+
|
|
806
|
+
# Test payload
|
|
807
|
+
payload = {
|
|
808
|
+
"graph_name": "langgraph_react_agent",
|
|
809
|
+
"messages": [
|
|
810
|
+
{
|
|
811
|
+
"role": "user",
|
|
812
|
+
"content": f"{tmp_public_url} 幫我分析這個檔案,做深度的人口狀況分析,然後產出一個相關的比較圖表給我看。",
|
|
813
|
+
}
|
|
814
|
+
],
|
|
815
|
+
"config": {
|
|
816
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
817
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
818
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
819
|
+
},
|
|
820
|
+
}
|
|
821
|
+
|
|
822
|
+
# Make the request
|
|
823
|
+
endpoint = "/api/langgraph/invoke"
|
|
824
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
825
|
+
print("-" * 50)
|
|
826
|
+
|
|
827
|
+
try:
|
|
828
|
+
response = self.api_post(endpoint, payload)
|
|
829
|
+
|
|
830
|
+
# Basic assertions to verify the response
|
|
831
|
+
self.assertIsNotNone(response)
|
|
832
|
+
|
|
833
|
+
# Extract the content field from the response
|
|
834
|
+
if "content" in response:
|
|
835
|
+
response_content = response["content"]
|
|
836
|
+
else:
|
|
837
|
+
self.fail("Response does not contain 'content' field")
|
|
838
|
+
|
|
839
|
+
# Define validation criteria
|
|
840
|
+
validation_criteria = """
|
|
841
|
+
請確認回應是否符合以下條件:
|
|
842
|
+
1. 回應中有包含各縣市的人口數據分析,至少提及三個以上的縣市名稱及其人口狀況
|
|
843
|
+
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的圖表
|
|
844
|
+
3. 分析內容應該涵蓋人口結構的深度分析,例如年齡分布、老化指數、人口增減等
|
|
845
|
+
4. 是否在回答結尾包含一個笑話
|
|
846
|
+
|
|
847
|
+
所有條件都必須滿足,特別是必須有各縣市的人口分析並含有Google Storage的圖表URL。
|
|
848
|
+
"""
|
|
849
|
+
|
|
850
|
+
# Validate with LLM
|
|
851
|
+
validation_result = self.validate_with_llm(
|
|
852
|
+
response_content, validation_criteria
|
|
853
|
+
)
|
|
854
|
+
|
|
855
|
+
# Assert that the validation passed
|
|
856
|
+
self.assertTrue(
|
|
857
|
+
validation_result["pass"],
|
|
858
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
859
|
+
)
|
|
860
|
+
|
|
861
|
+
# Additional check for Google Storage URL
|
|
862
|
+
self.assertTrue(
|
|
863
|
+
"https://storage.googleapis.com" in response_content,
|
|
864
|
+
"Response does not contain a Google Storage URL",
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
except Exception as e:
|
|
868
|
+
print(
|
|
869
|
+
f"test_langgraph_population_analysis: Test failed with error: {str(e)}"
|
|
870
|
+
)
|
|
871
|
+
raise
|
|
872
|
+
|
|
873
|
+
def test_langgraph_wind_power_flowchart(self):
|
|
874
|
+
"""測試風力發電計畫PDF分析與流程圖生成功能,檢查重點:
|
|
875
|
+
1. 是否能正確分析PDF中的風力發電計畫內容
|
|
876
|
+
2. 是否生成相關的流程圖並提供Google Storage URL
|
|
877
|
+
"""
|
|
878
|
+
|
|
879
|
+
current_dir = Path(__file__).parent
|
|
880
|
+
pdf_path = (
|
|
881
|
+
current_dir
|
|
882
|
+
/ "test_files"
|
|
883
|
+
/ "1120701A海廣離岸風力發電計畫環境影響說明書-C04.PDF"
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
# 確保檔案存在
|
|
887
|
+
self.assertTrue(pdf_path.exists(), f"Test file not found at {pdf_path}")
|
|
888
|
+
|
|
889
|
+
# 將絕對路徑轉為字串
|
|
890
|
+
pdf_path_str = str(pdf_path)
|
|
891
|
+
tmp_public_url = upload_and_get_tmp_public_url(
|
|
892
|
+
pdf_path_str,
|
|
893
|
+
"https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
894
|
+
"sebastian.hsu@gmail.com",
|
|
895
|
+
)
|
|
896
|
+
|
|
897
|
+
# Test payload
|
|
898
|
+
payload = {
|
|
899
|
+
"graph_name": "langgraph_react_agent",
|
|
900
|
+
"messages": [
|
|
901
|
+
{
|
|
902
|
+
"role": "user",
|
|
903
|
+
"content": f"{tmp_public_url} 幫我分析這個檔案,針對風力發電計畫,生成一張流程圖給我。",
|
|
904
|
+
}
|
|
905
|
+
],
|
|
906
|
+
"config": {
|
|
907
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
908
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
909
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
910
|
+
},
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
# Make the request
|
|
914
|
+
endpoint = "/api/langgraph/invoke"
|
|
915
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
916
|
+
print("-" * 50)
|
|
917
|
+
|
|
918
|
+
try:
|
|
919
|
+
response = self.api_post(endpoint, payload)
|
|
920
|
+
|
|
921
|
+
# Basic assertions to verify the response
|
|
922
|
+
self.assertIsNotNone(response)
|
|
923
|
+
|
|
924
|
+
# Extract the content field from the response
|
|
925
|
+
if "content" in response:
|
|
926
|
+
response_content = response["content"]
|
|
927
|
+
else:
|
|
928
|
+
self.fail("Response does not contain 'content' field")
|
|
929
|
+
|
|
930
|
+
# Define validation criteria
|
|
931
|
+
validation_criteria = """
|
|
932
|
+
請確認回應是否符合以下條件:
|
|
933
|
+
1. 回應中有包含風力發電計畫的分析內容,例如計畫目標、執行步驟、環境影響等
|
|
934
|
+
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的流程圖
|
|
935
|
+
3. 分析內容應該專注於風力發電計畫的程序或流程,而非僅是一般性描述
|
|
936
|
+
4. 是否在回答結尾包含一個笑話
|
|
937
|
+
|
|
938
|
+
所有條件都必須滿足,特別是必須有風力發電計畫的分析並含有Google Storage的流程圖URL。
|
|
939
|
+
"""
|
|
940
|
+
|
|
941
|
+
# Validate with LLM
|
|
942
|
+
validation_result = self.validate_with_llm(
|
|
943
|
+
response_content, validation_criteria
|
|
944
|
+
)
|
|
945
|
+
|
|
946
|
+
# Assert that the validation passed
|
|
947
|
+
self.assertTrue(
|
|
948
|
+
validation_result["pass"],
|
|
949
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
950
|
+
)
|
|
951
|
+
|
|
952
|
+
# Additional check for Google Storage URL
|
|
953
|
+
self.assertTrue(
|
|
954
|
+
"https://storage.googleapis.com" in response_content,
|
|
955
|
+
"Response does not contain a Google Storage URL",
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
except Exception as e:
|
|
959
|
+
print(
|
|
960
|
+
f"test_langgraph_wind_power_flowchart: Test failed with error: {str(e)}"
|
|
961
|
+
)
|
|
962
|
+
raise
|
|
963
|
+
|
|
964
|
+
def test_langgraph_oauth_flow_diagram(self):
|
|
965
|
+
"""測試OAuth流程圖生成功能,檢查重點:
|
|
966
|
+
1. 是否能正確生成OAuth認證流程圖
|
|
967
|
+
2. 是否提供Google Storage URL連結到生成的圖表
|
|
968
|
+
"""
|
|
969
|
+
# Test payload based on the curl command
|
|
970
|
+
payload = {
|
|
971
|
+
"graph_name": "langgraph_react_agent",
|
|
972
|
+
"messages": [
|
|
973
|
+
{
|
|
974
|
+
"role": "user",
|
|
975
|
+
"content": "我想做一個 oauth 的流程,幫我生出一個流程表",
|
|
976
|
+
}
|
|
977
|
+
],
|
|
978
|
+
"config": {
|
|
979
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
980
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
981
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
982
|
+
},
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
# Make the request
|
|
986
|
+
endpoint = "/api/langgraph/invoke"
|
|
987
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
988
|
+
print("-" * 50)
|
|
989
|
+
|
|
990
|
+
try:
|
|
991
|
+
response = self.api_post(endpoint, payload)
|
|
992
|
+
|
|
993
|
+
# Basic assertions to verify the response
|
|
994
|
+
self.assertIsNotNone(response)
|
|
995
|
+
|
|
996
|
+
# Extract the content field from the response
|
|
997
|
+
if "content" in response:
|
|
998
|
+
response_content = response["content"]
|
|
999
|
+
else:
|
|
1000
|
+
self.fail("Response does not contain 'content' field")
|
|
1001
|
+
|
|
1002
|
+
# Define validation criteria
|
|
1003
|
+
validation_criteria = """
|
|
1004
|
+
請確認回應是否符合以下條件:
|
|
1005
|
+
1. 回應中有詳細描述OAuth認證流程的步驟,必須包含關鍵步驟如授權請求、令牌交換等
|
|
1006
|
+
2. 回應中包含至少一個以 "https://storage.googleapis.com" 開頭的URL,這個URL應該指向一個生成的流程圖
|
|
1007
|
+
3. 回應應該提供清晰的OAuth流程解釋,包括不同參與者(如用戶、客戶端應用、授權伺服器等)之間的互動
|
|
1008
|
+
4. 是否在回答結尾包含一個笑話
|
|
1009
|
+
|
|
1010
|
+
所有條件都必須滿足,特別是必須有OAuth流程的詳細描述,並含有Google Storage的流程圖URL。
|
|
1011
|
+
"""
|
|
1012
|
+
|
|
1013
|
+
# Validate with LLM
|
|
1014
|
+
validation_result = self.validate_with_llm(
|
|
1015
|
+
response_content, validation_criteria
|
|
1016
|
+
)
|
|
1017
|
+
|
|
1018
|
+
# Assert that the validation passed
|
|
1019
|
+
self.assertTrue(
|
|
1020
|
+
validation_result["pass"],
|
|
1021
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1022
|
+
)
|
|
1023
|
+
|
|
1024
|
+
# Additional check for Google Storage URL
|
|
1025
|
+
self.assertTrue(
|
|
1026
|
+
"https://storage.googleapis.com" in response_content,
|
|
1027
|
+
"Response does not contain a Google Storage URL",
|
|
1028
|
+
)
|
|
1029
|
+
|
|
1030
|
+
except Exception as e:
|
|
1031
|
+
print(
|
|
1032
|
+
f"test_langgraph_oauth_flow_diagram: Test failed with error: {str(e)}"
|
|
1033
|
+
)
|
|
1034
|
+
raise
|
|
1035
|
+
|
|
1036
|
+
def test_langgraph_moda_news_dall_e(self):
|
|
1037
|
+
"""測試多節點處理數位發展部新聞流程,檢查重點:
|
|
1038
|
+
1. 是否會抓到今天日期,或截至今天日期的數位發展部相關新聞
|
|
1039
|
+
2. 是否有評分新聞 (1-10分),並標示出「最可愛」的新聞
|
|
1040
|
+
3. 是否有產出一張圖片,並帶有 URL
|
|
1041
|
+
"""
|
|
1042
|
+
# Test payload based on the curl command
|
|
1043
|
+
payload = {
|
|
1044
|
+
"graph_name": "langgraph_react_agent",
|
|
1045
|
+
"messages": [
|
|
1046
|
+
{
|
|
1047
|
+
"role": "user",
|
|
1048
|
+
"content": "好,那個你幫我那個啟動幾個多個節點,然後第一個節點請你幫我上網搜尋。 上網搜尋今天那個我們那個數位發展部的夥伴,或者最近一個禮拜數位發展部的那個夥伴有沒有什麼新聞好。 然後第二個節點,你幫我做一件事,你幫我做幫我把這些新聞評分數,評分一到十分。 那哪個新聞你覺得最可愛。 然後第三個節點,你幫我做一件事。 就是你把這個分數最可愛的那一個新聞挑出來以後,你幫我生成一個prompt,這個prompt是我要把你丟進打理畫圖用的prompt。那第四個節點你才真的呼叫打理把那個圖給畫出來,你幫我依序執行這個這個工作流程好不好,謝謝。",
|
|
1049
|
+
}
|
|
1050
|
+
],
|
|
1051
|
+
"config": {
|
|
1052
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1053
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1054
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
1055
|
+
},
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
# Make the request
|
|
1059
|
+
endpoint = "/api/langgraph/invoke"
|
|
1060
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1061
|
+
print("-" * 50)
|
|
1062
|
+
|
|
1063
|
+
try:
|
|
1064
|
+
response = self.api_post(endpoint, payload)
|
|
1065
|
+
|
|
1066
|
+
# Basic assertions to verify the response
|
|
1067
|
+
self.assertIsNotNone(response)
|
|
1068
|
+
|
|
1069
|
+
# Extract the content field from the response
|
|
1070
|
+
if "content" in response:
|
|
1071
|
+
response_content = response["content"]
|
|
1072
|
+
else:
|
|
1073
|
+
self.fail("Response does not contain 'content' field")
|
|
1074
|
+
|
|
1075
|
+
# Define validation criteria based on the test requirements
|
|
1076
|
+
validation_criteria = f"""
|
|
1077
|
+
請確認回應是否符合以下條件:
|
|
1078
|
+
1. 回應中有包含今天({self.local_time.strftime("%Y-%m-%d")})或最近一週內的數位發展部相關新聞資訊
|
|
1079
|
+
2. 回應中有對新聞進行1-10分的評分,並明確指出哪則新聞「最可愛」或分數最高
|
|
1080
|
+
3. 回應中包含至少一個圖片URL(通常是以http或https開頭的網址)
|
|
1081
|
+
4. 是否在回答結尾包含一個笑話
|
|
1082
|
+
|
|
1083
|
+
所有條件都必須滿足,特別是必須有數位發展部相關新聞、評分結果以及最終生成的圖片URL。
|
|
1084
|
+
"""
|
|
1085
|
+
|
|
1086
|
+
# Validate with LLM
|
|
1087
|
+
validation_result = self.validate_with_llm(
|
|
1088
|
+
response_content, validation_criteria
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
# Assert that the validation passed
|
|
1092
|
+
self.assertTrue(
|
|
1093
|
+
validation_result["pass"],
|
|
1094
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1095
|
+
)
|
|
1096
|
+
|
|
1097
|
+
# Additional check for an image URL
|
|
1098
|
+
self.assertTrue(
|
|
1099
|
+
"http" in response_content.lower()
|
|
1100
|
+
and (
|
|
1101
|
+
"jpg" in response_content.lower()
|
|
1102
|
+
or "png" in response_content.lower()
|
|
1103
|
+
or "https://storage.googleapis.com" in response_content
|
|
1104
|
+
),
|
|
1105
|
+
"Response does not contain a valid image URL",
|
|
1106
|
+
)
|
|
1107
|
+
|
|
1108
|
+
except Exception as e:
|
|
1109
|
+
print(f"test_langgraph_moda_news_dall_e: Test failed with error: {str(e)}")
|
|
1110
|
+
raise
|
|
1111
|
+
|
|
1112
|
+
def test_langgraph_global_disaster_news(self):
|
|
1113
|
+
"""測試深度研究災難新聞流程,檢查重點:
|
|
1114
|
+
1. 是否有收集全球災難新聞
|
|
1115
|
+
2. 是否以表格方式呈現災難資料
|
|
1116
|
+
3. 是否提供新聞來源
|
|
1117
|
+
"""
|
|
1118
|
+
# Test payload based on the curl command
|
|
1119
|
+
payload = {
|
|
1120
|
+
"graph_name": "langgraph_react_agent",
|
|
1121
|
+
"messages": [
|
|
1122
|
+
{
|
|
1123
|
+
"role": "user",
|
|
1124
|
+
"content": "請幫我進行深度研究,深度研究時請遵循以下三個步驟\n第一步驟:\n身為一個專業的全球新聞蒐集分析人員,請透過網路幫我收集Google News、路透社、美聯社、CNN、BBC、法新社、歐洲傳媒應急中心、公共透視網路、台灣聯合報及東森新聞,不使用不可信賴媒體及模擬資料,盡力確保可收集到全球的災難新聞。產生的研究報告文件名稱請以「xxxx年xx月xx日 全球災難新聞收集與追蹤」這樣的格式生成。收集時間請以UTC+8時區為基準,收集從2025年2月24日15:00到2025年2月25日15:00 的24 小時內,全球在時區內發生的災難事件及發生時間,包括大型自然災害或人為災難的人數統計,包括「傷亡」、「失蹤」、「受影響」、「流離失所」、「避難」等。自然災難類型包括但不限於地震、風災、火山爆發、寒流、大雪、冰雹、雪崩、土石流、野火、山火之類的極端氣候災難,人為災害包括 空難、戰爭、大型交通事故、海難、建物倒塌、疫情、中毒等並整理成表格,以繁體中文輸出,表格名請加上當天的年月日,格式為「xxxx年xx月xx日」並按照亞洲、歐洲、美洲、大洋洲、非洲等五大洲排列,後面要加上國家、省市別做完整地點呈現\n第二個表格請搜集以2025年2月24日為基準過去 96 小時的全球新聞中的災難報導。請確認事件發生時間在區間內,若是非區間內發生,請在說明欄清楚說明原因。並再三確認報導更新時間是否在區間內,也就是從前三天到當天,四天中發生的災難後續報導。\n第三個表格請就第一和第二個表格中收集到的災難事件中,逐條就每個災難進行250字的災難摘要及資料來源連結。並收集有關房屋(棟)的損壞統計,包括「受損」、「毁損」等。請務必以表格呈現,不要逐條展示。\n第二步驟:\n我要復盤上述資料都來自於可信賴的國際或台灣新聞媒體即時新聞報導,並且要找到三個不同的資訊來源,交差比對確認災難真實發生的時間點。第一個表格要有詳細的災難發生地點,每條災難收集的時間條件是指災難發生時間而非新聞發布時間在時間區間內,若災難發生時間不在時間區間內,請移到第二個表格。第二個表格是在過去96個小時不重複第一個表格的時間區間中新聞媒體對於災難的後續報導,第三個部份也請用表格呈現,而不是條列式。請檢視每個表格的災難資料,並合併相同的災難事件,確保每條事件只有一筆, 我很怕你使用到不可信的網路媒體資料,如:維基百科或是災難預言、天氣預警、模擬訊息或是你預訓練的資料Youtube及專題報導等。 若無傷亡或防屋毀損實際統計數據,請勿收集。\n第三步驟:\n請將下列附加檔案的表格一和表格二匯的每條災難事件透過網路可信賴媒體交叉比對時間和真實性後,匯整到原有的表格一和二中並合併相同的災難資訊,重新整理完整的表格三。再重新檢查每筆資料都符合各表格的時間區間及規範,重新盤點100遍,幫我整理出最完整的表格一到三。每筆資料請幫我再三搜尋可信賴媒體進行交差比對,務求每條事件都在真實世界中發生,地點明確,發生時間可驗證,若無傷亡資料就不收集。要確認表格一加上表格二的條目,能完整在表格三中呈現,不可多也不可少。",
|
|
1125
|
+
}
|
|
1126
|
+
],
|
|
1127
|
+
"config": {
|
|
1128
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1129
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1130
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
1131
|
+
},
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
# Make the request
|
|
1135
|
+
endpoint = "/api/langgraph/invoke"
|
|
1136
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1137
|
+
print("-" * 50)
|
|
1138
|
+
|
|
1139
|
+
try:
|
|
1140
|
+
response = self.api_post(endpoint, payload)
|
|
1141
|
+
|
|
1142
|
+
# Basic assertions to verify the response
|
|
1143
|
+
self.assertIsNotNone(response)
|
|
1144
|
+
|
|
1145
|
+
# Extract the content field from the response
|
|
1146
|
+
if "content" in response:
|
|
1147
|
+
response_content = response["content"]
|
|
1148
|
+
else:
|
|
1149
|
+
self.fail("Response does not contain 'content' field")
|
|
1150
|
+
|
|
1151
|
+
# Define validation criteria based on the test requirements
|
|
1152
|
+
validation_criteria = """
|
|
1153
|
+
請確認回應是否符合以下條件:
|
|
1154
|
+
1. 回應中包含災難新聞信息(至少提到了一些具體災難事件)
|
|
1155
|
+
2. 回應中有表格呈現(HTML表格標籤或是文字表格形式呈現災難數據)
|
|
1156
|
+
3. 回應中提供了新聞來源(至少包含一個可識別的媒體來源名稱如CNN、BBC、路透社等)
|
|
1157
|
+
4. 回應中有提及災難事件的類型(自然災害或人為災害)
|
|
1158
|
+
5. 回應中有提及災難事件的地理位置(國家、城市等)
|
|
1159
|
+
6. 回應中是否在結尾包含一個笑話
|
|
1160
|
+
|
|
1161
|
+
所有條件都必須滿足,特別是必須有災難新聞、表格呈現方式及新聞來源引用。
|
|
1162
|
+
"""
|
|
1163
|
+
|
|
1164
|
+
# Validate with LLM
|
|
1165
|
+
validation_result = self.validate_with_llm(
|
|
1166
|
+
response_content, validation_criteria
|
|
1167
|
+
)
|
|
1168
|
+
|
|
1169
|
+
# Assert that the validation passed
|
|
1170
|
+
self.assertTrue(
|
|
1171
|
+
validation_result["pass"],
|
|
1172
|
+
f"LLM validation failed in {self._testMethodName}: {validation_result['reason']}, LLM response: {response_content}",
|
|
1173
|
+
)
|
|
1174
|
+
|
|
1175
|
+
# Additional checks for tables and sources
|
|
1176
|
+
self.assertTrue(
|
|
1177
|
+
"|" in response_content
|
|
1178
|
+
or "<table" in response_content.lower()
|
|
1179
|
+
or "表格" in response_content,
|
|
1180
|
+
"Response does not appear to contain any tables",
|
|
1181
|
+
)
|
|
1182
|
+
|
|
1183
|
+
# # Check for news sources
|
|
1184
|
+
# news_sources = [
|
|
1185
|
+
# "CNN",
|
|
1186
|
+
# "BBC",
|
|
1187
|
+
# "路透社",
|
|
1188
|
+
# "美聯社",
|
|
1189
|
+
# "法新社",
|
|
1190
|
+
# "聯合報",
|
|
1191
|
+
# "東森",
|
|
1192
|
+
# ]
|
|
1193
|
+
# sources_found = any(source in response_content for source in news_sources)
|
|
1194
|
+
# self.assertTrue(
|
|
1195
|
+
# sources_found,
|
|
1196
|
+
# "Response does not reference any recognizable news sources",
|
|
1197
|
+
# )
|
|
1198
|
+
|
|
1199
|
+
except Exception as e:
|
|
1200
|
+
print(
|
|
1201
|
+
f"test_langgraph_global_disaster_news: Test failed with error: {str(e)}"
|
|
1202
|
+
)
|
|
1203
|
+
raise
|
|
1204
|
+
|
|
1205
|
+
def test_langgraph_date_time_comparison(self):
|
|
1206
|
+
"""測試日期時間比較功能,檢查重點:
|
|
1207
|
+
1. 當使用者僅指定日期時間而未明確要求比較時,agent 是否能自動使用 current_date_time 工具獲取當前時間
|
|
1208
|
+
2. agent 是否能自動使用 compare_date_time 工具比較指定時間與當前時間
|
|
1209
|
+
3. agent 是否能正確判斷指定時間是過去還是未來並提供解釋
|
|
1210
|
+
"""
|
|
1211
|
+
# 共用的系統提示
|
|
1212
|
+
system_prompt = "如果使用者的問題中有指定日期時間,不要預設它是未來或過去,一定要先使用 current_date_time 和 compare_date_time 這兩個工具,以取得現在的日期時間並判斷使用者指定的日期時間是過去或未來,然後再進行後續的動作。"
|
|
1213
|
+
|
|
1214
|
+
# 共用的配置
|
|
1215
|
+
base_config = {
|
|
1216
|
+
"system_prompt": system_prompt,
|
|
1217
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1218
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
1219
|
+
}
|
|
1220
|
+
|
|
1221
|
+
# 測試過去時間
|
|
1222
|
+
past_payload = {
|
|
1223
|
+
"graph_name": "langgraph_react_agent",
|
|
1224
|
+
"messages": [
|
|
1225
|
+
{
|
|
1226
|
+
"role": "user",
|
|
1227
|
+
"content": "2020年1月1日發生了什麼重要事件?",
|
|
1228
|
+
}
|
|
1229
|
+
],
|
|
1230
|
+
"config": base_config,
|
|
1231
|
+
}
|
|
1232
|
+
|
|
1233
|
+
# 測試未來時間
|
|
1234
|
+
future_payload = {
|
|
1235
|
+
"graph_name": "langgraph_react_agent",
|
|
1236
|
+
"messages": [
|
|
1237
|
+
{
|
|
1238
|
+
"role": "user",
|
|
1239
|
+
"content": "2030年12月31日會有什麼重要活動?",
|
|
1240
|
+
}
|
|
1241
|
+
],
|
|
1242
|
+
"config": base_config,
|
|
1243
|
+
}
|
|
1244
|
+
|
|
1245
|
+
# 測試端點
|
|
1246
|
+
endpoint = "/api/langgraph/invoke"
|
|
1247
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1248
|
+
print("-" * 50)
|
|
1249
|
+
|
|
1250
|
+
try:
|
|
1251
|
+
# 測試過去時間
|
|
1252
|
+
print("Testing past date comparison...")
|
|
1253
|
+
past_response = self.api_post(endpoint, past_payload)
|
|
1254
|
+
self.assertIsNotNone(past_response)
|
|
1255
|
+
|
|
1256
|
+
if "content" in past_response:
|
|
1257
|
+
past_response_content = past_response["content"]
|
|
1258
|
+
else:
|
|
1259
|
+
self.fail("Response does not contain 'content' field")
|
|
1260
|
+
|
|
1261
|
+
# 定義過去時間的驗證標準
|
|
1262
|
+
past_validation_criteria = """
|
|
1263
|
+
請確認回應是否符合以下條件:
|
|
1264
|
+
1. 回應中是否提到或暗示 2020年1月1日 是過去的時間
|
|
1265
|
+
2. 回應中是否有跡象表明 agent 使用了 current_date_time 工具獲取當前時間(例如提到「根據當前時間」、「現在是...」等)
|
|
1266
|
+
3. 回應中是否有跡象表明 agent 使用了 compare_date_time 工具比較時間(例如提到「比較結果」、「早於當前時間」等)
|
|
1267
|
+
4. 回應中是否包含關於 2020年1月1日 發生的重要事件的資訊
|
|
1268
|
+
"""
|
|
1269
|
+
|
|
1270
|
+
# 使用 LLM 驗證
|
|
1271
|
+
past_validation_result = self.validate_with_llm(
|
|
1272
|
+
past_response_content, past_validation_criteria
|
|
1273
|
+
)
|
|
1274
|
+
|
|
1275
|
+
# 驗證結果
|
|
1276
|
+
self.assertTrue(
|
|
1277
|
+
past_validation_result["pass"],
|
|
1278
|
+
f"LLM validation failed for past date in {self._testMethodName}: {past_validation_result['reason']}, LLM response: {past_response_content}",
|
|
1279
|
+
)
|
|
1280
|
+
|
|
1281
|
+
# 測試未來時間
|
|
1282
|
+
print("Testing future date comparison...")
|
|
1283
|
+
future_response = self.api_post(endpoint, future_payload)
|
|
1284
|
+
self.assertIsNotNone(future_response)
|
|
1285
|
+
|
|
1286
|
+
if "content" in future_response:
|
|
1287
|
+
future_response_content = future_response["content"]
|
|
1288
|
+
else:
|
|
1289
|
+
self.fail("Response does not contain 'content' field")
|
|
1290
|
+
|
|
1291
|
+
# 定義未來時間的驗證標準
|
|
1292
|
+
future_validation_criteria = """
|
|
1293
|
+
請確認回應是否符合以下條件:
|
|
1294
|
+
1. 回應中是否提到或暗示 2030年12月31日 是未來的時間
|
|
1295
|
+
2. 回應中是否有跡象表明 agent 使用了 current_date_time 工具獲取當前時間(例如提到「根據當前時間」、「現在是...」等)
|
|
1296
|
+
3. 回應中是否有跡象表明 agent 使用了 compare_date_time 工具比較時間(例如提到「比較結果」、「晚於當前時間」等)
|
|
1297
|
+
4. 回應中是否適當地處理了關於未來日期的問題(例如表明無法預測未來具體事件,但可能提供一些合理的推測或建議)
|
|
1298
|
+
"""
|
|
1299
|
+
|
|
1300
|
+
# 使用 LLM 驗證
|
|
1301
|
+
future_validation_result = self.validate_with_llm(
|
|
1302
|
+
future_response_content, future_validation_criteria
|
|
1303
|
+
)
|
|
1304
|
+
|
|
1305
|
+
# 驗證結果
|
|
1306
|
+
self.assertTrue(
|
|
1307
|
+
future_validation_result["pass"],
|
|
1308
|
+
f"LLM validation failed for future date in {self._testMethodName}: {future_validation_result['reason']}, LLM response: {future_response_content}",
|
|
1309
|
+
)
|
|
1310
|
+
|
|
1311
|
+
except Exception as e:
|
|
1312
|
+
print(
|
|
1313
|
+
f"test_langgraph_date_time_comparison: Test failed with error: {str(e)}"
|
|
1314
|
+
)
|
|
1315
|
+
raise
|
|
1316
|
+
|
|
1317
|
+
def test_langgraph_react_agent_business_flow(self):
|
|
1318
|
+
"""Test the langgraph_react_agent with a business flow example."""
|
|
1319
|
+
# Test payload
|
|
1320
|
+
payload = {
|
|
1321
|
+
"graph_name": "langgraph_react_agent",
|
|
1322
|
+
"messages": [
|
|
1323
|
+
{
|
|
1324
|
+
"role": "user",
|
|
1325
|
+
"content": "我想要請你 給我 一個業務流的範例,我只要三個節點,然後我還要針對這個業務流範例當中的其中中間的一個工作流程,然後來 進行示意,然後我那個工作流程也只需要三個節點,那原因是因為我要做成簡報,這樣子比較清晰容易看懂,那業務流 他是人如何工作的,的,的一個重點,記錄人員互動、決策跟情緒體驗,那也因此在業務流的時候呢,我需要請你幫我畫成使用者旅程地圖,裡面有酷酷點跟笑笑點,對不起,那請你節點幫我增加為五個好,然後再來工作流程的地方,它則是聚焦在任務跟文件,還有系統間的流動,它是一個操作的程序跟規則,那我,我也需要,我要修改前面的說法,要改成五個節點,那業務流是使用使用者旅程庫庫地圖,那工作流程的話,我希望它是一個,也許是一個工作流程程圖,或者是時序圖,然後看你覺得哪一個的表現比較明確,那這個內容呢,我需要以內政部的任何一個轄下的任何一個業務來進行舉例,然後我我想要請你幫我現在畫出來。",
|
|
1326
|
+
},
|
|
1327
|
+
{
|
|
1328
|
+
"role": "assistant",
|
|
1329
|
+
"content": "**我瞭解您需要一個業務流程的範例,特別是以內政部的業務為例,包含:**\n1. **一個使用者旅程地圖(業務流)- 5個節點,記錄人員互動、決策和情緒體驗**\n2. **一個工作流程程圖或時序圖 - 5個節點,聚焦在任務、文件和系統間的流動**\n**我會先幫您創建這兩個圖表。讓我們以內政部戶政司的「結婚登記」業務為例。**\n**首先,讓我建立使用者旅程地圖(業務流):**",
|
|
1330
|
+
},
|
|
1331
|
+
{
|
|
1332
|
+
"role": "user",
|
|
1333
|
+
"content": "等一下,你在第二個跟第三個給我的連結啊,裡面都是亂買,你到底在幹什麼?你重新確認一下。",
|
|
1334
|
+
},
|
|
1335
|
+
{"role": "assistant", "content": ""},
|
|
1336
|
+
{"role": "user", "content": "你還好嗎?你有沒有在動作?"},
|
|
1337
|
+
],
|
|
1338
|
+
"config": {
|
|
1339
|
+
"system_prompt": "回答後你都會在結尾講個笑話,並加上 emoji",
|
|
1340
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1341
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
1342
|
+
},
|
|
1343
|
+
}
|
|
1344
|
+
|
|
1345
|
+
# Make the request
|
|
1346
|
+
endpoint = "/api/langgraph/invoke"
|
|
1347
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1348
|
+
print("-" * 50)
|
|
1349
|
+
|
|
1350
|
+
try:
|
|
1351
|
+
response = self.api_post(endpoint, payload)
|
|
1352
|
+
|
|
1353
|
+
# Basic assertions to verify the response
|
|
1354
|
+
self.assertIsNotNone(response)
|
|
1355
|
+
|
|
1356
|
+
# Check for error response format
|
|
1357
|
+
if "detail" in response:
|
|
1358
|
+
error_message = response["detail"]
|
|
1359
|
+
# Fail if the error is about empty message content
|
|
1360
|
+
self.assertNotIn(
|
|
1361
|
+
"messages.3: all messages must have non-empty content except for the optional final assistant message",
|
|
1362
|
+
error_message,
|
|
1363
|
+
f"API returned expected error about empty content: {error_message}",
|
|
1364
|
+
)
|
|
1365
|
+
print(
|
|
1366
|
+
f"API returned an error, but not the empty content error: {error_message}"
|
|
1367
|
+
)
|
|
1368
|
+
# Check for successful response format
|
|
1369
|
+
else:
|
|
1370
|
+
# For successful responses, verify content key exists at the top level
|
|
1371
|
+
self.assertIn("content", response, "Response missing 'content' field")
|
|
1372
|
+
print(f"Response received successfully with content!")
|
|
1373
|
+
|
|
1374
|
+
except Exception as e:
|
|
1375
|
+
self.fail(f"Error testing API: {str(e)}")
|
|
1376
|
+
|
|
1377
|
+
def test_auth_token_verify_api(self):
|
|
1378
|
+
"""測試 auth token verification API,檢查重點:
|
|
1379
|
+
1. 有效 token 的驗證
|
|
1380
|
+
2. 無效 token 的錯誤處理
|
|
1381
|
+
3. 缺少 token 參數的錯誤處理
|
|
1382
|
+
"""
|
|
1383
|
+
# Test with valid-looking token
|
|
1384
|
+
endpoint = "/api/auth/token_verify"
|
|
1385
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1386
|
+
print("-" * 50)
|
|
1387
|
+
|
|
1388
|
+
try:
|
|
1389
|
+
# Test case 1: Valid token format (though may not be valid in backend)
|
|
1390
|
+
valid_token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.test.token"
|
|
1391
|
+
|
|
1392
|
+
# Use form data for POST request
|
|
1393
|
+
import requests
|
|
1394
|
+
url = f"{self.base_url}{endpoint}"
|
|
1395
|
+
response = requests.post(url, data={"access_token": valid_token})
|
|
1396
|
+
|
|
1397
|
+
# Expect either 200 (valid) or 401 (invalid token) or 500 (service unavailable)
|
|
1398
|
+
self.assertIn(response.status_code, [200, 401, 500],
|
|
1399
|
+
f"Unexpected status code: {response.status_code}")
|
|
1400
|
+
|
|
1401
|
+
if response.status_code == 200:
|
|
1402
|
+
json_response = response.json()
|
|
1403
|
+
self.assertIn("is_success", json_response)
|
|
1404
|
+
print(f"Token verification succeeded: {json_response}")
|
|
1405
|
+
elif response.status_code == 401:
|
|
1406
|
+
json_response = response.json()
|
|
1407
|
+
self.assertIn("detail", json_response)
|
|
1408
|
+
self.assertIn("Invalid", json_response["detail"])
|
|
1409
|
+
print(f"Token verification failed as expected: {json_response}")
|
|
1410
|
+
elif response.status_code == 500:
|
|
1411
|
+
# Service might not be configured
|
|
1412
|
+
json_response = response.json()
|
|
1413
|
+
print(f"Service unavailable (expected in dev): {json_response}")
|
|
1414
|
+
|
|
1415
|
+
# Test case 2: Missing token parameter
|
|
1416
|
+
response_missing = requests.post(url, data={})
|
|
1417
|
+
self.assertEqual(response_missing.status_code, 422,
|
|
1418
|
+
"Missing token should return 422 validation error")
|
|
1419
|
+
|
|
1420
|
+
json_response_missing = response_missing.json()
|
|
1421
|
+
self.assertIn("detail", json_response_missing)
|
|
1422
|
+
print(f"Missing token handled correctly: {json_response_missing}")
|
|
1423
|
+
|
|
1424
|
+
# Test case 3: Empty token
|
|
1425
|
+
response_empty = requests.post(url, data={"access_token": ""})
|
|
1426
|
+
self.assertIn(response_empty.status_code, [400, 401, 500],
|
|
1427
|
+
"Empty token should return 400, 401, or 500")
|
|
1428
|
+
print(f"Empty token handled with status: {response_empty.status_code}")
|
|
1429
|
+
|
|
1430
|
+
except Exception as e:
|
|
1431
|
+
print(f"test_auth_token_verify_api: Test failed with error: {str(e)}")
|
|
1432
|
+
raise
|
|
1433
|
+
|
|
1434
|
+
def test_langgraph_react_agent_social_housing(self):
|
|
1435
|
+
"""Test the langgraph_react_agent with social housing application analysis."""
|
|
1436
|
+
# Test payload
|
|
1437
|
+
payload = {
|
|
1438
|
+
"graph_name": "langgraph_react_agent",
|
|
1439
|
+
"messages": [
|
|
1440
|
+
{
|
|
1441
|
+
"role": "user",
|
|
1442
|
+
"content": "幫我分析這分申請資料,我的檔案已經在你的system prompt裡面了,不需要再做其他的處理,直接使用system prompt裡的資料進行分析",
|
|
1443
|
+
}
|
|
1444
|
+
],
|
|
1445
|
+
"config": {
|
|
1446
|
+
"system_prompt": "<Context>\n您是一位非常細心且專業的中華民國內政部審查社宅入住資格的資深審查員\n</Context>\n\n<Objective>\n您的目標是在審查使用者是否符合社宅入住資格\n</Objective>\n\n<Style>\n請保持中立及客觀,以精煉的方式分析資料,確保過程簡潔明瞭。\n</Style>\n\n<Tone>\n專業且耐心的語氣。\n</Tone>\n\n<Audience>\n此流程設計專門供機關內的相關同仁使用,他們需清楚知道哪些是合格的資料,哪些不是,以便後續決策。\n<Audience>\n\n<第一步驟>\n<審查規則>\n1. 年滿18歲(含)以上之中華民國國民\n2. 有於北北基桃設籍、就學、就業任一需求者\n3. 於北北基桃無自有住宅者或個別持有小於40平方公尺\n4. 家庭成員每人每月平均所得不超過新臺幣59,150元(舉例:新臺幣60,000元就是超過59,150元)\n</審查規則>\n\n<Response>\n1. 請幫我列出完整的審查結果,通過打V,不通過打X,並且列出不通過的原因\n2. 幫我把上面的內容產出一個表格回傳給我\n</Response>\n</第一步驟>\n\n<第二步驟>\n幫我依照<第一步驟>產生的結果,生產一個html的頁面報告,html頁面裡面要顯示以下資訊,你不要直接給我html的程式\n1. 請幫我畫一張圓餅圖分析年齡分佈\n2. 請幫我把原始審查資料跟<第一步驟>的產出做合併\n3. 請幫我做一個搜尋工具快速搜尋審查資料\n</第二步驟>\n\n\n\n\n\n以下為附加檔案內容:\n\n檔名:\n社宅申請模擬資料.csv\n檔案內容:\n申請人姓名,性別,出生年月日,婚姻狀況,身分證字號,電話,電子郵件,職業,戶籍地址,戶籍地址是否承租,通訊地址,通訊地址是否承租,緊急聯絡人,稱謂,緊急聯絡人電話,申請類別,申請戶類型,家具承租方案,配偶姓名,配偶身分證字號,家庭成員數量,持有住宅平方公尺數,平均家庭成員月收入\n林志明,男,1985-06-12,已婚,A123456789,0912-345-678,zhiming.lin@example.com,軟體工程師,新北市板橋區文化路一段100號5樓,是,新北市板橋區文化路一段100號5樓,是,林志豪,兄弟,0922-123-456,設籍,一般戶,同步租,王美玲,B234567890,3,0,45000\n陳雅婷,女,2001-05-06,未婚,B287654321,0933-876-543,yating.chen@example.com,學生,台北市信義區松仁路50號12樓,否,新北市新莊區中正路200號3樓,是,陳大明,父親,0955-987-654,就學,一般戶,買斷,,,1,0,38000\n張家豪,男,1978-11-30,已婚,C198765432,0977-654-321,jiahao.zhang@example.com,公務員,台北市大安區和平東路二段106號7樓,是,台北市大安區和平東路二段106號7樓,是,張明德,父親,0910-234-567,設籍,現職警消人員,同步租,李佩珊,D123456789,4,0,52000\n黃麗華,女,1965-08-15,喪偶,E234567890,0988-765-432,lihua.huang@example.com,退休教師,新北市三重區重新路一段88號4樓,否,新北市三重區重新路一段88號4樓,否,黃志成,兒子,0923-456-789,設籍,65歲以上老人,同步租,,,1,0,25000\n吳建志,男,1982-04-20,已婚,F123456789,0932-123-456,jianzhih.wu@example.com,銀行經理,台北市中山區南京東路三段25號9樓,是,台北市中山區南京東路三段25號9樓,是,吳大維,父親,0912-876-543,就業,一般戶,買斷,林美琪,G234567890,5,15,60000\n李小芳,女,1992-12-05,已婚,H123456789,0956-789-123,xiaofang.li@example.com,設計師,基隆市中正區中正路100號3樓,是,台北市松山區民生東路四段133號6樓,是,李大中,父親,0933-222-111,就業,未成年子女三人以上,同步租,王大明,I234567890,5,0,42000\n王俊傑,男,1988-07-15,未婚,J123456789,0978-456-123,junjie.wang@example.com,工程師,桃園市中壢區中央西路二段30號5樓,否,新北市新店區北新路三段100號7樓,是,王大華,父親,0910-876-543,就業,身心障礙,買斷,,,1,0,35000\n蔡美玲,女,1975-09-28,離婚,K123456789,0933-789-456,meiling.tsai@example.com,會計師,新北市永和區永和路一段50號4樓,是,新北市永和區永和路一段50號4樓,是,蔡明哲,兄弟,0922-333-444,就業,特殊境遇家庭,同步租,,,2,0,40000\n鄭志偉,男,1980-02-14,已婚,L123456789,0955-123-789,zhiwei.zheng@example.com,教師,台北市文山區木柵路一段100號3樓,否,台北市文山區木柵路一段100號3樓,否,鄭大勇,父親,0912-345-678,就業,一般戶,買斷,陳美美,M234567890,3,20,48000\n林美華,女,2003-01-03,未婚,N123456789,0978-789-123,meihua.lin@example.com,學生,新北市汐止區大同路一段150號5樓,是,新北市汐止區大同路一段150號5樓,是,林大明,父親,0933-456-789,就學,一般戶,同步租,,,1,0,36000\n張大為,男,1972-06-30,已婚,O123456789,0910-234-567,dawei.zhang@example.com,建築師,台北市大安區復興南路一段200號7樓,否,台北市大安區復興南路一段200號7樓,否,張小明,兒子,0922-123-456,設籍,一般戶,買斷,王麗麗,P234567890,3,25,65000\n陳俊宏,男,1990-08-12,未婚,Q123456789,0933-222-111,junhong.chen@example.com,軍職人員,桃園市龜山區文化一路100號5樓,是,桃園市龜山區文化一路100號5樓,是,陳大勇,父親,0955-123-456,就業,軍職人員,同步租,,,1,0,42000\n楊雅琪,女,1987-04-15,已婚,R123456789,0978-456-789,yaqi.yang@example.com,行銷經理,新北市中和區中和路100號6樓,是,新北市中和區中和路100號6樓,是,楊大明,父親,0912-345-678,設籍,一般戶,買斷,李志明,S234567890,2,0,55000\n劉大偉,男,1968-12-25,已婚,T123456789,0955-789-123,dawei.liu@example.com,計程車司機,基隆市安樂區安樂路二段50號3樓,否,基隆市安樂區安樂路二段50號3樓,否,劉小明,兒子,0933-456-789,設籍,低收入戶,同步租,張美美,U234567890,4,0,28000\n高美玲,女,1993-03-08,未婚,V123456789,0910-123-456,meiling.gao@example.com,設計師,台北市內湖區內湖路一段300號8樓,是,台北市內湖區內湖路一段300號8樓,是,高大明,父親,0922-789-123,就業,一般戶,買斷,,,1,0,38000\n鄭美美,女,1983-09-18,離婚,W123456789,0933-789-123,meimei.zheng@example.com,餐廳經理,新北市新莊區新莊路100號4樓,是,新北市新莊區新莊路100號4樓,是,鄭大勇,父親,0955-456-789,設籍,特殊境遇家庭,同步租,,,2,0,32000\n林志豪,男,1991-05-20,未婚,X123456789,0978-123-456,zhihao.lin@example.com,工程師,台北市士林區士林路100號5樓,否,台北市士林區士林路100號5樓,否,林大明,父親,0912-789-123,就業,一般戶,買斷,,,1,0,45000\n王美玲,女,1979-11-12,已婚,Y123456789,0955-123-456,meiling.wang@example.com,會計師,新北市板橋區民生路100號6樓,是,新北市板橋區民生路100號6樓,是,王大明,父親,0933-123-456,設籍,一般戶,同步租,李大偉,Z234567890,3,0,50000\n陳志明,男,1960-02-28,已婚,A234567891,0910-789-456,zhiming.chen@example.com,退休公務員,台北市北投區石牌路一段100號3樓,否,台北市北投區石牌路一段100號3樓,否,陳小明,兒子,0922-456-789,設籍,65歲以上老人,同步租,林美美,B234567891,2,0,30000\n莊雅婷,女,2002-08-08,未婚,C234567891,0933-456-123,yating.zhuang@example.com,學生,桃園市桃園區中正路100號5樓,是,桃園市桃園區中正路100號5樓,是,莊大明,父親,0955-789-456,就學,原住民,買斷,,,1,0,37000",
|
|
1447
|
+
"botrun_flow_lang_url": "https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app",
|
|
1448
|
+
"user_id": "sebastian.hsu@gmail.com",
|
|
1449
|
+
},
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1452
|
+
# Make the request
|
|
1453
|
+
endpoint = "/api/langgraph/invoke"
|
|
1454
|
+
print(f"\nTesting API: {self.base_url}{endpoint}")
|
|
1455
|
+
print("-" * 50)
|
|
1456
|
+
|
|
1457
|
+
try:
|
|
1458
|
+
response = self.api_post(endpoint, payload)
|
|
1459
|
+
|
|
1460
|
+
# Basic assertions to verify the response
|
|
1461
|
+
self.assertIsNotNone(response)
|
|
1462
|
+
|
|
1463
|
+
# Validation criteria for social housing application analysis
|
|
1464
|
+
validation_criteria = """
|
|
1465
|
+
請驗證回應是否包含以下內容:
|
|
1466
|
+
1. 完整的社宅申請審查結果,包括通過/不通過標記
|
|
1467
|
+
2. 以Markdown格式顯示的審查結果表格(應包含 | 字符作為表格格式)
|
|
1468
|
+
3. 一個以"https://storage.googleapis.com"開頭的HTML頁面URL
|
|
1469
|
+
4. 基於提供的人口統計數據的分析
|
|
1470
|
+
"""
|
|
1471
|
+
|
|
1472
|
+
# Validate the response with LLM
|
|
1473
|
+
if "content" in response:
|
|
1474
|
+
validation_result = self.validate_with_llm(
|
|
1475
|
+
response["content"], validation_criteria
|
|
1476
|
+
)
|
|
1477
|
+
|
|
1478
|
+
# Additionally, directly check for markdown table and storage URL
|
|
1479
|
+
content = response["content"]
|
|
1480
|
+
has_markdown_table = "|" in content and "-|-" in content
|
|
1481
|
+
has_storage_url = "https://storage.googleapis.com" in content
|
|
1482
|
+
|
|
1483
|
+
# Custom assertions for specific requirements
|
|
1484
|
+
self.assertTrue(has_markdown_table, "回應中不包含Markdown表格")
|
|
1485
|
+
self.assertTrue(
|
|
1486
|
+
has_storage_url,
|
|
1487
|
+
"回應中不包含Google Cloud Storage網址",
|
|
1488
|
+
)
|
|
1489
|
+
|
|
1490
|
+
# Assert that validation passed
|
|
1491
|
+
self.assertTrue(
|
|
1492
|
+
validation_result.get("pass", False),
|
|
1493
|
+
f"驗證失敗:{validation_result.get('reason', '未提供原因')}",
|
|
1494
|
+
)
|
|
1495
|
+
|
|
1496
|
+
print(f"回應驗證成功:{validation_result.get('reason', '')}")
|
|
1497
|
+
else:
|
|
1498
|
+
self.fail("Response does not contain 'content' field")
|
|
1499
|
+
|
|
1500
|
+
except Exception as e:
|
|
1501
|
+
self.fail(f"Error testing API: {str(e)}")
|
|
1502
|
+
|
|
1503
|
+
|
|
1504
|
+
def run_with_base_url(base_url=None):
|
|
1505
|
+
"""Run the tests with an optional custom base URL
|
|
1506
|
+
|
|
1507
|
+
Args:
|
|
1508
|
+
base_url: Optional base URL to override the default
|
|
1509
|
+
"""
|
|
1510
|
+
# If a base URL is provided, set it as a class attribute
|
|
1511
|
+
if base_url:
|
|
1512
|
+
TestAPIFunctionality.base_url = base_url
|
|
1513
|
+
|
|
1514
|
+
# Run the tests
|
|
1515
|
+
unittest.main(argv=["first-arg-is-ignored"], exit=False)
|
|
1516
|
+
|
|
1517
|
+
|
|
1518
|
+
if __name__ == "__main__":
|
|
1519
|
+
unittest.main()
|
|
1520
|
+
# single test test_langgraph_multinode_news_dall_e
|
|
1521
|
+
# run_with_base_url("https://botrun-flow-lang-fastapi-dev-36186877499.asia-east1.run.app")
|
|
1522
|
+
# test_suite = unittest.TestSuite()
|
|
1523
|
+
# test_suite.addTest(TestAPIFunctionality("test_langgraph_multinode_news_dall_e"))
|
|
1524
|
+
# runner = unittest.TextTestRunner()
|
|
1525
|
+
# runner.run(test_suite)
|