optexity 0.1.5.1__py3-none-any.whl → 0.1.5.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- optexity/inference/core/interaction/handle_click.py +4 -1
- optexity/inference/core/interaction/handle_command.py +24 -3
- optexity/inference/core/interaction/handle_hover.py +83 -0
- optexity/inference/core/interaction/handle_keypress.py +26 -0
- optexity/inference/core/interaction/handle_select_utils.py +11 -1
- optexity/inference/core/logging.py +8 -1
- optexity/inference/core/run_automation.py +6 -3
- optexity/inference/core/run_extraction.py +37 -0
- optexity/inference/core/run_interaction.py +10 -0
- optexity/inference/core/run_two_fa.py +12 -7
- optexity/inference/infra/browser.py +187 -52
- optexity/inference/infra/utils.py +98 -0
- optexity/schema/actions/extraction_action.py +9 -3
- optexity/schema/actions/interaction_action.py +38 -4
- optexity/schema/actions/two_fa_action.py +1 -1
- optexity/schema/automation.py +2 -7
- optexity/schema/task.py +7 -2
- optexity/utils/utils.py +14 -0
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/METADATA +1 -1
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/RECORD +24 -22
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/WHEEL +1 -1
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/entry_points.txt +0 -0
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/licenses/LICENSE +0 -0
- {optexity-0.1.5.1.dist-info → optexity-0.1.5.3.dist-info}/top_level.txt +0 -0
|
@@ -59,8 +59,11 @@ async def click_element_index(
|
|
|
59
59
|
return
|
|
60
60
|
|
|
61
61
|
async def _actual_click_element():
|
|
62
|
+
print(
|
|
63
|
+
f"Clicking element with index: {index} and button: {click_element_action.button}"
|
|
64
|
+
)
|
|
62
65
|
action_model = browser.backend_agent.ActionModel(
|
|
63
|
-
**{"click": {"index": index}}
|
|
66
|
+
**{"click": {"index": index, "button": click_element_action.button}}
|
|
64
67
|
)
|
|
65
68
|
await browser.backend_agent.multi_act([action_model])
|
|
66
69
|
|
|
@@ -13,6 +13,7 @@ from optexity.inference.infra.browser import Browser
|
|
|
13
13
|
from optexity.schema.actions.interaction_action import (
|
|
14
14
|
CheckAction,
|
|
15
15
|
ClickElementAction,
|
|
16
|
+
HoverAction,
|
|
16
17
|
InputTextAction,
|
|
17
18
|
SelectOptionAction,
|
|
18
19
|
UncheckAction,
|
|
@@ -32,6 +33,7 @@ async def command_based_action_with_retry(
|
|
|
32
33
|
| CheckAction
|
|
33
34
|
| UploadFileAction
|
|
34
35
|
| UncheckAction
|
|
36
|
+
| HoverAction
|
|
35
37
|
),
|
|
36
38
|
browser: Browser,
|
|
37
39
|
memory: Memory,
|
|
@@ -83,7 +85,7 @@ async def command_based_action_with_retry(
|
|
|
83
85
|
)
|
|
84
86
|
elif isinstance(action, InputTextAction):
|
|
85
87
|
await input_text_locator(
|
|
86
|
-
action, locator, max_timeout_seconds_per_try
|
|
88
|
+
action, locator, browser, max_timeout_seconds_per_try
|
|
87
89
|
)
|
|
88
90
|
elif isinstance(action, SelectOptionAction):
|
|
89
91
|
await select_option_locator(
|
|
@@ -102,6 +104,8 @@ async def command_based_action_with_retry(
|
|
|
102
104
|
await uncheck_locator(
|
|
103
105
|
action, locator, max_timeout_seconds_per_try, browser
|
|
104
106
|
)
|
|
107
|
+
elif isinstance(action, HoverAction):
|
|
108
|
+
await hover_locator(locator, max_timeout_seconds_per_try)
|
|
105
109
|
elif isinstance(action, UploadFileAction):
|
|
106
110
|
await upload_file_locator(action, locator)
|
|
107
111
|
logger.debug(
|
|
@@ -149,7 +153,9 @@ async def click_locator(
|
|
|
149
153
|
)
|
|
150
154
|
else:
|
|
151
155
|
await locator.click(
|
|
152
|
-
|
|
156
|
+
button=click_element_action.button,
|
|
157
|
+
no_wait_after=True,
|
|
158
|
+
timeout=max_timeout_seconds_per_try * 1000,
|
|
153
159
|
)
|
|
154
160
|
|
|
155
161
|
if click_element_action.expect_download:
|
|
@@ -163,6 +169,7 @@ async def click_locator(
|
|
|
163
169
|
async def input_text_locator(
|
|
164
170
|
input_text_action: InputTextAction,
|
|
165
171
|
locator: Locator,
|
|
172
|
+
browser: Browser,
|
|
166
173
|
max_timeout_seconds_per_try: float,
|
|
167
174
|
):
|
|
168
175
|
|
|
@@ -172,12 +179,19 @@ async def input_text_locator(
|
|
|
172
179
|
no_wait_after=True,
|
|
173
180
|
timeout=max_timeout_seconds_per_try * 1000,
|
|
174
181
|
)
|
|
175
|
-
|
|
182
|
+
elif input_text_action.fill_or_type == "type":
|
|
176
183
|
await locator.type(
|
|
177
184
|
input_text_action.input_text,
|
|
178
185
|
no_wait_after=True,
|
|
179
186
|
timeout=max_timeout_seconds_per_try * 1000,
|
|
180
187
|
)
|
|
188
|
+
else:
|
|
189
|
+
page = await browser.get_current_page()
|
|
190
|
+
if page is None:
|
|
191
|
+
return
|
|
192
|
+
for char in input_text_action.input_text:
|
|
193
|
+
await page.keyboard.press(char)
|
|
194
|
+
await asyncio.sleep(0.1)
|
|
181
195
|
|
|
182
196
|
if input_text_action.press_enter:
|
|
183
197
|
await locator.press("Enter")
|
|
@@ -211,6 +225,13 @@ async def uncheck_locator(
|
|
|
211
225
|
)
|
|
212
226
|
|
|
213
227
|
|
|
228
|
+
async def hover_locator(
|
|
229
|
+
locator: Locator,
|
|
230
|
+
max_timeout_seconds_per_try: float,
|
|
231
|
+
):
|
|
232
|
+
await locator.hover(no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000)
|
|
233
|
+
|
|
234
|
+
|
|
214
235
|
async def upload_file_locator(upload_file_action: UploadFileAction, locator: Locator):
|
|
215
236
|
await locator.set_input_files(upload_file_action.file_path)
|
|
216
237
|
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
|
|
3
|
+
from optexity.inference.core.interaction.handle_command import (
|
|
4
|
+
command_based_action_with_retry,
|
|
5
|
+
)
|
|
6
|
+
from optexity.inference.core.interaction.utils import get_index_from_prompt
|
|
7
|
+
from optexity.inference.infra.browser import Browser
|
|
8
|
+
from optexity.schema.actions.interaction_action import HoverAction
|
|
9
|
+
from optexity.schema.memory import Memory
|
|
10
|
+
from optexity.schema.task import Task
|
|
11
|
+
|
|
12
|
+
logger = logging.getLogger(__name__)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
async def handle_hover_element(
|
|
16
|
+
hover_element_action: HoverAction,
|
|
17
|
+
task: Task,
|
|
18
|
+
memory: Memory,
|
|
19
|
+
browser: Browser,
|
|
20
|
+
max_timeout_seconds_per_try: float,
|
|
21
|
+
max_tries: int,
|
|
22
|
+
):
|
|
23
|
+
|
|
24
|
+
if hover_element_action.command and not hover_element_action.skip_command:
|
|
25
|
+
last_error = await command_based_action_with_retry(
|
|
26
|
+
hover_element_action,
|
|
27
|
+
browser,
|
|
28
|
+
memory,
|
|
29
|
+
task,
|
|
30
|
+
max_tries,
|
|
31
|
+
max_timeout_seconds_per_try,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
if last_error is None:
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
if not hover_element_action.skip_prompt:
|
|
38
|
+
logger.debug(
|
|
39
|
+
f"Executing prompt-based action: {hover_element_action.__class__.__name__}"
|
|
40
|
+
)
|
|
41
|
+
await hover_element_index(hover_element_action, browser, memory, task)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
async def hover_element_index(
|
|
45
|
+
hover_element_action: HoverAction,
|
|
46
|
+
browser: Browser,
|
|
47
|
+
memory: Memory,
|
|
48
|
+
task: Task,
|
|
49
|
+
):
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
index = await get_index_from_prompt(
|
|
53
|
+
memory, hover_element_action.prompt_instructions, browser, task
|
|
54
|
+
)
|
|
55
|
+
if index is None:
|
|
56
|
+
return
|
|
57
|
+
|
|
58
|
+
print(f"Hovering element with index: {index}")
|
|
59
|
+
|
|
60
|
+
async def _actual_hover_element():
|
|
61
|
+
try:
|
|
62
|
+
action_model = browser.backend_agent.ActionModel(
|
|
63
|
+
**{"hover": {"index": index}}
|
|
64
|
+
)
|
|
65
|
+
await browser.backend_agent.multi_act([action_model])
|
|
66
|
+
except Exception as e:
|
|
67
|
+
logger.error(f"Error in hover_element_index: {e} trying right click")
|
|
68
|
+
node = await browser.backend_agent.browser_session.get_element_by_index(
|
|
69
|
+
index
|
|
70
|
+
)
|
|
71
|
+
if node is None:
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
backend_page = (
|
|
75
|
+
await browser.backend_agent.browser_session.get_current_page()
|
|
76
|
+
)
|
|
77
|
+
element = await backend_page.get_element(node.backend_node_id)
|
|
78
|
+
await element.click(button="right")
|
|
79
|
+
|
|
80
|
+
await _actual_hover_element()
|
|
81
|
+
except Exception as e:
|
|
82
|
+
logger.error(f"Error in hover_element_index: {e}")
|
|
83
|
+
return
|
|
@@ -14,3 +14,29 @@ async def handle_key_press(
|
|
|
14
14
|
|
|
15
15
|
if keypress_action.type == KeyPressType.ENTER:
|
|
16
16
|
await page.keyboard.press("Enter")
|
|
17
|
+
if keypress_action.type == KeyPressType.TAB:
|
|
18
|
+
await page.keyboard.press("Tab")
|
|
19
|
+
if keypress_action.type == KeyPressType.ZERO:
|
|
20
|
+
await page.keyboard.press("0")
|
|
21
|
+
if keypress_action.type == KeyPressType.ONE:
|
|
22
|
+
await page.keyboard.press("1")
|
|
23
|
+
if keypress_action.type == KeyPressType.TWO:
|
|
24
|
+
await page.keyboard.press("2")
|
|
25
|
+
if keypress_action.type == KeyPressType.THREE:
|
|
26
|
+
await page.keyboard.press("3")
|
|
27
|
+
if keypress_action.type == KeyPressType.FOUR:
|
|
28
|
+
await page.keyboard.press("4")
|
|
29
|
+
if keypress_action.type == KeyPressType.FIVE:
|
|
30
|
+
await page.keyboard.press("5")
|
|
31
|
+
if keypress_action.type == KeyPressType.SIX:
|
|
32
|
+
await page.keyboard.press("6")
|
|
33
|
+
if keypress_action.type == KeyPressType.SEVEN:
|
|
34
|
+
await page.keyboard.press("7")
|
|
35
|
+
if keypress_action.type == KeyPressType.EIGHT:
|
|
36
|
+
await page.keyboard.press("8")
|
|
37
|
+
if keypress_action.type == KeyPressType.NINE:
|
|
38
|
+
await page.keyboard.press("9")
|
|
39
|
+
if keypress_action.type == KeyPressType.SLASH:
|
|
40
|
+
await page.keyboard.press("/")
|
|
41
|
+
if keypress_action.type == KeyPressType.SPACE:
|
|
42
|
+
await page.keyboard.press("Space")
|
|
@@ -57,9 +57,19 @@ async def smart_select(
|
|
|
57
57
|
options: list[SelectOptionValue], patterns: list[str], memory: Memory
|
|
58
58
|
):
|
|
59
59
|
# Get all options from the <select>
|
|
60
|
-
|
|
60
|
+
## TODO: remove this once we have a better way to handle select one
|
|
61
61
|
matched_values = []
|
|
62
62
|
|
|
63
|
+
if len(options) == 0:
|
|
64
|
+
return []
|
|
65
|
+
if len(options) == 1:
|
|
66
|
+
return [options[0].value]
|
|
67
|
+
if len(options) == 2 and "Select One" in [o.value for o in options]:
|
|
68
|
+
if options[0].value == "Select One":
|
|
69
|
+
return [options[1].value]
|
|
70
|
+
else:
|
|
71
|
+
return [options[0].value]
|
|
72
|
+
|
|
63
73
|
for p in patterns:
|
|
64
74
|
# If pattern contains regex characters, treat as regex
|
|
65
75
|
is_regex = p.startswith("^") or p.endswith("$") or ".*" in p
|
|
@@ -364,7 +364,14 @@ async def save_latest_memory_state_locally(
|
|
|
364
364
|
await f.write(json.dumps(task.input_parameters, indent=4))
|
|
365
365
|
|
|
366
366
|
async with aiofiles.open(step_directory / "secure_parameters.json", "w") as f:
|
|
367
|
-
|
|
367
|
+
secure_parameters = {
|
|
368
|
+
key: [
|
|
369
|
+
a.model_dump(exclude_none=True, exclude_defaults=True)
|
|
370
|
+
for a in value
|
|
371
|
+
]
|
|
372
|
+
for key, value in task.secure_parameters.items()
|
|
373
|
+
}
|
|
374
|
+
await f.write(json.dumps(secure_parameters, indent=4))
|
|
368
375
|
|
|
369
376
|
async with aiofiles.open(step_directory / "generated_variables.json", "w") as f:
|
|
370
377
|
await f.write(json.dumps(memory.variables.generated_variables, indent=4))
|
|
@@ -25,7 +25,6 @@ from optexity.inference.core.run_interaction import (
|
|
|
25
25
|
run_interaction_action,
|
|
26
26
|
)
|
|
27
27
|
from optexity.inference.core.run_python_script import run_python_script_action
|
|
28
|
-
from optexity.inference.core.run_two_fa import run_two_fa_action
|
|
29
28
|
from optexity.inference.infra.browser import Browser
|
|
30
29
|
from optexity.schema.actions.interaction_action import DownloadUrlAsPdfAction
|
|
31
30
|
from optexity.schema.automation import ActionNode, ForLoopNode, IfElseNode
|
|
@@ -55,11 +54,14 @@ async def run_automation(task: Task, child_process_id: int):
|
|
|
55
54
|
logger.info(f"Task {task.task_id} started running")
|
|
56
55
|
memory = None
|
|
57
56
|
browser = None
|
|
57
|
+
|
|
58
58
|
try:
|
|
59
59
|
await start_task_in_server(task)
|
|
60
60
|
memory = Memory()
|
|
61
|
+
|
|
61
62
|
browser = Browser(
|
|
62
63
|
memory=memory,
|
|
64
|
+
user_data_dir=f"/tmp/userdata_{task.task_id}",
|
|
63
65
|
headless=False,
|
|
64
66
|
channel=task.automation.browser_channel,
|
|
65
67
|
debug_port=9222 + child_process_id,
|
|
@@ -67,9 +69,12 @@ async def run_automation(task: Task, child_process_id: int):
|
|
|
67
69
|
proxy_session_id=task.proxy_session_id(
|
|
68
70
|
settings.PROXY_PROVIDER if task.use_proxy else None
|
|
69
71
|
),
|
|
72
|
+
is_dedicated=task.is_dedicated,
|
|
70
73
|
)
|
|
71
74
|
await browser.start()
|
|
72
75
|
|
|
76
|
+
browser.memory = memory
|
|
77
|
+
|
|
73
78
|
automation = task.automation
|
|
74
79
|
|
|
75
80
|
memory.automation_state.step_index = -1
|
|
@@ -274,8 +279,6 @@ async def run_action_node(
|
|
|
274
279
|
await run_extraction_action(
|
|
275
280
|
action_node.extraction_action, memory, browser, task
|
|
276
281
|
)
|
|
277
|
-
elif action_node.two_fa_action:
|
|
278
|
-
await run_two_fa_action(action_node.two_fa_action, memory)
|
|
279
282
|
elif action_node.python_script_action:
|
|
280
283
|
await run_python_script_action(
|
|
281
284
|
action_node.python_script_action, memory, browser
|
|
@@ -4,12 +4,14 @@ import traceback
|
|
|
4
4
|
import aiofiles
|
|
5
5
|
import httpx
|
|
6
6
|
|
|
7
|
+
from optexity.inference.core.run_two_fa import run_two_fa_action
|
|
7
8
|
from optexity.inference.infra.browser import Browser
|
|
8
9
|
from optexity.inference.models import GeminiModels, get_llm_model
|
|
9
10
|
from optexity.schema.actions.extraction_action import (
|
|
10
11
|
ExtractionAction,
|
|
11
12
|
LLMExtraction,
|
|
12
13
|
NetworkCallExtraction,
|
|
14
|
+
PythonScriptExtraction,
|
|
13
15
|
ScreenshotExtraction,
|
|
14
16
|
StateExtraction,
|
|
15
17
|
)
|
|
@@ -51,6 +53,14 @@ async def run_extraction_action(
|
|
|
51
53
|
task,
|
|
52
54
|
extraction_action.unique_identifier,
|
|
53
55
|
)
|
|
56
|
+
elif extraction_action.python_script:
|
|
57
|
+
await handle_python_script_extraction(
|
|
58
|
+
extraction_action.python_script,
|
|
59
|
+
memory,
|
|
60
|
+
browser,
|
|
61
|
+
task,
|
|
62
|
+
extraction_action.unique_identifier,
|
|
63
|
+
)
|
|
54
64
|
elif extraction_action.screenshot:
|
|
55
65
|
await handle_screenshot_extraction(
|
|
56
66
|
extraction_action.screenshot,
|
|
@@ -65,6 +75,8 @@ async def run_extraction_action(
|
|
|
65
75
|
browser,
|
|
66
76
|
extraction_action.unique_identifier,
|
|
67
77
|
)
|
|
78
|
+
elif extraction_action.two_fa_action:
|
|
79
|
+
await run_two_fa_action(extraction_action.two_fa_action, memory)
|
|
68
80
|
|
|
69
81
|
|
|
70
82
|
async def handle_state_extraction(
|
|
@@ -225,6 +237,31 @@ async def handle_network_call_extraction(
|
|
|
225
237
|
)
|
|
226
238
|
|
|
227
239
|
|
|
240
|
+
async def handle_python_script_extraction(
|
|
241
|
+
python_script_extraction: PythonScriptExtraction,
|
|
242
|
+
memory: Memory,
|
|
243
|
+
browser: Browser,
|
|
244
|
+
task: Task,
|
|
245
|
+
unique_identifier: str | None = None,
|
|
246
|
+
):
|
|
247
|
+
local_vars = {}
|
|
248
|
+
exec(python_script_extraction.script, {}, local_vars)
|
|
249
|
+
code_fn = local_vars["code_fn"]
|
|
250
|
+
axtree = memory.browser_states[-1].axtree
|
|
251
|
+
result = await code_fn(axtree)
|
|
252
|
+
if result is not None:
|
|
253
|
+
memory.variables.output_data.append(
|
|
254
|
+
OutputData(
|
|
255
|
+
unique_identifier=unique_identifier,
|
|
256
|
+
json_data=result,
|
|
257
|
+
)
|
|
258
|
+
)
|
|
259
|
+
else:
|
|
260
|
+
logger.warning(
|
|
261
|
+
f"No result from Python script extraction: {python_script_extraction.script}"
|
|
262
|
+
)
|
|
263
|
+
|
|
264
|
+
|
|
228
265
|
async def download_request(
|
|
229
266
|
network_call: NetworkRequest, download_filename: str, task: Task, memory: Memory
|
|
230
267
|
):
|
|
@@ -12,6 +12,7 @@ from optexity.inference.core.interaction.handle_check import (
|
|
|
12
12
|
handle_uncheck_element,
|
|
13
13
|
)
|
|
14
14
|
from optexity.inference.core.interaction.handle_click import handle_click_element
|
|
15
|
+
from optexity.inference.core.interaction.handle_hover import handle_hover_element
|
|
15
16
|
from optexity.inference.core.interaction.handle_input import handle_input_text
|
|
16
17
|
from optexity.inference.core.interaction.handle_keypress import handle_key_press
|
|
17
18
|
from optexity.inference.core.interaction.handle_select import handle_select_option
|
|
@@ -95,6 +96,15 @@ async def run_interaction_action(
|
|
|
95
96
|
interaction_action.max_timeout_seconds_per_try,
|
|
96
97
|
interaction_action.max_tries,
|
|
97
98
|
)
|
|
99
|
+
elif interaction_action.hover:
|
|
100
|
+
await handle_hover_element(
|
|
101
|
+
interaction_action.hover,
|
|
102
|
+
task,
|
|
103
|
+
memory,
|
|
104
|
+
browser,
|
|
105
|
+
interaction_action.max_timeout_seconds_per_try,
|
|
106
|
+
interaction_action.max_tries,
|
|
107
|
+
)
|
|
98
108
|
elif interaction_action.go_back:
|
|
99
109
|
await handle_go_back(interaction_action.go_back, memory, browser)
|
|
100
110
|
elif interaction_action.download_url_as_pdf:
|
|
@@ -33,6 +33,7 @@ async def run_two_fa_action(two_fa_action: TwoFAAction, memory: Memory):
|
|
|
33
33
|
|
|
34
34
|
elapsed = 0
|
|
35
35
|
messages = None
|
|
36
|
+
code = None
|
|
36
37
|
|
|
37
38
|
while elapsed < two_fa_action.max_wait_time:
|
|
38
39
|
messages = await fetch_messages(
|
|
@@ -109,12 +110,16 @@ async def fetch_messages(
|
|
|
109
110
|
end_2fa_time=end_2fa_time,
|
|
110
111
|
)
|
|
111
112
|
|
|
112
|
-
|
|
113
|
+
try:
|
|
114
|
+
async with httpx.AsyncClient(timeout=30.0) as client:
|
|
113
115
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
116
|
+
response = await client.post(
|
|
117
|
+
url, json=body.model_dump(mode="json"), headers=headers
|
|
118
|
+
)
|
|
119
|
+
response.raise_for_status()
|
|
120
|
+
response_data = FetchMessagesResponse.model_validate(response.json())
|
|
119
121
|
|
|
120
|
-
|
|
122
|
+
return response_data.messages
|
|
123
|
+
except Exception as e:
|
|
124
|
+
logger.error(f"Error fetching messages: {e}")
|
|
125
|
+
return []
|
|
@@ -2,35 +2,48 @@ import asyncio
|
|
|
2
2
|
import base64
|
|
3
3
|
import json
|
|
4
4
|
import logging
|
|
5
|
+
import pathlib
|
|
5
6
|
import re
|
|
7
|
+
import shutil
|
|
6
8
|
from typing import Literal
|
|
7
9
|
from uuid import uuid4
|
|
8
10
|
|
|
11
|
+
import patchright.async_api
|
|
12
|
+
import playwright.async_api
|
|
9
13
|
from browser_use import Agent, BrowserSession, ChatGoogle
|
|
10
14
|
from browser_use.browser.views import BrowserStateSummary
|
|
11
15
|
from patchright._impl._errors import TimeoutError as PatchrightTimeoutError
|
|
12
16
|
from playwright._impl._errors import TimeoutError as PlaywrightTimeoutError
|
|
13
17
|
from playwright.async_api import Download, Locator, Page, Request, Response
|
|
14
18
|
|
|
19
|
+
from optexity.inference.infra.utils import _download_extension, _extract_extension
|
|
15
20
|
from optexity.schema.memory import Memory, NetworkRequest, NetworkResponse
|
|
16
21
|
from optexity.utils.settings import settings
|
|
17
22
|
|
|
18
23
|
logger = logging.getLogger(__name__)
|
|
19
24
|
|
|
25
|
+
_global_playwright: (
|
|
26
|
+
playwright.async_api.Playwright | patchright.async_api.Playwright | None
|
|
27
|
+
) = None
|
|
28
|
+
_global_context: (
|
|
29
|
+
playwright.async_api.BrowserContext | patchright.async_api.BrowserContext | None
|
|
30
|
+
) = None
|
|
31
|
+
|
|
20
32
|
|
|
21
33
|
class Browser:
|
|
22
34
|
def __init__(
|
|
23
35
|
self,
|
|
24
36
|
memory: Memory,
|
|
25
|
-
user_data_dir: str
|
|
37
|
+
user_data_dir: str,
|
|
26
38
|
headless: bool = False,
|
|
27
|
-
proxy: str = None,
|
|
39
|
+
proxy: str | None = None,
|
|
28
40
|
stealth: bool = True,
|
|
29
41
|
backend: Literal["browser-use", "browserbase"] = "browser-use",
|
|
30
42
|
debug_port: int = 9222,
|
|
31
43
|
channel: Literal["chromium", "chrome"] = "chromium",
|
|
32
44
|
use_proxy: bool = False,
|
|
33
45
|
proxy_session_id: str | None = None,
|
|
46
|
+
is_dedicated: bool = False,
|
|
34
47
|
):
|
|
35
48
|
|
|
36
49
|
if proxy:
|
|
@@ -44,9 +57,15 @@ class Browser:
|
|
|
44
57
|
self.debug_port = debug_port
|
|
45
58
|
self.use_proxy = use_proxy
|
|
46
59
|
self.proxy_session_id = proxy_session_id
|
|
47
|
-
self.playwright
|
|
60
|
+
self.playwright: (
|
|
61
|
+
playwright.async_api.Playwright | patchright.async_api.Playwright | None
|
|
62
|
+
) = None
|
|
48
63
|
self.browser = None
|
|
49
|
-
self.context
|
|
64
|
+
self.context: (
|
|
65
|
+
playwright.async_api.BrowserContext
|
|
66
|
+
| patchright.async_api.BrowserContext
|
|
67
|
+
| None
|
|
68
|
+
) = None
|
|
50
69
|
self.page = None
|
|
51
70
|
self.cdp_url = f"http://localhost:{self.debug_port}"
|
|
52
71
|
self.backend_agent = None
|
|
@@ -54,16 +73,108 @@ class Browser:
|
|
|
54
73
|
self.memory = memory
|
|
55
74
|
self.page_to_target_id = []
|
|
56
75
|
self.previous_total_pages = 0
|
|
57
|
-
|
|
76
|
+
self.is_dedicated = is_dedicated
|
|
58
77
|
self.active_downloads = 0
|
|
59
78
|
self.all_active_downloads_done = asyncio.Event()
|
|
60
79
|
self.all_active_downloads_done.set()
|
|
61
80
|
|
|
62
81
|
self.network_calls: list[NetworkResponse | NetworkRequest] = []
|
|
63
82
|
|
|
83
|
+
self.extensions = [
|
|
84
|
+
# {
|
|
85
|
+
# "name": "optexity recorder",
|
|
86
|
+
# "id": "pbaganbicadeoacahamnbgohafchgakp",
|
|
87
|
+
# "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dpbaganbicadeoacahamnbgohafchgakp%26uc",
|
|
88
|
+
# },
|
|
89
|
+
{
|
|
90
|
+
"name": "I still don't care about cookies",
|
|
91
|
+
"id": "edibdbjcniadpccecjdfdjjppcpchdlm",
|
|
92
|
+
"url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dedibdbjcniadpccecjdfdjjppcpchdlm%26uc",
|
|
93
|
+
},
|
|
94
|
+
# {
|
|
95
|
+
# "name": "popupoff",
|
|
96
|
+
# "id": "kiodaajmphnkcajieajajinghpejdjai",
|
|
97
|
+
# "url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dkiodaajmphnkcajieajajinghpejdjai%26uc",
|
|
98
|
+
# },
|
|
99
|
+
{
|
|
100
|
+
"name": "ublock origin",
|
|
101
|
+
"id": "ddkjiahejlhfcafbddmgiahcphecmpfh",
|
|
102
|
+
"url": "https://clients2.google.com/service/update2/crx?response=redirect&prodversion=133&acceptformat=crx3&x=id%3Dddkjiahejlhfcafbddmgiahcphecmpfh%26uc",
|
|
103
|
+
},
|
|
104
|
+
]
|
|
105
|
+
|
|
64
106
|
async def start(self):
|
|
107
|
+
global _global_playwright, _global_context
|
|
65
108
|
logger.debug("Starting browser")
|
|
66
109
|
try:
|
|
110
|
+
cache_dir = pathlib.Path("/tmp/extensions")
|
|
111
|
+
cache_dir.mkdir(parents=True, exist_ok=True)
|
|
112
|
+
extension_paths = []
|
|
113
|
+
loaded_extension_names = []
|
|
114
|
+
for ext in self.extensions:
|
|
115
|
+
ext_dir = cache_dir / ext["id"]
|
|
116
|
+
crx_file = cache_dir / f'{ext["id"]}.crx'
|
|
117
|
+
|
|
118
|
+
# Check if extension is already extracted
|
|
119
|
+
if ext_dir.exists() and (ext_dir / "manifest.json").exists():
|
|
120
|
+
logger.info(
|
|
121
|
+
f'✅ Using cached {ext["name"]} extension from {ext_dir}'
|
|
122
|
+
)
|
|
123
|
+
extension_paths.append(str(ext_dir))
|
|
124
|
+
loaded_extension_names.append(ext["name"])
|
|
125
|
+
continue
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
# Download extension if not cached
|
|
129
|
+
if not crx_file.exists():
|
|
130
|
+
logger.info(f'📦 Downloading {ext["name"]} extension...')
|
|
131
|
+
_download_extension(ext["url"], crx_file)
|
|
132
|
+
else:
|
|
133
|
+
logger.info(f'📦 Found cached {ext["name"]} .crx file')
|
|
134
|
+
|
|
135
|
+
# Extract extension
|
|
136
|
+
logger.info(f'📂 Extracting {ext["name"]} extension...')
|
|
137
|
+
_extract_extension(crx_file, ext_dir)
|
|
138
|
+
|
|
139
|
+
extension_paths.append(str(ext_dir))
|
|
140
|
+
loaded_extension_names.append(ext["name"])
|
|
141
|
+
logger.info(f'✅ Successfully loaded {ext["name"]}')
|
|
142
|
+
|
|
143
|
+
except Exception as e:
|
|
144
|
+
logger.error(
|
|
145
|
+
f'❌ Failed to setup {ext["name"]} extension: {e}',
|
|
146
|
+
exc_info=True,
|
|
147
|
+
)
|
|
148
|
+
continue
|
|
149
|
+
|
|
150
|
+
if not extension_paths:
|
|
151
|
+
logger.error("⚠️ No extensions were loaded successfully!")
|
|
152
|
+
|
|
153
|
+
logger.info(f"Loaded extensions: {', '.join(loaded_extension_names)}")
|
|
154
|
+
|
|
155
|
+
args = [
|
|
156
|
+
"--disable-site-isolation-trials",
|
|
157
|
+
"--disable-web-security",
|
|
158
|
+
"--disable-features=IsolateOrigins,site-per-process",
|
|
159
|
+
"--allow-running-insecure-content",
|
|
160
|
+
"--ignore-certificate-errors",
|
|
161
|
+
"--ignore-ssl-errors",
|
|
162
|
+
"--ignore-certificate-errors-spki-list",
|
|
163
|
+
"--enable-extensions",
|
|
164
|
+
"--disable-extensions-file-access-check",
|
|
165
|
+
"--disable-extensions-http-throttling",
|
|
166
|
+
]
|
|
167
|
+
|
|
168
|
+
if extension_paths:
|
|
169
|
+
disable_except = (
|
|
170
|
+
f'--disable-extensions-except={",".join(extension_paths)}'
|
|
171
|
+
)
|
|
172
|
+
load_extension = f'--load-extension={",".join(extension_paths)}'
|
|
173
|
+
args.append(disable_except)
|
|
174
|
+
args.append(load_extension)
|
|
175
|
+
logger.info(f"Extension args: {disable_except}")
|
|
176
|
+
logger.info(f"Extension args: {load_extension}")
|
|
177
|
+
|
|
67
178
|
if self.playwright is not None:
|
|
68
179
|
await self.playwright.stop()
|
|
69
180
|
|
|
@@ -98,44 +209,64 @@ class Browser:
|
|
|
98
209
|
if settings.PROXY_PASSWORD is not None:
|
|
99
210
|
proxy["password"] = settings.PROXY_PASSWORD
|
|
100
211
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
212
|
+
if (
|
|
213
|
+
_global_playwright is None
|
|
214
|
+
or _global_context is None
|
|
215
|
+
or not self.is_dedicated
|
|
216
|
+
):
|
|
217
|
+
self.playwright = await async_playwright().start()
|
|
218
|
+
self.context = await self.playwright.chromium.launch_persistent_context(
|
|
219
|
+
channel=self.channel,
|
|
220
|
+
user_data_dir=self.user_data_dir,
|
|
221
|
+
headless=self.headless,
|
|
222
|
+
proxy=proxy,
|
|
223
|
+
args=[
|
|
224
|
+
# "--start-fullscreen",
|
|
225
|
+
"--disable-popup-blocking",
|
|
226
|
+
"--window-size=1920,1080",
|
|
227
|
+
f"--remote-debugging-port={self.debug_port}",
|
|
228
|
+
"--disable-gpu",
|
|
229
|
+
"--disable-background-networking",
|
|
230
|
+
]
|
|
231
|
+
+ args,
|
|
232
|
+
chromium_sandbox=False,
|
|
233
|
+
no_viewport=True,
|
|
234
|
+
)
|
|
235
|
+
_global_playwright = self.playwright
|
|
236
|
+
_global_context = self.context
|
|
117
237
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
)
|
|
238
|
+
async def log_request(req: Request):
|
|
239
|
+
await self.log_request(req)
|
|
121
240
|
|
|
122
|
-
|
|
123
|
-
|
|
241
|
+
async def handle_random_download(download: Download):
|
|
242
|
+
await self.handle_random_download(download)
|
|
124
243
|
|
|
125
|
-
|
|
126
|
-
|
|
244
|
+
async def handle_random_url_downloads(resp: Response):
|
|
245
|
+
await self.handle_random_url_downloads(resp)
|
|
127
246
|
|
|
128
|
-
|
|
129
|
-
|
|
247
|
+
self.context.on("request", log_request)
|
|
248
|
+
self.context.on("response", handle_random_url_downloads)
|
|
130
249
|
|
|
131
|
-
|
|
132
|
-
|
|
250
|
+
self.context.on(
|
|
251
|
+
"page", lambda p: (p.on("download", handle_random_download))
|
|
252
|
+
)
|
|
133
253
|
|
|
134
|
-
self.
|
|
135
|
-
|
|
136
|
-
|
|
254
|
+
elif self.is_dedicated:
|
|
255
|
+
self.context = _global_context
|
|
256
|
+
self.playwright = _global_playwright
|
|
257
|
+
for i in range(len(self.context.pages) - 1, 0, -1):
|
|
258
|
+
await self.context.pages[i].close()
|
|
259
|
+
else:
|
|
260
|
+
raise ValueError(
|
|
261
|
+
"Browser is not dedicated and global playwright and context are not set"
|
|
262
|
+
)
|
|
137
263
|
|
|
138
|
-
self.
|
|
264
|
+
# self.context = await self.browser.new_context(
|
|
265
|
+
# no_viewport=True, ignore_https_errors=True
|
|
266
|
+
# )
|
|
267
|
+
|
|
268
|
+
# self.page = await self.context.new_page()
|
|
269
|
+
self.page = self.context.pages[0]
|
|
139
270
|
|
|
140
271
|
browser_session = BrowserSession(cdp_url=self.cdp_url, keep_alive=True)
|
|
141
272
|
|
|
@@ -162,7 +293,7 @@ class Browser:
|
|
|
162
293
|
raise e
|
|
163
294
|
|
|
164
295
|
async def stop(self):
|
|
165
|
-
logger.debug("Stopping
|
|
296
|
+
logger.debug("Stopping backend agent")
|
|
166
297
|
if self.backend_agent is not None:
|
|
167
298
|
logger.debug("Stopping backend agent")
|
|
168
299
|
self.backend_agent.stop()
|
|
@@ -174,21 +305,25 @@ class Browser:
|
|
|
174
305
|
logger.debug("Browser session reset")
|
|
175
306
|
self.backend_agent = None
|
|
176
307
|
|
|
177
|
-
if self.
|
|
178
|
-
logger.debug("Stopping context")
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
308
|
+
if not self.is_dedicated:
|
|
309
|
+
logger.debug("Stopping context and playwright and browser as not dedicated")
|
|
310
|
+
if self.context is not None:
|
|
311
|
+
logger.debug("Stopping context")
|
|
312
|
+
await self.context.close()
|
|
313
|
+
self.context = None
|
|
314
|
+
|
|
315
|
+
if self.browser is not None:
|
|
316
|
+
logger.debug("Stopping browser")
|
|
317
|
+
await self.browser.close()
|
|
318
|
+
self.browser = None
|
|
319
|
+
|
|
320
|
+
if self.playwright is not None:
|
|
321
|
+
logger.debug("Stopping playwright")
|
|
322
|
+
await self.playwright.stop()
|
|
323
|
+
self.playwright = None
|
|
324
|
+
shutil.rmtree(self.user_data_dir, ignore_errors=True)
|
|
325
|
+
else:
|
|
326
|
+
logger.debug("browser not stopped as dedicated")
|
|
192
327
|
|
|
193
328
|
async def get_current_page(self) -> Page | None:
|
|
194
329
|
if self.context is None:
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
|
|
5
|
+
logging.basicConfig(level=logging.INFO)
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _download_extension(url: str, output_path: Path) -> None:
|
|
10
|
+
"""Download extension .crx file."""
|
|
11
|
+
import urllib.request
|
|
12
|
+
|
|
13
|
+
try:
|
|
14
|
+
logger.info(f"Downloading from: {url}")
|
|
15
|
+
with urllib.request.urlopen(url) as response:
|
|
16
|
+
content = response.read()
|
|
17
|
+
logger.info(f"Downloaded {len(content)} bytes")
|
|
18
|
+
with open(output_path, "wb") as f:
|
|
19
|
+
f.write(content)
|
|
20
|
+
logger.info(f"Saved to: {output_path}")
|
|
21
|
+
except Exception as e:
|
|
22
|
+
raise Exception(f"Failed to download extension: {e}")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _extract_extension(crx_path: Path, extract_dir: Path) -> None:
|
|
26
|
+
"""Extract .crx file to directory."""
|
|
27
|
+
import os
|
|
28
|
+
import shutil
|
|
29
|
+
import zipfile
|
|
30
|
+
|
|
31
|
+
# Remove existing directory
|
|
32
|
+
if extract_dir.exists():
|
|
33
|
+
shutil.rmtree(extract_dir)
|
|
34
|
+
|
|
35
|
+
extract_dir.mkdir(parents=True, exist_ok=True)
|
|
36
|
+
|
|
37
|
+
try:
|
|
38
|
+
# CRX files are ZIP files with a header, try to extract as ZIP
|
|
39
|
+
with zipfile.ZipFile(crx_path, "r") as zip_ref:
|
|
40
|
+
zip_ref.extractall(extract_dir)
|
|
41
|
+
|
|
42
|
+
# Verify manifest exists
|
|
43
|
+
if not (extract_dir / "manifest.json").exists():
|
|
44
|
+
raise Exception("No manifest.json found in extension")
|
|
45
|
+
|
|
46
|
+
logger.info("✅ Extracted as regular ZIP file")
|
|
47
|
+
|
|
48
|
+
except zipfile.BadZipFile:
|
|
49
|
+
logger.info("📦 Processing CRX header...")
|
|
50
|
+
# CRX files have a header before the ZIP data
|
|
51
|
+
with open(crx_path, "rb") as f:
|
|
52
|
+
# Read CRX header to find ZIP start
|
|
53
|
+
magic = f.read(4)
|
|
54
|
+
if magic != b"Cr24":
|
|
55
|
+
raise Exception(f"Invalid CRX file format. Magic: {magic}")
|
|
56
|
+
|
|
57
|
+
version = int.from_bytes(f.read(4), "little")
|
|
58
|
+
logger.info(f"CRX version: {version}")
|
|
59
|
+
|
|
60
|
+
if version == 2:
|
|
61
|
+
pubkey_len = int.from_bytes(f.read(4), "little")
|
|
62
|
+
sig_len = int.from_bytes(f.read(4), "little")
|
|
63
|
+
f.seek(16 + pubkey_len + sig_len)
|
|
64
|
+
elif version == 3:
|
|
65
|
+
header_len = int.from_bytes(f.read(4), "little")
|
|
66
|
+
f.seek(12 + header_len)
|
|
67
|
+
else:
|
|
68
|
+
raise Exception(f"Unsupported CRX version: {version}")
|
|
69
|
+
|
|
70
|
+
# Extract ZIP data
|
|
71
|
+
zip_data = f.read()
|
|
72
|
+
logger.info(f"ZIP data size: {len(zip_data)} bytes")
|
|
73
|
+
|
|
74
|
+
# Write ZIP data to temp file and extract
|
|
75
|
+
import tempfile
|
|
76
|
+
|
|
77
|
+
with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as temp_zip:
|
|
78
|
+
temp_zip.write(zip_data)
|
|
79
|
+
temp_zip.flush()
|
|
80
|
+
|
|
81
|
+
with zipfile.ZipFile(temp_zip.name, "r") as zip_ref:
|
|
82
|
+
zip_ref.extractall(extract_dir)
|
|
83
|
+
|
|
84
|
+
os.unlink(temp_zip.name)
|
|
85
|
+
|
|
86
|
+
# Remove 'key' from manifest if present (can cause issues)
|
|
87
|
+
manifest_path = extract_dir / "manifest.json"
|
|
88
|
+
if manifest_path.exists():
|
|
89
|
+
data = json.loads(manifest_path.read_text())
|
|
90
|
+
logger.info(f"Manifest version: {data.get('manifest_version')}")
|
|
91
|
+
logger.info(f"Extension name: {data.get('name')}")
|
|
92
|
+
|
|
93
|
+
if "key" in data:
|
|
94
|
+
logger.info("Removing 'key' field from manifest")
|
|
95
|
+
del data["key"]
|
|
96
|
+
manifest_path.write_text(json.dumps(data, indent=2))
|
|
97
|
+
else:
|
|
98
|
+
raise Exception("manifest.json not found after extraction")
|
|
@@ -3,6 +3,7 @@ from uuid import uuid4
|
|
|
3
3
|
|
|
4
4
|
from pydantic import BaseModel, Field, field_validator, model_validator
|
|
5
5
|
|
|
6
|
+
from optexity.schema.actions.two_fa_action import TwoFAAction
|
|
6
7
|
from optexity.utils.utils import build_model
|
|
7
8
|
|
|
8
9
|
|
|
@@ -53,13 +54,16 @@ class LLMExtraction(BaseModel):
|
|
|
53
54
|
return self
|
|
54
55
|
|
|
55
56
|
def replace(self, pattern: str, replacement: str):
|
|
57
|
+
self.extraction_instructions = self.extraction_instructions.replace(
|
|
58
|
+
pattern, replacement
|
|
59
|
+
)
|
|
56
60
|
return self
|
|
57
61
|
|
|
58
62
|
|
|
59
63
|
class NetworkCallExtraction(BaseModel):
|
|
60
64
|
url_pattern: Optional[str] = None
|
|
61
|
-
extract_from: None | Literal["request", "response"] =
|
|
62
|
-
download_from: None | Literal["request", "response"] =
|
|
65
|
+
extract_from: None | Literal["request", "response"] = "response"
|
|
66
|
+
download_from: None | Literal["request", "response"] = "response"
|
|
63
67
|
download_filename: str | None = None
|
|
64
68
|
|
|
65
69
|
@model_validator(mode="before")
|
|
@@ -109,6 +113,7 @@ class ExtractionAction(BaseModel):
|
|
|
109
113
|
python_script: Optional[PythonScriptExtraction] = None
|
|
110
114
|
screenshot: Optional[ScreenshotExtraction] = None
|
|
111
115
|
state: Optional[StateExtraction] = None
|
|
116
|
+
two_fa_action: TwoFAAction | None = None
|
|
112
117
|
|
|
113
118
|
@model_validator(mode="after")
|
|
114
119
|
def validate_one_extraction(cls, model: "ExtractionAction"):
|
|
@@ -119,12 +124,13 @@ class ExtractionAction(BaseModel):
|
|
|
119
124
|
"python_script": model.python_script,
|
|
120
125
|
"screenshot": model.screenshot,
|
|
121
126
|
"state": model.state,
|
|
127
|
+
"two_fa_action": model.two_fa_action,
|
|
122
128
|
}
|
|
123
129
|
non_null = [k for k, v in provided.items() if v is not None]
|
|
124
130
|
|
|
125
131
|
if len(non_null) != 1:
|
|
126
132
|
raise ValueError(
|
|
127
|
-
"Exactly one of llm, networkcall, python_script, or
|
|
133
|
+
"Exactly one of llm, networkcall, python_script, screenshot, state, or two_fa_action must be provided"
|
|
128
134
|
)
|
|
129
135
|
|
|
130
136
|
return model
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
from enum import Enum, unique
|
|
2
|
-
from typing import Literal
|
|
2
|
+
from typing import Any, Literal
|
|
3
3
|
from uuid import uuid4
|
|
4
4
|
|
|
5
5
|
from pydantic import BaseModel, Field, model_validator
|
|
@@ -69,6 +69,10 @@ class UncheckAction(BaseAction):
|
|
|
69
69
|
pass
|
|
70
70
|
|
|
71
71
|
|
|
72
|
+
class HoverAction(BaseAction):
|
|
73
|
+
pass
|
|
74
|
+
|
|
75
|
+
|
|
72
76
|
class SelectOptionAction(BaseAction):
|
|
73
77
|
select_values: list[str]
|
|
74
78
|
expect_download: bool = False
|
|
@@ -100,6 +104,7 @@ class ClickElementAction(BaseAction):
|
|
|
100
104
|
double_click: bool = False
|
|
101
105
|
expect_download: bool = False
|
|
102
106
|
download_filename: str | None = None
|
|
107
|
+
button: Literal["left", "right", "middle"] = "left"
|
|
103
108
|
|
|
104
109
|
@model_validator(mode="after")
|
|
105
110
|
def set_download_filename(cls, model: "ClickElementAction"):
|
|
@@ -121,7 +126,7 @@ class ClickElementAction(BaseAction):
|
|
|
121
126
|
class InputTextAction(BaseAction):
|
|
122
127
|
input_text: str | None = None
|
|
123
128
|
is_slider: bool = False
|
|
124
|
-
fill_or_type: Literal["fill", "type"] = "fill"
|
|
129
|
+
fill_or_type: Literal["fill", "type", "key_press"] = "fill"
|
|
125
130
|
press_enter: bool = False
|
|
126
131
|
|
|
127
132
|
@model_validator(mode="after")
|
|
@@ -217,10 +222,33 @@ class KeyPressType(str, Enum):
|
|
|
217
222
|
DELETE = "Delete"
|
|
218
223
|
BACKSPACE = "Backspace"
|
|
219
224
|
ESCAPE = "Escape"
|
|
225
|
+
ZERO = "0"
|
|
226
|
+
ONE = "1"
|
|
227
|
+
TWO = "2"
|
|
228
|
+
THREE = "3"
|
|
229
|
+
FOUR = "4"
|
|
230
|
+
FIVE = "5"
|
|
231
|
+
SIX = "6"
|
|
232
|
+
SEVEN = "7"
|
|
233
|
+
EIGHT = "8"
|
|
234
|
+
NINE = "9"
|
|
235
|
+
SLASH = "/"
|
|
236
|
+
SPACE = "Space"
|
|
220
237
|
|
|
221
238
|
|
|
222
239
|
class KeyPressAction(BaseModel):
|
|
223
|
-
type: KeyPressType
|
|
240
|
+
type: KeyPressType | Any
|
|
241
|
+
|
|
242
|
+
@model_validator(mode="after")
|
|
243
|
+
def validate_type(self):
|
|
244
|
+
if self.type is None:
|
|
245
|
+
raise ValueError("type is required")
|
|
246
|
+
return self
|
|
247
|
+
|
|
248
|
+
def replace(self, pattern: str, replacement: str):
|
|
249
|
+
if self.type:
|
|
250
|
+
self.type = self.type.replace(pattern, replacement).strip('"')
|
|
251
|
+
return self
|
|
224
252
|
|
|
225
253
|
|
|
226
254
|
class AgenticTask(BaseModel):
|
|
@@ -252,6 +280,7 @@ class InteractionAction(BaseModel):
|
|
|
252
280
|
select_option: SelectOptionAction | None = None
|
|
253
281
|
check: CheckAction | None = None
|
|
254
282
|
uncheck: UncheckAction | None = None
|
|
283
|
+
hover: HoverAction | None = None
|
|
255
284
|
download_url_as_pdf: DownloadUrlAsPdfAction | None = None
|
|
256
285
|
scroll: ScrollAction | None = None
|
|
257
286
|
upload_file: UploadFileAction | None = None
|
|
@@ -274,6 +303,7 @@ class InteractionAction(BaseModel):
|
|
|
274
303
|
"select_option": model.select_option,
|
|
275
304
|
"check": model.check,
|
|
276
305
|
"uncheck": model.uncheck,
|
|
306
|
+
"hover": model.hover,
|
|
277
307
|
"download_url_as_pdf": model.download_url_as_pdf,
|
|
278
308
|
"scroll": model.scroll,
|
|
279
309
|
"upload_file": model.upload_file,
|
|
@@ -291,7 +321,7 @@ class InteractionAction(BaseModel):
|
|
|
291
321
|
|
|
292
322
|
if len(non_null) != 1:
|
|
293
323
|
raise ValueError(
|
|
294
|
-
"Exactly one of click_element, input_text, select_option, check, uncheck, download_url_as_pdf, scroll, upload_file, go_to_url, go_back, switch_tab, close_current_tab, close_all_but_last_tab, close_tabs_until, key_press, or agentic_task must be provided"
|
|
324
|
+
"Exactly one of click_element, input_text, select_option, check, uncheck, hover, download_url_as_pdf, scroll, upload_file, go_to_url, go_back, switch_tab, close_current_tab, close_all_but_last_tab, close_tabs_until, key_press, or agentic_task must be provided"
|
|
295
325
|
)
|
|
296
326
|
|
|
297
327
|
if (
|
|
@@ -314,6 +344,8 @@ class InteractionAction(BaseModel):
|
|
|
314
344
|
self.check.replace(pattern, replacement)
|
|
315
345
|
if self.uncheck:
|
|
316
346
|
self.uncheck.replace(pattern, replacement)
|
|
347
|
+
if self.hover:
|
|
348
|
+
self.hover.replace(pattern, replacement)
|
|
317
349
|
if self.download_url_as_pdf:
|
|
318
350
|
self.download_url_as_pdf.replace(pattern, replacement)
|
|
319
351
|
if self.close_tabs_until:
|
|
@@ -326,5 +358,7 @@ class InteractionAction(BaseModel):
|
|
|
326
358
|
self.go_to_url.replace(pattern, replacement)
|
|
327
359
|
if self.upload_file:
|
|
328
360
|
self.upload_file.replace(pattern, replacement)
|
|
361
|
+
if self.key_press:
|
|
362
|
+
self.key_press.replace(pattern, replacement)
|
|
329
363
|
|
|
330
364
|
return self
|
optexity/schema/automation.py
CHANGED
|
@@ -7,7 +7,6 @@ from optexity.schema.actions.assertion_action import AssertionAction
|
|
|
7
7
|
from optexity.schema.actions.extraction_action import ExtractionAction
|
|
8
8
|
from optexity.schema.actions.interaction_action import InteractionAction
|
|
9
9
|
from optexity.schema.actions.misc_action import PythonScriptAction
|
|
10
|
-
from optexity.schema.actions.two_fa_action import TwoFAAction
|
|
11
10
|
from optexity.utils.utils import get_onepassword_value, get_totp_code
|
|
12
11
|
|
|
13
12
|
logger = logging.getLogger(__name__)
|
|
@@ -68,7 +67,6 @@ class ActionNode(BaseModel):
|
|
|
68
67
|
assertion_action: AssertionAction | None = None
|
|
69
68
|
extraction_action: ExtractionAction | None = None
|
|
70
69
|
python_script_action: PythonScriptAction | None = None
|
|
71
|
-
two_fa_action: TwoFAAction | None = None
|
|
72
70
|
before_sleep_time: float = 0.0
|
|
73
71
|
end_sleep_time: float = 5.0
|
|
74
72
|
expect_new_tab: bool = False
|
|
@@ -83,13 +81,12 @@ class ActionNode(BaseModel):
|
|
|
83
81
|
"assertion_action": model.assertion_action,
|
|
84
82
|
"extraction_action": model.extraction_action,
|
|
85
83
|
"python_script_action": model.python_script_action,
|
|
86
|
-
"two_fa_action": model.two_fa_action,
|
|
87
84
|
}
|
|
88
85
|
non_null = [k for k, v in provided.items() if v is not None]
|
|
89
86
|
|
|
90
87
|
if len(non_null) != 1:
|
|
91
88
|
raise ValueError(
|
|
92
|
-
"Exactly one of interaction_action, assertion_action, extraction_action, python_script_action
|
|
89
|
+
"Exactly one of interaction_action, assertion_action, extraction_action, python_script_action must be provided"
|
|
93
90
|
)
|
|
94
91
|
|
|
95
92
|
assert (
|
|
@@ -104,7 +101,7 @@ class ActionNode(BaseModel):
|
|
|
104
101
|
user_set = model.__pydantic_fields_set__
|
|
105
102
|
|
|
106
103
|
if "end_sleep_time" not in user_set:
|
|
107
|
-
if model.assertion_action or model.extraction_action
|
|
104
|
+
if model.assertion_action or model.extraction_action:
|
|
108
105
|
model.end_sleep_time = 0.0
|
|
109
106
|
|
|
110
107
|
if "before_sleep_time" not in user_set:
|
|
@@ -130,8 +127,6 @@ class ActionNode(BaseModel):
|
|
|
130
127
|
self.extraction_action.replace(pattern, replacement)
|
|
131
128
|
if self.python_script_action:
|
|
132
129
|
pass
|
|
133
|
-
if self.two_fa_action:
|
|
134
|
-
pass
|
|
135
130
|
|
|
136
131
|
return self
|
|
137
132
|
|
optexity/schema/task.py
CHANGED
|
@@ -60,7 +60,9 @@ class Task(BaseModel):
|
|
|
60
60
|
started_at: Optional[datetime] = None
|
|
61
61
|
completed_at: Optional[datetime] = None
|
|
62
62
|
error: Optional[str] = None
|
|
63
|
-
status: Literal[
|
|
63
|
+
status: Literal[
|
|
64
|
+
"queued", "allocated", "running", "success", "failed", "cancelled", "killed"
|
|
65
|
+
]
|
|
64
66
|
is_cloud: bool = False
|
|
65
67
|
save_directory: Path = Field(default=Path("/tmp/optexity"))
|
|
66
68
|
use_proxy: bool = False
|
|
@@ -70,6 +72,7 @@ class Task(BaseModel):
|
|
|
70
72
|
max_retries: int = 1
|
|
71
73
|
api_key: str
|
|
72
74
|
callback_url: CallbackUrl | None = None
|
|
75
|
+
is_dedicated: bool = False
|
|
73
76
|
|
|
74
77
|
class Config:
|
|
75
78
|
json_encoders = {datetime: lambda v: v.isoformat() if v is not None else None}
|
|
@@ -102,7 +105,9 @@ class Task(BaseModel):
|
|
|
102
105
|
unique_parameter_name: self.input_parameters[unique_parameter_name]
|
|
103
106
|
for unique_parameter_name in self.unique_parameter_names
|
|
104
107
|
}
|
|
105
|
-
self.dedup_key =
|
|
108
|
+
self.dedup_key = (
|
|
109
|
+
json.dumps(self.unique_parameters, sort_keys=True) + self.user_id
|
|
110
|
+
)
|
|
106
111
|
|
|
107
112
|
for a, b in [
|
|
108
113
|
(self.automation.parameters.input_parameters, self.input_parameters),
|
optexity/utils/utils.py
CHANGED
|
@@ -3,6 +3,7 @@ import logging
|
|
|
3
3
|
import os
|
|
4
4
|
from pathlib import Path
|
|
5
5
|
from typing import List, Optional
|
|
6
|
+
from urllib.parse import urlparse
|
|
6
7
|
|
|
7
8
|
import aiofiles
|
|
8
9
|
import pyotp
|
|
@@ -74,3 +75,16 @@ async def get_onepassword_value(vault_name: str, item_name: str, field_name: str
|
|
|
74
75
|
)
|
|
75
76
|
|
|
76
77
|
return str_value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def clean_url(url: str) -> str:
|
|
81
|
+
if not url.startswith(("http://", "https://")):
|
|
82
|
+
url = "http://" + url # needed for urlparse
|
|
83
|
+
|
|
84
|
+
parsed = urlparse(url)
|
|
85
|
+
domain = parsed.netloc.lower()
|
|
86
|
+
|
|
87
|
+
if domain.startswith("www."):
|
|
88
|
+
domain = domain[4:]
|
|
89
|
+
|
|
90
|
+
return domain
|
|
@@ -29,52 +29,54 @@ optexity/inference/agents/two_fa_extraction/__init__.py,sha256=47DEQpj8HBSa-_TIm
|
|
|
29
29
|
optexity/inference/agents/two_fa_extraction/prompt.py,sha256=eAqz_InZeyTnFqPMeYm-xyF7rtFWMBIF7nQW3QJnW08,1328
|
|
30
30
|
optexity/inference/agents/two_fa_extraction/two_fa_extraction.py,sha256=UcBo_Iyx6Kqas-fUZpJgos5R-t2hQ2PZUFjtHmO9Rh0,1444
|
|
31
31
|
optexity/inference/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
32
|
-
optexity/inference/core/logging.py,sha256=
|
|
32
|
+
optexity/inference/core/logging.py,sha256=0iMJq6f2mC3xphoEbQ_LqwzwkMP96B7NKst-rW4pCyM,14129
|
|
33
33
|
optexity/inference/core/run_assertion.py,sha256=cHI_A_ffe-T7XeTfTqiF3i_KIRe9ioYKSEM1rKZmq0o,2311
|
|
34
|
-
optexity/inference/core/run_automation.py,sha256=
|
|
35
|
-
optexity/inference/core/run_extraction.py,sha256=
|
|
36
|
-
optexity/inference/core/run_interaction.py,sha256=
|
|
34
|
+
optexity/inference/core/run_automation.py,sha256=Xns3ic4zByRN9Owgn7cRfh-WnN_kohm2_EZMt9EVHLw,16815
|
|
35
|
+
optexity/inference/core/run_extraction.py,sha256=JLdMIUM0syc3yffjkrDQ3SM2VID-tZW-Zoi7ZD3uURM,9089
|
|
36
|
+
optexity/inference/core/run_interaction.py,sha256=R1llSQKAQUmll4n03x34TEBgxn5Q8jC2mborJ1EAszw,9739
|
|
37
37
|
optexity/inference/core/run_python_script.py,sha256=WjnCmckZz7YmoLTGBLZeFWhhS1s_x0-kgyKYTM17JHI,547
|
|
38
|
-
optexity/inference/core/run_two_fa.py,sha256=
|
|
38
|
+
optexity/inference/core/run_two_fa.py,sha256=5B3YNtLBpQYwZ73oyD2wTL-GAZMbZdMGb1csR4ApQxc,4162
|
|
39
39
|
optexity/inference/core/interaction/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
40
|
optexity/inference/core/interaction/handle_agentic_task.py,sha256=MSURLDM-2Hw37sipa3lqUBh1NNFrGmx2DPGJsBGYTDg,2617
|
|
41
41
|
optexity/inference/core/interaction/handle_check.py,sha256=_LEA5V6h4x8fozAvPsGrFUZUaO0uU34z2R9VzGIOdio,1489
|
|
42
|
-
optexity/inference/core/interaction/handle_click.py,sha256=
|
|
43
|
-
optexity/inference/core/interaction/handle_command.py,sha256=
|
|
42
|
+
optexity/inference/core/interaction/handle_click.py,sha256=z8fvGcFY7IzMcns6Qq0br0OIW1zH66rzYtXBcAB-fP8,2425
|
|
43
|
+
optexity/inference/core/interaction/handle_command.py,sha256=OIExBFmfmWGpj_fmjTrWl7Fo5dZVr0_N_lweB5oAaW8,9175
|
|
44
|
+
optexity/inference/core/interaction/handle_hover.py,sha256=QpRwfDkK1P4mA5cbblgCAZpAN0v6K9XgRIe4zJ1AKRg,2591
|
|
44
45
|
optexity/inference/core/interaction/handle_input.py,sha256=-Yug3fBWumCp0xQt1UVX8dqCHKs_535VheMjOHMlv2Q,2317
|
|
45
|
-
optexity/inference/core/interaction/handle_keypress.py,sha256=
|
|
46
|
+
optexity/inference/core/interaction/handle_keypress.py,sha256=81MkFvYzAdMWgG2UvpZQJB42WeMzUHoxqeJZU4v_-CQ,1630
|
|
46
47
|
optexity/inference/core/interaction/handle_select.py,sha256=z-HoFqcuvsfW_fxJrmsz9RASlqef8HA7zJUErBs-9_o,3242
|
|
47
|
-
optexity/inference/core/interaction/handle_select_utils.py,sha256=
|
|
48
|
+
optexity/inference/core/interaction/handle_select_utils.py,sha256=1-o7t49QuNj5nENB_e2j_53j7mbaywC5lWMn55QPUZk,4498
|
|
48
49
|
optexity/inference/core/interaction/handle_upload.py,sha256=Yrt9KZ-rmgJ-DK5IntIXWG-jcw7mbVYs5iSrZUMkRvc,1842
|
|
49
50
|
optexity/inference/core/interaction/utils.py,sha256=_TMt0XBIIJi2K7nVQGf4PMZ-c9SCDehcMIEGPQx1GS8,2842
|
|
50
51
|
optexity/inference/core/two_factor_auth/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
51
52
|
optexity/inference/infra/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
52
|
-
optexity/inference/infra/browser.py,sha256=
|
|
53
|
+
optexity/inference/infra/browser.py,sha256=NmmJjmqpQ_AITiNU1lDxAM8AaXE7obguRwPFZsSssBs,22143
|
|
53
54
|
optexity/inference/infra/browser_extension.py,sha256=Ur5_8DrHcgRkD9uulDsVaaj01z1WcsvqHxHok4p4RoM,589
|
|
55
|
+
optexity/inference/infra/utils.py,sha256=2BLTC93oHHKC_XDp1aas2lEeupjyHkNSMbyuKrLA418,3399
|
|
54
56
|
optexity/inference/models/__init__.py,sha256=f66ea3I8ibNrkn4Uf_wYIn-2eoP-ReXds03LgaopOfA,667
|
|
55
57
|
optexity/inference/models/gemini.py,sha256=ToncY6Ft4kOgIm6qREBVsScT8FG-JCrdWsBxYOgU0is,4100
|
|
56
58
|
optexity/inference/models/human.py,sha256=K2X6Ohg7xeTWDYkJY6lOAwS9T3nX7YST-cvd8nL1Ydw,394
|
|
57
59
|
optexity/inference/models/llm_model.py,sha256=nZvcrQs4XDI07ckFUf6o-TzvyqW-I4WIbos7WEo4hz8,7211
|
|
58
60
|
optexity/schema/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
59
|
-
optexity/schema/automation.py,sha256=
|
|
61
|
+
optexity/schema/automation.py,sha256=NFy6jxjXNkv4LF5sa9TH1-_ti3QARnCnsww1dsONJfg,15921
|
|
60
62
|
optexity/schema/callback.py,sha256=MlN41A6oKG7QX01_w0tsxyAFKWnoCVsu_mjpWzPMYuE,519
|
|
61
63
|
optexity/schema/inference.py,sha256=8mP49IRU-cRxbsC4NnoGZhd5isvdocCuMVspPBOQV9o,2864
|
|
62
64
|
optexity/schema/memory.py,sha256=e3AMDAivCF_KnKbeDugqVLib8UN_6dr3ksUCiWaeIGM,3072
|
|
63
|
-
optexity/schema/task.py,sha256=
|
|
65
|
+
optexity/schema/task.py,sha256=7r7MiWp9_QaS1_vQXsFlZkpMsAjSiDFNsOl4MxUFfdE,6896
|
|
64
66
|
optexity/schema/token_usage.py,sha256=iwZjUqTrNhrEBMeeZNMWqD4qs2toKDczRVoDTOLLNso,2114
|
|
65
67
|
optexity/schema/actions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
66
68
|
optexity/schema/actions/assertion_action.py,sha256=lcD6h7phacrEoB5j-It9qP9Ym3TKb1bkv1aa8tqwG-Q,1991
|
|
67
|
-
optexity/schema/actions/extraction_action.py,sha256
|
|
68
|
-
optexity/schema/actions/interaction_action.py,sha256=
|
|
69
|
+
optexity/schema/actions/extraction_action.py,sha256=s3PeaogGJA3C34R-W2kfMk0DNvUpGS0BLAfH_xdNhYA,5082
|
|
70
|
+
optexity/schema/actions/interaction_action.py,sha256=9bm1FkBj2-iklClnR302Rir6d4jkgKjpZvTM2fAYn7E,11611
|
|
69
71
|
optexity/schema/actions/misc_action.py,sha256=VZvaVemTlCoUwSomA42EX74L81-ICDYNWqJAuWNtyyE,312
|
|
70
72
|
optexity/schema/actions/prompts.py,sha256=GZud5T2kQvQKhAXHmAnalVUP8iMcDz8be3jRp-vAInk,1772
|
|
71
|
-
optexity/schema/actions/two_fa_action.py,sha256=
|
|
73
|
+
optexity/schema/actions/two_fa_action.py,sha256=fcnuxlM3B4RguPUDw18RvMUpzLdPBRVGVC3gM_e1wLE,564
|
|
72
74
|
optexity/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
73
75
|
optexity/utils/settings.py,sha256=h6StXzYslRgZf0c8k43-kOxoa77dOgDvSOvfQUi5yI8,1864
|
|
74
|
-
optexity/utils/utils.py,sha256=
|
|
75
|
-
optexity-0.1.5.
|
|
76
|
-
optexity-0.1.5.
|
|
77
|
-
optexity-0.1.5.
|
|
78
|
-
optexity-0.1.5.
|
|
79
|
-
optexity-0.1.5.
|
|
80
|
-
optexity-0.1.5.
|
|
76
|
+
optexity/utils/utils.py,sha256=4RZs_EFJlAJukYsEb-w5XgrL3VZ3rORTNF5tGSSzqBM,2883
|
|
77
|
+
optexity-0.1.5.3.dist-info/licenses/LICENSE,sha256=WpSBqSAcwd68PmS3zRsfACJOz-u-UfTzftsEnzp4ZCY,1065
|
|
78
|
+
optexity-0.1.5.3.dist-info/METADATA,sha256=1M6zbsuB5kWyXSHw_m0kKxZdKBlP5cfYMwCo4M1laXc,9826
|
|
79
|
+
optexity-0.1.5.3.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
|
|
80
|
+
optexity-0.1.5.3.dist-info/entry_points.txt,sha256=hcn77ooRr6a_N8fo0vij3Fpo6waqc9ijpaScQ7Kj35k,47
|
|
81
|
+
optexity-0.1.5.3.dist-info/top_level.txt,sha256=OZEtBX8IabC8EnBrNW98z7NzdGQsjFhHleSthhjjEMM,9
|
|
82
|
+
optexity-0.1.5.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|