optexity 0.1.5.1__tar.gz → 0.1.5.3__tar.gz

{optexity-0.1.5.1 → optexity-0.1.5.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: optexity
-Version: 0.1.5.1
+Version: 0.1.5.3
 Summary: Optexity is a platform for building and running browser and computer agents.
 Author-email: Optexity <founders@optexity.com>
 Requires-Python: >=3.11

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/interaction/handle_click.py

@@ -59,8 +59,11 @@ async def click_element_index(
             return
 
         async def _actual_click_element():
+            print(
+                f"Clicking element with index: {index} and button: {click_element_action.button}"
+            )
             action_model = browser.backend_agent.ActionModel(
-                **{"click": {"index": index}}
+                **{"click": {"index": index, "button": click_element_action.button}}
             )
             await browser.backend_agent.multi_act([action_model])
 

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/interaction/handle_command.py

@@ -13,6 +13,7 @@ from optexity.inference.infra.browser import Browser
 from optexity.schema.actions.interaction_action import (
     CheckAction,
     ClickElementAction,
+    HoverAction,
     InputTextAction,
     SelectOptionAction,
     UncheckAction,
@@ -32,6 +33,7 @@ async def command_based_action_with_retry(
         | CheckAction
         | UploadFileAction
         | UncheckAction
+        | HoverAction
     ),
     browser: Browser,
     memory: Memory,
@@ -83,7 +85,7 @@ async def command_based_action_with_retry(
                     )
                 elif isinstance(action, InputTextAction):
                     await input_text_locator(
-                        action, locator, max_timeout_seconds_per_try
+                        action, locator, browser, max_timeout_seconds_per_try
                     )
                 elif isinstance(action, SelectOptionAction):
                     await select_option_locator(
@@ -102,6 +104,8 @@ async def command_based_action_with_retry(
                     await uncheck_locator(
                         action, locator, max_timeout_seconds_per_try, browser
                     )
+                elif isinstance(action, HoverAction):
+                    await hover_locator(locator, max_timeout_seconds_per_try)
                 elif isinstance(action, UploadFileAction):
                     await upload_file_locator(action, locator)
                 logger.debug(
@@ -149,7 +153,9 @@ async def click_locator(
             )
         else:
             await locator.click(
-                no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000
+                button=click_element_action.button,
+                no_wait_after=True,
+                timeout=max_timeout_seconds_per_try * 1000,
             )
 
     if click_element_action.expect_download:
@@ -163,6 +169,7 @@ async def click_locator(
 async def input_text_locator(
     input_text_action: InputTextAction,
     locator: Locator,
+    browser: Browser,
     max_timeout_seconds_per_try: float,
 ):
 
@@ -172,12 +179,19 @@ async def input_text_locator(
             no_wait_after=True,
             timeout=max_timeout_seconds_per_try * 1000,
         )
-    else:
+    elif input_text_action.fill_or_type == "type":
         await locator.type(
             input_text_action.input_text,
             no_wait_after=True,
             timeout=max_timeout_seconds_per_try * 1000,
         )
+    else:
+        page = await browser.get_current_page()
+        if page is None:
+            return
+        for char in input_text_action.input_text:
+            await page.keyboard.press(char)
+            await asyncio.sleep(0.1)
 
     if input_text_action.press_enter:
         await locator.press("Enter")
@@ -211,6 +225,13 @@ async def uncheck_locator(
     )
 
 
+async def hover_locator(
+    locator: Locator,
+    max_timeout_seconds_per_try: float,
+):
+    await locator.hover(no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000)
+
+
 async def upload_file_locator(upload_file_action: UploadFileAction, locator: Locator):
     await locator.set_input_files(upload_file_action.file_path)
 

optexity-0.1.5.3/optexity/inference/core/interaction/handle_hover.py

@@ -0,0 +1,83 @@
+import logging
+
+from optexity.inference.core.interaction.handle_command import (
+    command_based_action_with_retry,
+)
+from optexity.inference.core.interaction.utils import get_index_from_prompt
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import HoverAction
+from optexity.schema.memory import Memory
+from optexity.schema.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+async def handle_hover_element(
+    hover_element_action: HoverAction,
+    task: Task,
+    memory: Memory,
+    browser: Browser,
+    max_timeout_seconds_per_try: float,
+    max_tries: int,
+):
+
+    if hover_element_action.command and not hover_element_action.skip_command:
+        last_error = await command_based_action_with_retry(
+            hover_element_action,
+            browser,
+            memory,
+            task,
+            max_tries,
+            max_timeout_seconds_per_try,
+        )
+
+        if last_error is None:
+            return
+
+    if not hover_element_action.skip_prompt:
+        logger.debug(
+            f"Executing prompt-based action: {hover_element_action.__class__.__name__}"
+        )
+        await hover_element_index(hover_element_action, browser, memory, task)
+
+
+async def hover_element_index(
+    hover_element_action: HoverAction,
+    browser: Browser,
+    memory: Memory,
+    task: Task,
+):
+
+    try:
+        index = await get_index_from_prompt(
+            memory, hover_element_action.prompt_instructions, browser, task
+        )
+        if index is None:
+            return
+
+        print(f"Hovering element with index: {index}")
+
+        async def _actual_hover_element():
+            try:
+                action_model = browser.backend_agent.ActionModel(
+                    **{"hover": {"index": index}}
+                )
+                await browser.backend_agent.multi_act([action_model])
+            except Exception as e:
+                logger.error(f"Error in hover_element_index: {e} trying right click")
+                node = await browser.backend_agent.browser_session.get_element_by_index(
+                    index
+                )
+                if node is None:
+                    return
+
+                backend_page = (
+                    await browser.backend_agent.browser_session.get_current_page()
+                )
+                element = await backend_page.get_element(node.backend_node_id)
+                await element.click(button="right")
+
+        await _actual_hover_element()
+    except Exception as e:
+        logger.error(f"Error in hover_element_index: {e}")
+        return

optexity-0.1.5.3/optexity/inference/core/interaction/handle_keypress.py

@@ -0,0 +1,42 @@
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import KeyPressAction, KeyPressType
+from optexity.schema.memory import Memory
+
+
+async def handle_key_press(
+    keypress_action: KeyPressAction,
+    memory: Memory,
+    browser: Browser,
+):
+    page = await browser.get_current_page()
+    if page is None:
+        return
+
+    if keypress_action.type == KeyPressType.ENTER:
+        await page.keyboard.press("Enter")
+    if keypress_action.type == KeyPressType.TAB:
+        await page.keyboard.press("Tab")
+    if keypress_action.type == KeyPressType.ZERO:
+        await page.keyboard.press("0")
+    if keypress_action.type == KeyPressType.ONE:
+        await page.keyboard.press("1")
+    if keypress_action.type == KeyPressType.TWO:
+        await page.keyboard.press("2")
+    if keypress_action.type == KeyPressType.THREE:
+        await page.keyboard.press("3")
+    if keypress_action.type == KeyPressType.FOUR:
+        await page.keyboard.press("4")
+    if keypress_action.type == KeyPressType.FIVE:
+        await page.keyboard.press("5")
+    if keypress_action.type == KeyPressType.SIX:
+        await page.keyboard.press("6")
+    if keypress_action.type == KeyPressType.SEVEN:
+        await page.keyboard.press("7")
+    if keypress_action.type == KeyPressType.EIGHT:
+        await page.keyboard.press("8")
+    if keypress_action.type == KeyPressType.NINE:
+        await page.keyboard.press("9")
+    if keypress_action.type == KeyPressType.SLASH:
+        await page.keyboard.press("/")
+    if keypress_action.type == KeyPressType.SPACE:
+        await page.keyboard.press("Space")

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/interaction/handle_select_utils.py

@@ -57,9 +57,19 @@ async def smart_select(
     options: list[SelectOptionValue], patterns: list[str], memory: Memory
 ):
     # Get all options from the <select>
-
+    ## TODO: remove this once we have a better way to handle select one
     matched_values = []
 
+    if len(options) == 0:
+        return []
+    if len(options) == 1:
+        return [options[0].value]
+    if len(options) == 2 and "Select One" in [o.value for o in options]:
+        if options[0].value == "Select One":
+            return [options[1].value]
+        else:
+            return [options[0].value]
+
     for p in patterns:
         # If pattern contains regex characters, treat as regex
         is_regex = p.startswith("^") or p.endswith("$") or ".*" in p

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/logging.py

@@ -364,7 +364,14 @@ async def save_latest_memory_state_locally(
             await f.write(json.dumps(task.input_parameters, indent=4))
 
         async with aiofiles.open(step_directory / "secure_parameters.json", "w") as f:
-            await f.write(json.dumps(task.secure_parameters, indent=4))
+            secure_parameters = {
+                key: [
+                    a.model_dump(exclude_none=True, exclude_defaults=True)
+                    for a in value
+                ]
+                for key, value in task.secure_parameters.items()
+            }
+            await f.write(json.dumps(secure_parameters, indent=4))
 
         async with aiofiles.open(step_directory / "generated_variables.json", "w") as f:
             await f.write(json.dumps(memory.variables.generated_variables, indent=4))

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/run_automation.py

@@ -25,7 +25,6 @@ from optexity.inference.core.run_interaction import (
     run_interaction_action,
 )
 from optexity.inference.core.run_python_script import run_python_script_action
-from optexity.inference.core.run_two_fa import run_two_fa_action
 from optexity.inference.infra.browser import Browser
 from optexity.schema.actions.interaction_action import DownloadUrlAsPdfAction
 from optexity.schema.automation import ActionNode, ForLoopNode, IfElseNode
@@ -55,11 +54,14 @@ async def run_automation(task: Task, child_process_id: int):
     logger.info(f"Task {task.task_id} started running")
     memory = None
     browser = None
+
     try:
         await start_task_in_server(task)
         memory = Memory()
+
         browser = Browser(
             memory=memory,
+            user_data_dir=f"/tmp/userdata_{task.task_id}",
             headless=False,
             channel=task.automation.browser_channel,
             debug_port=9222 + child_process_id,
@@ -67,9 +69,12 @@ async def run_automation(task: Task, child_process_id: int):
             proxy_session_id=task.proxy_session_id(
                 settings.PROXY_PROVIDER if task.use_proxy else None
             ),
+            is_dedicated=task.is_dedicated,
         )
         await browser.start()
 
+        browser.memory = memory
+
         automation = task.automation
 
         memory.automation_state.step_index = -1
@@ -274,8 +279,6 @@ async def run_action_node(
             await run_extraction_action(
                 action_node.extraction_action, memory, browser, task
             )
-        elif action_node.two_fa_action:
-            await run_two_fa_action(action_node.two_fa_action, memory)
         elif action_node.python_script_action:
             await run_python_script_action(
                 action_node.python_script_action, memory, browser

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/run_extraction.py

@@ -4,12 +4,14 @@ import traceback
 import aiofiles
 import httpx
 
+from optexity.inference.core.run_two_fa import run_two_fa_action
 from optexity.inference.infra.browser import Browser
 from optexity.inference.models import GeminiModels, get_llm_model
 from optexity.schema.actions.extraction_action import (
     ExtractionAction,
     LLMExtraction,
     NetworkCallExtraction,
+    PythonScriptExtraction,
     ScreenshotExtraction,
     StateExtraction,
 )
@@ -51,6 +53,14 @@ async def run_extraction_action(
             task,
             extraction_action.unique_identifier,
         )
+    elif extraction_action.python_script:
+        await handle_python_script_extraction(
+            extraction_action.python_script,
+            memory,
+            browser,
+            task,
+            extraction_action.unique_identifier,
+        )
     elif extraction_action.screenshot:
         await handle_screenshot_extraction(
             extraction_action.screenshot,
@@ -65,6 +75,8 @@ async def run_extraction_action(
             browser,
             extraction_action.unique_identifier,
         )
+    elif extraction_action.two_fa_action:
+        await run_two_fa_action(extraction_action.two_fa_action, memory)
 
 
 async def handle_state_extraction(
@@ -225,6 +237,31 @@ async def handle_network_call_extraction(
             )
 
 
+async def handle_python_script_extraction(
+    python_script_extraction: PythonScriptExtraction,
+    memory: Memory,
+    browser: Browser,
+    task: Task,
+    unique_identifier: str | None = None,
+):
+    local_vars = {}
+    exec(python_script_extraction.script, {}, local_vars)
+    code_fn = local_vars["code_fn"]
+    axtree = memory.browser_states[-1].axtree
+    result = await code_fn(axtree)
+    if result is not None:
+        memory.variables.output_data.append(
+            OutputData(
+                unique_identifier=unique_identifier,
+                json_data=result,
+            )
+        )
+    else:
+        logger.warning(
+            f"No result from Python script extraction: {python_script_extraction.script}"
+        )
+
+
 async def download_request(
     network_call: NetworkRequest, download_filename: str, task: Task, memory: Memory
 ):

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/run_interaction.py

@@ -12,6 +12,7 @@ from optexity.inference.core.interaction.handle_check import (
     handle_uncheck_element,
 )
 from optexity.inference.core.interaction.handle_click import handle_click_element
+from optexity.inference.core.interaction.handle_hover import handle_hover_element
 from optexity.inference.core.interaction.handle_input import handle_input_text
 from optexity.inference.core.interaction.handle_keypress import handle_key_press
 from optexity.inference.core.interaction.handle_select import handle_select_option
@@ -95,6 +96,15 @@ async def run_interaction_action(
                 interaction_action.max_timeout_seconds_per_try,
                 interaction_action.max_tries,
             )
+        elif interaction_action.hover:
+            await handle_hover_element(
+                interaction_action.hover,
+                task,
+                memory,
+                browser,
+                interaction_action.max_timeout_seconds_per_try,
+                interaction_action.max_tries,
+            )
         elif interaction_action.go_back:
             await handle_go_back(interaction_action.go_back, memory, browser)
         elif interaction_action.download_url_as_pdf:

{optexity-0.1.5.1 → optexity-0.1.5.3}/optexity/inference/core/run_two_fa.py

@@ -33,6 +33,7 @@ async def run_two_fa_action(two_fa_action: TwoFAAction, memory: Memory):
 
     elapsed = 0
     messages = None
+    code = None
 
     while elapsed < two_fa_action.max_wait_time:
         messages = await fetch_messages(
@@ -109,12 +110,16 @@ async def fetch_messages(
             end_2fa_time=end_2fa_time,
         )
 
-    async with httpx.AsyncClient(timeout=30.0) as client:
+    try:
+        async with httpx.AsyncClient(timeout=30.0) as client:
 
-        response = await client.post(
-            url, json=body.model_dump(mode="json"), headers=headers
-        )
-        response.raise_for_status()
-        response_data = FetchMessagesResponse.model_validate(response.json())
+            response = await client.post(
+                url, json=body.model_dump(mode="json"), headers=headers
+            )
+            response.raise_for_status()
+            response_data = FetchMessagesResponse.model_validate(response.json())
 
-        return response_data.messages
+            return response_data.messages
+    except Exception as e:
+        logger.error(f"Error fetching messages: {e}")
+        return []