optexity 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl

optexity/inference/core/interaction/handle_command.py

@@ -0,0 +1,261 @@
+import asyncio
+import logging
+
+from playwright.async_api import Locator
+
+from optexity.exceptions import AssertLocatorPresenceException
+from optexity.inference.core.interaction.handle_select_utils import (
+    SelectOptionValue,
+    smart_select,
+)
+from optexity.inference.core.interaction.utils import handle_download
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import (
+    CheckAction,
+    ClickElementAction,
+    InputTextAction,
+    SelectOptionAction,
+    UncheckAction,
+    UploadFileAction,
+)
+from optexity.schema.memory import BrowserState, Memory
+from optexity.schema.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+async def command_based_action_with_retry(
+    action: (
+        ClickElementAction
+        | InputTextAction
+        | SelectOptionAction
+        | CheckAction
+        | UploadFileAction
+        | UncheckAction
+    ),
+    browser: Browser,
+    memory: Memory,
+    task: Task,
+    max_tries: int,
+    max_timeout_seconds_per_try: float,
+):
+
+    if action.command is None or action.skip_command:
+        return
+
+    last_error = None
+
+    logger.debug(f"Executing command-based action: {action.__class__.__name__}")
+
+    for try_index in range(max_tries):
+        last_error = None
+        try:
+            # https://playwright.dev/docs/actionability
+            locator = await browser.get_locator_from_command(action.command)
+            if try_index == 0:
+                try:
+                    await locator.wait_for(
+                        state="visible", timeout=max_timeout_seconds_per_try * 1000
+                    )
+                except Exception as e:
+                    pass
+            is_visible = await locator.is_visible()
+
+            if is_visible:
+                browser_state_summary = await browser.get_browser_state_summary()
+                memory.browser_states[-1] = BrowserState(
+                    url=browser_state_summary.url,
+                    screenshot=browser_state_summary.screenshot,
+                    title=browser_state_summary.title,
+                    axtree=browser_state_summary.dom_state.llm_representation(),
+                )
+
+                if isinstance(action, ClickElementAction):
+                    await click_locator(
+                        action,
+                        locator,
+                        browser,
+                        memory,
+                        task,
+                        max_timeout_seconds_per_try,
+                    )
+                elif isinstance(action, InputTextAction):
+                    await input_text_locator(
+                        action, locator, max_timeout_seconds_per_try
+                    )
+                elif isinstance(action, SelectOptionAction):
+                    await select_option_locator(
+                        action,
+                        locator,
+                        browser,
+                        memory,
+                        task,
+                        max_timeout_seconds_per_try,
+                    )
+                elif isinstance(action, CheckAction):
+                    await check_locator(
+                        action, locator, max_timeout_seconds_per_try, browser
+                    )
+                elif isinstance(action, UncheckAction):
+                    await uncheck_locator(
+                        action, locator, max_timeout_seconds_per_try, browser
+                    )
+                elif isinstance(action, UploadFileAction):
+                    await upload_file_locator(action, locator)
+                logger.debug(
+                    f"{action.__class__.__name__} successful on try {try_index + 1}"
+                )
+                return
+            else:
+                await asyncio.sleep(max_timeout_seconds_per_try)
+                last_error = f"error: locator not visible"
+        except Exception as e:
+            last_error = f"error: {e}"
+            await asyncio.sleep(max_timeout_seconds_per_try)
+
+    if last_error is None:
+        last_error = "error in executing command"
+    logger.debug(
+        f"{action.__class__.__name__} failed after {max_tries} tries: {last_error}"
+    )
+
+    if last_error and action.assert_locator_presence:
+        logger.debug(
+            f"Error in {action.__class__.__name__} with assert_locator_presence: {action.__class__.__name__}: {last_error}"
+        )
+        raise AssertLocatorPresenceException(
+            message=f"Error in {action.__class__.__name__} with assert_locator_presence: {action.__class__.__name__}",
+            original_error=last_error,
+            command=action.command,
+        )
+    return last_error
+
+
+async def click_locator(
+    click_element_action: ClickElementAction,
+    locator: Locator,
+    browser: Browser,
+    memory: Memory,
+    task: Task,
+    max_timeout_seconds_per_try: float,
+):
+    async def _actual_click():
+
+        if click_element_action.double_click:
+            await locator.dblclick(
+                no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000
+            )
+        else:
+            await locator.click(
+                no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000
+            )
+
+    if click_element_action.expect_download:
+        await handle_download(
+            _actual_click, memory, browser, task, click_element_action.download_filename
+        )
+    else:
+        await _actual_click()
+
+
+async def input_text_locator(
+    input_text_action: InputTextAction,
+    locator: Locator,
+    max_timeout_seconds_per_try: float,
+):
+
+    if input_text_action.fill_or_type == "fill":
+        await locator.fill(
+            input_text_action.input_text,
+            no_wait_after=True,
+            timeout=max_timeout_seconds_per_try * 1000,
+        )
+    else:
+        await locator.type(
+            input_text_action.input_text,
+            no_wait_after=True,
+            timeout=max_timeout_seconds_per_try * 1000,
+        )
+
+    if input_text_action.press_enter:
+        await locator.press("Enter")
+
+
+async def check_locator(
+    action: CheckAction,
+    locator: Locator,
+    max_timeout_seconds_per_try: float,
+    browser: Browser,
+):
+    await locator.uncheck(
+        no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000
+    )
+    await asyncio.sleep(1)
+    locator = await browser.get_locator_from_command(action.command)
+    await locator.check(no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000)
+
+
+async def uncheck_locator(
+    action: UncheckAction,
+    locator: Locator,
+    max_timeout_seconds_per_try: float,
+    browser: Browser,
+):
+    await locator.check(no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000)
+    await asyncio.sleep(1)
+    locator = await browser.get_locator_from_command(action.command)
+    await locator.uncheck(
+        no_wait_after=True, timeout=max_timeout_seconds_per_try * 1000
+    )
+
+
+async def upload_file_locator(upload_file_action: UploadFileAction, locator: Locator):
+    await locator.set_input_files(upload_file_action.file_path)
+
+
+async def select_option_locator(
+    select_option_action: SelectOptionAction,
+    locator: Locator,
+    browser: Browser,
+    memory: Memory,
+    task: Task,
+    max_timeout_seconds_per_try: float,
+):
+    async def _actual_select_option():
+        options: list[dict[str, str]] = await locator.evaluate(
+            """
+        sel => Array.from(sel.options).map(o => ({
+            value: o.value,
+            label: o.label || o.textContent
+        }))
+    """
+        )
+
+        select_option_values = [
+            SelectOptionValue(value=o["value"], label=o["label"]) for o in options
+        ]
+
+        matched_values = await smart_select(
+            select_option_values, options, select_option_action.select_values, memory
+        )
+
+        logger.debug(
+            f"Matched values for {select_option_action.command}: {matched_values}"
+        )
+
+        await locator.select_option(
+            matched_values,
+            no_wait_after=True,
+            timeout=max_timeout_seconds_per_try * 1000,
+        )
+
+    if select_option_action.expect_download:
+        await handle_download(
+            _actual_select_option,
+            memory,
+            browser,
+            task,
+            select_option_action.download_filename,
+        )
+    else:
+        await _actual_select_option()

optexity/inference/core/interaction/handle_input.py

@@ -0,0 +1,76 @@
+import logging
+import re
+
+from optexity.inference.core.interaction.handle_command import (
+    command_based_action_with_retry,
+)
+from optexity.inference.core.interaction.utils import get_index_from_prompt
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import InputTextAction
+from optexity.schema.memory import Memory
+from optexity.schema.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+async def handle_input_text(
+    input_text_action: InputTextAction,
+    task: Task,
+    memory: Memory,
+    browser: Browser,
+    max_timeout_seconds_per_try: float,
+    max_tries: int,
+):
+
+    # {some english chars [0]}
+    INT_INDEX_PATTERN = re.compile(r"^\{([A-Za-z_][A-Za-z0-9_]*)\[(\d+)\]\}$")
+
+    if INT_INDEX_PATTERN.match(input_text_action.input_text) is not None:
+        logger.debug(
+            "Skipping input text because input variable was not present for this step"
+        )
+        return
+
+    if input_text_action.command and not input_text_action.skip_command:
+        last_error = await command_based_action_with_retry(
+            input_text_action,
+            browser,
+            memory,
+            task,
+            max_tries,
+            max_timeout_seconds_per_try,
+        )
+
+        if last_error is None:
+            return
+
+    if not input_text_action.skip_prompt:
+        logger.debug(
+            f"Executing prompt-based action: {input_text_action.__class__.__name__}"
+        )
+        await input_text_index(input_text_action, browser, memory)
+
+
+async def input_text_index(
+    input_text_action: InputTextAction, browser: Browser, memory: Memory
+):
+    try:
+        index = await get_index_from_prompt(
+            memory, input_text_action.prompt_instructions, browser
+        )
+        if index is None:
+            return
+
+        action_model = browser.backend_agent.ActionModel(
+            **{
+                "input": {
+                    "index": int(index),
+                    "text": input_text_action.input_text,
+                    "clear": True,
+                }
+            }
+        )
+        await browser.backend_agent.multi_act([action_model])
+    except Exception as e:
+        logger.error(f"Error in input_text_index: {e}")
+        return

optexity/inference/core/interaction/handle_keypress.py

@@ -0,0 +1,16 @@
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import KeyPressAction, KeyPressType
+from optexity.schema.memory import Memory
+
+
+async def handle_key_press(
+    keypress_action: KeyPressAction,
+    memory: Memory,
+    browser: Browser,
+):
+    page = await browser.get_current_page()
+    if page is None:
+        return
+
+    if keypress_action.type == KeyPressType.ENTER:
+        await page.keyboard.press("Enter")

optexity/inference/core/interaction/handle_select.py

@@ -0,0 +1,109 @@
+import logging
+
+from browser_use.dom.serializer.serializer import DOMTreeSerializer
+
+from optexity.inference.core.interaction.handle_command import (
+    command_based_action_with_retry,
+)
+from optexity.inference.core.interaction.handle_select_utils import (
+    SelectOptionValue,
+    smart_select,
+)
+from optexity.inference.core.interaction.utils import (
+    get_index_from_prompt,
+    handle_download,
+)
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import SelectOptionAction
+from optexity.schema.memory import Memory
+from optexity.schema.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+async def handle_select_option(
+    select_option_action: SelectOptionAction,
+    task: Task,
+    memory: Memory,
+    browser: Browser,
+    max_timeout_seconds_per_try: float,
+    max_tries: int,
+):
+
+    if select_option_action.command and not select_option_action.skip_command:
+        last_error = await command_based_action_with_retry(
+            select_option_action,
+            browser,
+            memory,
+            task,
+            max_tries,
+            max_timeout_seconds_per_try,
+        )
+
+        if last_error is None:
+            return
+
+    if not select_option_action.skip_prompt:
+        logger.debug(
+            f"Executing prompt-based action: {select_option_action.__class__.__name__}"
+        )
+        await select_option_index(select_option_action, browser, memory, task)
+
+
+async def select_option_index(
+    select_option_action: SelectOptionAction,
+    browser: Browser,
+    memory: Memory,
+    task: Task,
+):
+    ## TODO either perfect text match or agenic select value prediction
+    try:
+
+        index = await get_index_from_prompt(
+            memory, select_option_action.prompt_instructions, browser
+        )
+        if index is None:
+            return
+
+        node = await browser.backend_agent.browser_session.get_element_by_index(index)
+        if node is None:
+            return
+
+        select_option_values = DOMTreeSerializer(node)._extract_select_options(node)
+        if select_option_values is None:
+            return
+
+        all_options = select_option_values["all_options"]
+
+        all_options = [
+            SelectOptionValue(value=o["value"], label=o["text"]) for o in all_options
+        ]
+
+        matched_values = await smart_select(
+            all_options, select_option_action.select_values, memory
+        )
+
+        async def _actual_select_option():
+            action_model = browser.backend_agent.ActionModel(
+                **{
+                    "select_dropdown": {
+                        "index": int(index),
+                        "text": matched_values[0],
+                    }
+                }
+            )
+            await browser.backend_agent.multi_act([action_model])
+
+        if select_option_action.expect_download:
+            await handle_download(
+                _actual_select_option,
+                memory,
+                browser,
+                task,
+                select_option_action.download_filename,
+            )
+        else:
+            await _actual_select_option()
+    except Exception as e:
+        logger.error(f"Error in select_option_index: {e}")
+        return

optexity/inference/core/interaction/handle_select_utils.py

@@ -0,0 +1,132 @@
+import logging
+import re
+
+from pydantic import BaseModel
+
+from optexity.inference.agents.select_value_prediction.select_value_prediction import (
+    SelectValuePredictionAgent,
+)
+from optexity.schema.actions.interaction_action import Locator
+from optexity.schema.memory import Memory
+
+logger = logging.getLogger(__name__)
+select_value_prediction_agent = SelectValuePredictionAgent()
+
+
+class SelectOptionValue(BaseModel):
+    value: str
+    label: str
+
+
+def llm_select_match(
+    options: list[SelectOptionValue], patterns: list[str], memory: Memory
+) -> list[str]:
+    final_prompt, response, token_usage = (
+        select_value_prediction_agent.predict_select_value(
+            [o.model_dump() for o in options], patterns
+        )
+    )
+    memory.token_usage += token_usage
+    memory.browser_states[-1].final_prompt = final_prompt
+    memory.browser_states[-1].llm_response = response.model_dump()
+
+    matched_values = response.matched_values
+
+    all_values = [o.value for o in options]
+
+    final_matched_values = []
+    for value in matched_values:
+        if value in all_values:
+            final_matched_values.append(value)
+
+    return final_matched_values
+
+
+def score_match(pat: str, val: str) -> int:
+    # higher is better
+    if pat == val:
+        return 100
+    if val.startswith(pat):
+        return 80
+    if pat in val:
+        return 60
+    return 0
+
+
+async def smart_select(
+    options: list[SelectOptionValue], patterns: list[str], memory: Memory
+):
+    # Get all options from the <select>
+
+    matched_values = []
+
+    for p in patterns:
+        # If pattern contains regex characters, treat as regex
+        is_regex = p.startswith("^") or p.endswith("$") or ".*" in p
+
+        ## Check if reggex pattern and then try finding the option by value and label
+        if is_regex:
+            regex = re.compile(p)
+            for opt in options:
+                if regex.search(opt.value) or regex.search(opt.label):
+                    matched_values.append(opt.value)
+        else:
+            # try exact match
+            for opt in options:
+                if opt.value == p or opt.label == p:
+                    matched_values.append(opt.value)
+
+    if len(matched_values) == 0:
+        ## If no matches, check if all values are unique and try score matching of values
+
+        processed_values = [
+            (v.value.lower().replace(" ", ""), v.value) for v in options
+        ]
+
+        if len(processed_values) == len(set(processed_values)):
+            for p in patterns:
+                processed_pattern = p.lower().replace(" ", "")
+
+                best_score = 0
+                best_value = None
+
+                for processed_value, value in processed_values:
+                    score = score_match(processed_pattern, processed_value)
+                    if score > best_score:
+                        best_score = score
+                        best_value = value
+
+                if best_value is not None and best_score > 0:
+                    matched_values.append(best_value)
+
+    if len(matched_values) == 0:
+        processed_labels = [
+            (v.label.lower().replace(" ", ""), v.label) for v in options
+        ]
+
+        if len(processed_labels) == len(set(processed_labels)):
+            for p in patterns:
+                processed_pattern = p.lower().replace(" ", "")
+
+                best_score = 0
+                best_label = None
+                best_value = None
+
+                for opt in options:
+                    processed_label = opt.label.lower().replace(" ", "")
+                    score = score_match(processed_pattern, processed_label)
+                    if score > best_score:
+                        best_score = score
+                        best_label = opt.label
+                        best_value = opt.value
+
+                if best_label is not None and best_score > 0:
+                    matched_values.append(best_value)
+
+    if len(matched_values) == 0:
+        matched_values = llm_select_match(options, patterns, memory)
+
+    if len(matched_values) == 0:
+        matched_values = patterns
+
+    return matched_values

optexity/inference/core/interaction/handle_upload.py

@@ -0,0 +1,59 @@
+import logging
+
+from optexity.inference.core.interaction.handle_command import (
+    command_based_action_with_retry,
+)
+from optexity.inference.core.interaction.utils import get_index_from_prompt
+from optexity.inference.infra.browser import Browser
+from optexity.schema.actions.interaction_action import UploadFileAction
+from optexity.schema.memory import Memory
+from optexity.schema.task import Task
+
+logger = logging.getLogger(__name__)
+
+
+async def handle_upload_file(
+    upload_file_action: UploadFileAction,
+    task: Task,
+    memory: Memory,
+    browser: Browser,
+    max_timeout_seconds_per_try: float,
+    max_tries: int,
+):
+    if upload_file_action.command and not upload_file_action.skip_command:
+        last_error = await command_based_action_with_retry(
+            upload_file_action,
+            browser,
+            memory,
+            task,
+            max_tries,
+            max_timeout_seconds_per_try,
+        )
+        if last_error is None:
+            return
+
+    if not upload_file_action.skip_prompt:
+        logger.debug(
+            f"Executing prompt-based action: {upload_file_action.__class__.__name__}"
+        )
+        await upload_file_index(upload_file_action, browser, memory)
+
+
+async def upload_file_index(
+    upload_file_action: UploadFileAction, browser: Browser, memory: Memory
+):
+
+    try:
+        index = await get_index_from_prompt(
+            memory, upload_file_action.prompt_instructions, browser
+        )
+        if index is None:
+            return
+
+        action_model = browser.backend_agent.ActionModel(
+            **{"upload_file": {"index": index, "path": upload_file_action.file_path}}
+        )
+        await browser.backend_agent.multi_act([action_model])
+    except Exception as e:
+        logger.error(f"Error in upload_file_index: {e}")
+        return