optexity 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

optexity/examples/supabase_login.py

@@ -0,0 +1,75 @@
+from optexity.schema.automation import Automation
+
+description = "Supabase Login Example"
+endpoint_name = "supabase_login"
+automation_json = {
+    "url": "https://supabase.com",
+    "parameters": {
+        "input_parameters": {},
+        "secure_parameters": {
+            "username": [
+                {
+                    "onepassword": {
+                        "vault_name": "optexity_automation",
+                        "item_name": "supabase",
+                        "field_name": "username",
+                    }
+                }
+            ],
+            "password": [
+                {
+                    "onepassword": {
+                        "vault_name": "optexity_automation",
+                        "item_name": "supabase",
+                        "field_name": "password",
+                    }
+                }
+            ],
+        },
+        "generated_parameters": {},
+    },
+    "nodes": [
+        {
+            "type": "action_node",
+            "interaction_action": {
+                "click_element": {
+                    "command": 'get_by_role("link", name="Sign in")',
+                    "prompt_instructions": "Click the Sign in link",
+                }
+            },
+            "end_sleep_time": 1.0,
+        },
+        {
+            "type": "action_node",
+            "interaction_action": {
+                "input_text": {
+                    "command": 'get_by_role("textbox", name="Email")',
+                    "prompt_instructions": "Enter the email",
+                    "input_text": "{username[0]}",
+                }
+            },
+            "end_sleep_time": 1.0,
+        },
+        {
+            "type": "action_node",
+            "interaction_action": {
+                "input_text": {
+                    "command": 'get_by_role("textbox", name="Password")',
+                    "prompt_instructions": "Enter the password",
+                    "input_text": "{password[0]}",
+                    "press_enter": True,
+                }
+            },
+            "end_sleep_time": 1.0,
+        },
+        {
+            "type": "action_node",
+            "assertion_action": {
+                "llm": {"extraction_instructions": "Check if the login was successful"}
+            },
+            "end_sleep_time": 0.0,
+        },
+    ],
+}
+
+automation = Automation.model_validate(automation_json)

optexity/inference/agents/error_handler/error_handler.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Literal
+
+from pydantic import BaseModel
+
+from optexity.inference.agents.error_handler.prompt import system_prompt
+from optexity.inference.models import GeminiModels, get_llm_model
+from optexity.schema.token_usage import TokenUsage
+
+logger = logging.getLogger(__name__)
+
+
+class ErrorHandlerOutput(BaseModel):
+    error_type: Literal["website_not_loaded", "overlay_popup_blocking", "fatal_error"]
+    detailed_reason: str
+
+
+class ErrorHandlerAgent:
+    def __init__(self):
+        self.model = get_llm_model(GeminiModels.GEMINI_2_5_FLASH, True)
+
+    def classify_error(
+        self, command: str, screenshot: str
+    ) -> tuple[str, ErrorHandlerOutput, TokenUsage]:
+
+        final_prompt = f"""
+        [INPUT]
+        Command: {command}
+        [/INPUT]
+        """
+
+        response, token_usage = self.model.get_model_response_with_structured_output(
+            prompt=final_prompt,
+            response_schema=ErrorHandlerOutput,
+            screenshot=screenshot,
+            system_instruction=system_prompt,
+        )
+
+        return final_prompt, response, token_usage

optexity/inference/agents/error_handler/prompt.py

@@ -0,0 +1,60 @@
+system_prompt = """
+You are an expert error classification agent for an unattended (no human-in-the-loop) Playwright browser automation system.
+
+Your single task is to analyze the provided **Goal (playwright command), and Screenshot** to classify an error into one of three categories and provide a clear reason.
+
+This automation **cannot** ask a human for help; if the script is logically stuck and cannot proceed without new data or a code change, it is a **fatal error**.
+
+You MUST provide your output in a JSON format:
+
+```json
+{
+    "error_type": "website_not_loaded" | "overlay_popup_blocking" | "fatal_error",
+    "detailed_reason": "A summary of the error reason"
+}
+```
+
+-----
+
+### Error Classification Rules
+
+Here are the definitions for each `error_type`:
+
+**1. `website_not_loaded`**
+
+  * **Description:** This is a **transient error**. The page or a specific element is not *yet* available, but it is expected to appear.
+  * **Cause:** Typically caused by a slow network, a page still loading, or dynamic content (like a chart or data grid) still being rendered.
+  * **Common Clues:** `TimeoutError`, `waiting for selector`, "element is not visible yet".
+  * **Analysis:** The **screenshot** might show a blank page, a loading spinner, or a partially rendered page. The **goal** (e.g., "click button X") is to interact with an element that is *expected* on this page but hasn't appeared. This is NOT a fatal error, as a retry or longer wait could solve it.
+  * **Action:** The automation should typically wait longer, reload the page, or retry the action.
+  * **`detailed_reason`:** A brief summary, e.g., "Page is taking too long to load" or "Element `[selector]` not yet visible."
+
+**2. `overlay_popup_blocking`**
+
+  * **Description:** This is an **interruption error**. The target element *is* on the page, but it is obscured or blocked by another element on top of it.
+  * **Cause:** Cookie banners, subscription pop-ups, ad modals, chat widgets, or "support" buttons.
+  * **Common Clues:** "Element is not clickable at point," "Another element would receive the click," "Element is obscured."
+  * **Analysis:** The **screenshot** is key here. It will clearly show a pop-up or modal covering the content. The **goal** will be to interact with an element *behind* this overlay.
+  * **Action:** The automation should try to find and close the overlay (e.g., click an "Accept" or "Close" button).
+  * **`detailed_reason`:** Identify the blocking element, e.g., "A cookie consent pop-up is blocking the login button."
+
+**3. `fatal_error`**
+
+  * **Description:** This is a **permanent, non-recoverable error**. The automation is stuck, and a simple wait or reload **will not** fix the problem.
+  * **Cause:**
+      * **Wrong Page:** The script navigated to the wrong URL (e.g., got a 404, 500 server error). The **screenshot** would show this error page.
+      * **Permanently Missing Element:** A required element *does not exist* on the page (it's not just loading, it's missing from the DOM).
+          * **Analysis:** Use the **goal** (e.g., "Click the 'Next Step' button") and the **screenshot**. If the page in the screenshot appears *fully loaded* (no spinners, all other content is present) but the target element is *nowhere to be found*, it is a `fatal_error`. This indicates a change in the website's structure or a flaw in the automation script's logic.
+      * **Logical Failure:** The automation cannot proceed due to invalid data (e.g., "Incorrect username or password") or a business rule violation (e.g., "Item is out of stock"). The **screenshot** would show this error message clearly displayed on the page. Since the automation **cannot ask a human** for new data, this is fatal.
+  * **Action:** The automation must stop and report the failure.
+  * **`detailed_reason`:** This is **mandatory and must be specific**.
+      * *Good:* "Fatal error: The target element `#submit-payment` does not exist on the page, even though the page appears fully loaded."
+      * *Good:* "Fatal error: Login failed due to 'Invalid credentials' message shown on page. Automation cannot proceed without new data."
+      * *Good:* "Fatal error: Navigation failed with a 404 error page."
+
+-----
+
+### Your Task
+
+Analyze the following **Goal, and Screenshot** and provide your classification in the required JSON format.
+"""

optexity/inference/agents/index_prediction/action_prediction_locator_axtree.py

@@ -0,0 +1,45 @@
+import logging
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+from optexity.inference.agents.index_prediction.prompt import system_prompt
+from optexity.inference.models import GeminiModels, get_llm_model
+from optexity.schema.token_usage import TokenUsage
+
+logger = logging.getLogger(__name__)
+
+
+class IndexPredictionOutput(BaseModel):
+    index: int = Field(
+        description="The index of the interactive element in the axtree that would achieve the desired outcome. It is always greater than 0."
+    )
+
+
+class ActionPredictionLocatorAxtree:
+    def __init__(self):
+        self.model = get_llm_model(GeminiModels.GEMINI_2_5_FLASH, True)
+
+    def predict_action(
+        self, goal: str, axtree: str, screenshot: Optional[str] = None
+    ) -> tuple[str, IndexPredictionOutput, TokenUsage]:
+
+        final_prompt = f"""
+        [INPUT]
+        Goal: {goal}
+
+        [AXTREE] 
+        {axtree} 
+        [/AXTREE]
+
+        [/INPUT]
+        """
+
+        response, token_usage = self.model.get_model_response_with_structured_output(
+            prompt=final_prompt,
+            response_schema=IndexPredictionOutput,
+            screenshot=screenshot,
+            system_instruction=system_prompt,
+        )
+
+        return final_prompt, response, token_usage

optexity/inference/agents/index_prediction/prompt.py

@@ -0,0 +1,14 @@
+system_prompt = """
+You are an AI assistant tasked with identifying the correct interactive element on a webpage based on a user's goal and a provided web page structure (axtree).
+
+Your core responsibility is to translate a user's intended action, described through a goal into a specific numerical index from the given axtree. This index represents the interactive element (e.g., a button, a text field) that, if interacted with, would achieve the desired outcome.
+
+**Input You Will Receive:**
+
+* **Goal:** The description of the task to be accomplished on the webpage.
+* **Axtree:** A simplified representation of the webpage's interactive elements. Each interactive element is marked with a bracketed number, like `[1]`, which is its unique index.
+
+**Crucial Task Directives:**
+
+Your output must be a single numerical index from the axtree. This is because index-based interaction is more reliable than trying to replicate a playwright command, which can fail if the element isn't precisely found.
+"""

optexity/inference/agents/select_value_prediction/prompt.py

@@ -0,0 +1,20 @@
+system_prompt = """
+You are an AI assistant tasked with helping users select relevant options from a webpage dropdown menu. You will be given a list of dropdown options, each with a "value" and a "label", along with a list of user-provided patterns. Your goal is to identify and return the dropdown values that best correspond to the user's patterns, taking into account both exact and approximate matches. 
+
+Guidelines:
+- A pattern may closely resemble, partially match, or refer to either the "label" or "value" of an option.
+- Use your reasoning to determine the most appropriate matches, even if they are not exact.
+- Focus on what the user is likely looking for based on the patterns and the option labels/values.
+
+Example:
+Dropdown options: 
+[{"value": "AAPL", "label": "Apple Inc"}, {"value": "GOOGL", "label": "Google Inc"}, {"value": "MSFT", "label": "Microsoft Inc"}, {"value": "NVDA", "label": "NVIDIA Inc"}]
+User patterns: ["apple", "nvidia"]
+Expected output: ["AAPL", "NVDA"]
+(Rationale: "apple" most closely matches "Apple Inc" → "AAPL"; "nvidia" matches "NVIDIA Inc" → "NVDA".)
+
+Instructions:
+- Return only the matched dropdown values, as a Python list of strings (e.g., ["AAPL", "NVDA"]).
+- If there are no valid matches, return an empty Python list (e.g., []).
+- Do not include any explanations or formatting—just the list.
+"""

optexity/inference/agents/select_value_prediction/select_value_prediction.py

@@ -0,0 +1,39 @@
+import json
+import logging
+
+from pydantic import BaseModel, Field
+
+from optexity.inference.agents.select_value_prediction.prompt import system_prompt
+from optexity.inference.models import GeminiModels, get_llm_model
+from optexity.schema.token_usage import TokenUsage
+
+logger = logging.getLogger(__name__)
+
+
+class SelectValuePredictionOutput(BaseModel):
+    matched_values: list[str] = Field(default_factory=list)
+
+
+class SelectValuePredictionAgent:
+    def __init__(self):
+        self.model = get_llm_model(GeminiModels.GEMINI_2_5_FLASH, True)
+
+    def predict_select_value(
+        self, options: list[dict[str, str]], patterns: list[str]
+    ) -> tuple[str, SelectValuePredictionOutput, TokenUsage]:
+
+        final_prompt = f"""
+        [Actual Select Options]
+        {json.dumps(options, indent=4)}
+
+        [User Provided Patterns]
+        [{', '.join(patterns)}]
+        """
+
+        response, token_usage = self.model.get_model_response_with_structured_output(
+            prompt=final_prompt,
+            response_schema=SelectValuePredictionOutput,
+            system_instruction=system_prompt,
+        )
+
+        return final_prompt, response, token_usage

optexity/inference/agents/two_fa_extraction/prompt.py

@@ -0,0 +1,23 @@
+system_prompt = """
+You are an expert AI assistant specializing in extracting Two-Factor Authentication (2FA) codes from digital messages. Your goal is to accurately identify and extract ONLY valid 2FA codes from a provided list of messages. 
+
+Carefully follow these instructions:
+
+1. Read each message in the list, looking for explicit 2FA codes.
+2. Extract only the codes that are clearly intended for authentication—do not extract any other numbers, words, or irrelevant information.
+3. Exclude numbers or text from headers, footers, signatures, or unrelated content, even if they appear similar to codes.
+4. If there are multiple distinct 2FA codes across the messages, return all of them as a list.
+5. If you find no valid 2FA code in any message, return None.
+
+Sometimes you may be given additional, specific extraction instructions—always follow those if present and give them highest priority.
+
+Context: Messages may come from various platforms (such as email, chat, or Slack).
+
+**Input:**
+- A list of messages to analyze.
+
+**Output:**
+- The extracted 2FA code (as a string), a list of codes (if multiple are found), or None if no code exists.
+
+Carefully consider the content of each message and reason step-by-step before providing your answer. Return only the code(s), with no extra commentary or explanation.
+"""

optexity/inference/agents/two_fa_extraction/two_fa_extraction.py

@@ -0,0 +1,47 @@
+import json
+import logging
+
+from pydantic import BaseModel, Field
+
+from optexity.inference.agents.two_fa_extraction.prompt import system_prompt
+from optexity.inference.models import GeminiModels, get_llm_model
+from optexity.schema.inference import Message
+from optexity.schema.token_usage import TokenUsage
+
+logger = logging.getLogger(__name__)
+
+
+class TwoFAExtractionOutput(BaseModel):
+    code: str | list[str] | None = Field(
+        description="The 2FA code extracted from the messages."
+    )
+
+
+class TwoFAExtraction:
+    def __init__(self):
+        self.model = get_llm_model(GeminiModels.GEMINI_2_5_FLASH, True)
+
+    def extract_code(
+        self, instructions: str | None, messages: list[Message]
+    ) -> tuple[str, TwoFAExtractionOutput, TokenUsage]:
+
+        final_prompt = ""
+
+        if instructions is not None:
+            final_prompt += f"""
+            [EXTRACTION INSTRUCTIONS]
+            {instructions}
+            [/EXTRACTION INSTRUCTIONS]
+            """
+        final_prompt += f"""
+        [MESSAGES] 
+        {json.dumps([message.model_dump(include={"message_text"}) for message in messages], indent=2)} 
+        [/MESSAGES]
+        """
+
+        response, token_usage = self.model.get_model_response_with_structured_output(
+            prompt=final_prompt,
+            response_schema=TwoFAExtractionOutput,
+            system_instruction=system_prompt,
+        )
+        return final_prompt, response, token_usage

optexity/inference/child_process.py

@@ -0,0 +1,251 @@
+import argparse
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+from datetime import datetime, timedelta, timezone
+from urllib.parse import urljoin
+
+import httpx
+from fastapi import Body, FastAPI
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from uvicorn import run
+
+from optexity.inference.core.run_automation import run_automation
+from optexity.schema.inference import InferenceRequest
+from optexity.schema.task import Task
+from optexity.utils.settings import settings
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+class ChildProcessIdRequest(BaseModel):
+    new_child_process_id: str
+
+
+child_process_id = None
+task_running = False
+last_task_start_time = None
+task_queue: asyncio.Queue[Task] = asyncio.Queue()
+
+
+async def task_processor():
+    """Background worker that processes tasks from the queue one at a time."""
+    global task_running
+    global last_task_start_time
+    logger.info("Task processor started")
+
+    while True:
+        try:
+            # Get next task from queue (blocks until one is available)
+            task = await task_queue.get()
+            task_running = True
+            last_task_start_time = datetime.now()
+            await run_automation(task, child_process_id)
+
+        except asyncio.CancelledError:
+            logger.info("Task processor cancelled")
+            break
+        except Exception as e:
+            logger.error(f"Error in task processor: {e}")
+        finally:
+
+            task_running = False
+
+
+async def register_with_master():
+    """Register with master on startup (handles restarts automatically)."""
+    # Get my task metadata from ECS
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        response = await client.get("http://169.254.170.2/v3/task")
+        response.raise_for_status()
+        metadata = response.json()
+
+    my_task_arn = metadata["TaskARN"]
+    my_ip = metadata["Containers"][0]["Networks"][0]["IPv4Addresses"][0]
+
+    my_port = None
+    for binding in metadata["Containers"][0].get("NetworkBindings", []):
+        if binding["containerPort"] == settings.CHILD_PORT_OFFSET:
+            my_port = binding["hostPort"]
+            break
+
+    if not my_port:
+        logger.error("Could not find host port binding")
+        raise ValueError("Host port not found in metadata")
+
+    # Register with master
+    async with httpx.AsyncClient(timeout=30.0) as client:
+        response = await client.post(
+            f"http://{settings.SERVER_URL}/register_child",
+            json={"task_arn": my_task_arn, "private_ip": my_ip, "port": my_port},
+        )
+        response.raise_for_status()
+
+    logger.info(f"Registered with master: {response.json()}")
+
+
+def get_app_with_endpoints(is_aws: bool, child_id: int):
+    global child_process_id
+    child_process_id = child_id
+
+    @asynccontextmanager
+    async def lifespan(app: FastAPI):
+        """Lifespan context manager for startup and shutdown."""
+        # Startup
+
+        if is_aws:
+            asyncio.create_task(register_with_master())
+
+        logger.info("Registered with master")
+        asyncio.create_task(task_processor())
+        logger.info("Task processor background task started")
+        yield
+        # Shutdown (if needed in the future)
+        logger.info("Shutting down task processor")
+
+    app = FastAPI(title="Optexity Inference", lifespan=lifespan)
+
+    @app.get("/is_task_running", tags=["info"])
+    async def is_task_running():
+        """Is task running endpoint."""
+        return task_running
+
+    @app.get("/health", tags=["info"])
+    async def health():
+        """Health check endpoint."""
+        global last_task_start_time
+        if (
+            task_running
+            and last_task_start_time
+            and datetime.now() - last_task_start_time > timedelta(minutes=15)
+        ):
+            return JSONResponse(
+                status_code=503,
+                content={
+                    "status": "unhealthy",
+                    "message": "Task not finished in the last 15 minutes",
+                },
+            )
+        return JSONResponse(
+            status_code=200,
+            content={
+                "status": "healthy",
+                "task_running": task_running,
+                "queued_tasks": task_queue.qsize(),
+            },
+        )
+
+    @app.post("/set_child_process_id", tags=["info"])
+    async def set_child_process_id(request: ChildProcessIdRequest):
+        """Set child process id endpoint."""
+        global child_process_id
+        child_process_id = int(request.new_child_process_id)
+        return JSONResponse(
+            content={"success": True, "message": "Child process id has been set"},
+            status_code=200,
+        )
+
+    @app.post("/allocate_task")
+    async def allocate_task(task: Task = Body(...)):
+        """Get details of a specific task."""
+        try:
+
+            await task_queue.put(task)
+            return JSONResponse(
+                content={"success": True, "message": "Task has been allocated"},
+                status_code=202,
+            )
+        except Exception as e:
+            logger.error(f"Error allocating task {task.task_id}: {e}")
+            return JSONResponse(
+                content={"success": False, "message": str(e)}, status_code=500
+            )
+
+    if not is_aws:
+
+        @app.post("/inference")
+        async def inference(inference_request: InferenceRequest = Body(...)):
+            response_data: dict | None = None
+            try:
+
+                async with httpx.AsyncClient(timeout=30.0) as client:
+                    url = urljoin(settings.SERVER_URL, settings.INFERENCE_ENDPOINT)
+                    headers = {"x-api-key": settings.API_KEY}
+                    response = await client.post(
+                        url, json=inference_request.model_dump(), headers=headers
+                    )
+                    response_data = response.json()
+                    response.raise_for_status()
+
+                task_data = response_data["task"]
+
+                task = Task.model_validate_json(task_data)
+                if task.use_proxy and settings.PROXY_URL is None:
+                    raise ValueError(
+                        "PROXY_URL is not set and is required when use_proxy is True"
+                    )
+                task.allocated_at = datetime.now(timezone.utc)
+                await task_queue.put(task)
+
+                return JSONResponse(
+                    content={
+                        "success": True,
+                        "message": "Task has been allocated",
+                        "task_id": task.task_id,
+                    },
+                    status_code=202,
+                )
+
+            except Exception as e:
+                error = str(e)
+                if response_data is not None:
+                    error = response_data.get("error", str(e))
+
+                logger.error(f"❌ Error fetching recordings: {error}")
+                return JSONResponse({"success": False, "error": error}, status_code=500)
+
+    return app
+
+
+def main():
+    """Main function to run the server."""
+    parser = argparse.ArgumentParser(
+        description="Dynamic API endpoint generator for Optexity recordings"
+    )
+
+    parser.add_argument(
+        "--host",
+        type=str,
+        default="0.0.0.0",
+        help="Host to bind the server to (default: 0.0.0.0)",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        help="Port to run the server ",
+    )
+    parser.add_argument(
+        "--child_process_id",
+        type=int,
+        help="Child process ID",
+    )
+    parser.add_argument(
+        "--is_aws",
+        action="store_true",
+        help="Is child process",
+        default=False,
+    )
+
+    args = parser.parse_args()
+
+    app = get_app_with_endpoints(is_aws=args.is_aws, child_id=args.child_process_id)
+
+    # Start the server (this is blocking and manages its own event loop)
+    logger.info(f"Starting server on {args.host}:{args.port}")
+    run(app, host=args.host, port=args.port)
+
+
+if __name__ == "__main__":
+    main()