clerk-sdk 0.1.9__py3-none-any.whl → 0.2.0__py3-none-any.whl

clerk/gui_automation/ui_actions/base.py

@@ -0,0 +1,200 @@
+from typing import Literal, Self, Union, List, Optional
+from pydantic import BaseModel, Field, model_validator
+from ..client_actor import get_screen
+from ..exceptions.modality.exc import TargetModalityError
+from ..action_model.model import (
+    ImageB64,
+    Coords,
+    Screenshot,
+    Anchor,
+)
+import os
+
+ModalityType = Union[Literal["icon"], Literal["text"]]
+TARGET_IMAGES_PATH = os.path.join(os.getcwd(), "targets")
+
+
+def to_full_img_path(img: Union[str, ImageB64]) -> str:
+    """
+    Add prefix if provided `img` is a string, otherwise return a default value which will later be evaluated
+    `False` in `_is_path`
+    """
+    if isinstance(img, ImageB64):
+        return ""
+    return os.path.join(TARGET_IMAGES_PATH, img)
+
+
+ActionTypes = Literal[
+    "left_click",
+    "right_click",
+    "middle_click",
+    "double_click",
+    "send_keys",
+    "press_keys",
+    "wait_for",
+    "open_app",
+    "force_close_app",
+    "maximize_window",
+    "minimize_window",
+    "close_window",
+    "activate_window",
+    "save_files",
+    "delete_files",
+    "get_file",
+    "get_text",
+    "paste_text",
+    "scroll",
+]
+
+
+class BaseAction(BaseModel):
+    """
+    BaseAction class represents a base model for UI actions.
+
+    Attributes:
+        action_type (ActionTypes): Type of UI action to execute.
+        target_name (Optional[str]): A readable representation of a target which is set automatically when validating the target and is used in the AM for logging.
+        target (Optional[Union[str, ImageB64]]): Target of the UI action. It can be provided as a string, an instance of the ImageB64 class, or a path to an image.
+        anchors (List[Anchor]): List of anchor points for the UI action.
+        is_awaited (bool): A flag to signal whether the target should appear immediately or is awaited. Should be set to `True` in WaitFor
+        widget_bbox: (Optional[Coords]): The bounding box coordinates of the widget. If set, the call to the action module will be bypassed.
+
+    Methods:
+        _get_center_coords(bbox: Coords) -> Union[List[int], List[float]]:
+            Returns the center coordinates of a bounding box.
+
+        check_target(cls, value):
+            Validator function to check the target modality.
+
+        _prepare_payload():
+            Prepares the payload for the UI action.
+
+        _prepare_payload_test(screen_id: str, bbox: CoordsType, is_last: bool = False):
+            Prepares the payload for the detection test.
+
+        test(screen_id: str, bbox: CoordsType, is_last: bool = False):
+            Performs a detection test using the provided payload.
+
+        left(anchor: Union[str, ImageB64]):
+            Adds a left anchor point to the list of anchors.
+
+        right(anchor: Union[str, ImageB64]):
+            Adds a right anchor point to the list of anchors.
+
+        above(anchor: Union[str, ImageB64]):
+            Adds an above anchor point to the list of anchors.
+
+        below(anchor: Union[str, ImageB64]):
+            Adds a below anchor point to the list of anchors.
+
+        _is_path(value: str) -> bool:
+            Checks if a given value is a valid file path.
+
+        do():
+            Placeholder method for executing the UI action.
+    """
+
+    action_type: ActionTypes = Field(..., description="Type of ui action to execute")
+    target_name: Optional[str] = Field(default=None)
+    target: Optional[Union[str, ImageB64]] = Field(default=None)
+    anchors: List[Anchor] = []
+    click_offset: List[int] = [0, 0]
+    is_awaited: bool = False
+    widget_bbox: Optional[Coords] = None
+
+    @model_validator(mode="after")
+    def validate_target_and_set_name(self) -> Self:
+        target = self.target
+        if isinstance(target, str):  # either text target or img path
+            full_image_path = to_full_img_path(target)
+            if self._is_path(full_image_path):
+                self.target = ImageB64.from_path(full_image_path)
+                self.target_name = target  # Set target name as path for logging
+            else:
+                # Set target name as provided string for logging
+                self.target_name = target
+            return self
+        elif isinstance(target, ImageB64):
+            self.target_name = "provided as obj with value in b64"
+            return self
+        elif target is None:
+            self.target_name = "not_provided"
+            return self
+        raise TargetModalityError()
+
+    def _get_center_coords(self, bbox: Coords) -> Union[List[int], List[float]]:
+        w: Union[int, float] = bbox.value[2] - bbox.value[0]
+        h: Union[int, float] = bbox.value[3] - bbox.value[1]
+        xcenter: Union[int, float] = bbox.value[0] + w // 2 + self.click_offset[0]
+        ycenter: Union[int, float] = bbox.value[1] + h // 2 + self.click_offset[1]
+        return [xcenter, ycenter]
+
+    def _prepare_payload(self):
+        payload: Screenshot = Screenshot(
+            screen_b64=ImageB64(value=get_screen()),
+            target=self.target,
+            anchors=self.anchors,
+            is_awaited=self.is_awaited,
+            target_name=self.target_name,
+        )
+        return payload
+
+    def left(self, anchor: Union[str, ImageB64]):
+        value: Union[str, ImageB64] = (
+            anchor
+            if not self._is_path(to_full_img_path(anchor))
+            else ImageB64.from_path(to_full_img_path(anchor))
+        )
+        self.anchors.append(Anchor(value=value, relation="left"))
+        return self
+
+    def right(self, anchor: Union[str, ImageB64]):
+        value: Union[str, ImageB64] = (
+            anchor
+            if not self._is_path(to_full_img_path(anchor))
+            else ImageB64.from_path(to_full_img_path(anchor))
+        )
+        self.anchors.append(Anchor(value=value, relation="right"))
+        return self
+
+    def above(self, anchor: Union[str, ImageB64]):
+        value: Union[str, ImageB64] = (
+            anchor
+            if not self._is_path(to_full_img_path(anchor))
+            else ImageB64.from_path(to_full_img_path(anchor))
+        )
+        self.anchors.append(Anchor(value=value, relation="above"))
+        return self
+
+    def below(self, anchor: Union[str, ImageB64]):
+        value: Union[str, ImageB64] = (
+            anchor
+            if not self._is_path(to_full_img_path(anchor))
+            else ImageB64.from_path(to_full_img_path(anchor))
+        )
+        self.anchors.append(Anchor(value=value, relation="below"))
+        return self
+
+    def offset(self, x: int = 0, y: int = 0):
+        """
+        Add a pixel offset to the click action (coordinates start at the top-left corner of the screen).
+        Args:
+            x (int): Horizontal offset (left to right).
+            y (int): Vertical offset (top to bottom).
+        Returns:
+            BaseAction: BaseAction instance with the updated click offset.
+        Usage:
+            # click 10 pixels to the right and 20 pixels above the center of the target.
+            LeftClick(target="target").offset(x=10, y=-20).do()
+        """
+        self.click_offset = [x, y]
+        return self
+
+    @staticmethod
+    def _is_path(value: str) -> bool:
+        if not os.path.isfile(value):
+            return os.path.isfile(value)
+        return True
+
+    def do(self):
+        pass

clerk/gui_automation/ui_actions/support.py

@@ -0,0 +1,68 @@
+import os
+from typing import Optional
+from backoff._typing import Details
+
+from clerk.utils.save_artifact import save_artifact
+from ..client_actor import get_screen
+
+
+_MAP = {
+    "y": True,
+    "yes": True,
+    "t": True,
+    "true": True,
+    "on": True,
+    "1": True,
+    "n": False,
+    "no": False,
+    "f": False,
+    "false": False,
+    "off": False,
+    "0": False,
+}
+
+
+def strtobool(value):
+    try:
+        return _MAP[str(value).lower()]
+    except KeyError:
+        raise ValueError('"{}" is not a valid bool value'.format(value))
+
+
+def save_screenshot(filename: str, sub_folder: Optional[str] = None) -> str:
+    """
+    Save a screenshot into the process instance folder.
+
+    This function retrieves the base64 representation of the screen from the target environment using the 'get_screen' function.
+    Then, it saves the screenshot into the process instance folder using the 'save_file_into_instance_folder' function.
+
+    Args:
+        filename (str): The name of the file to save the screenshot as.
+        sub_folder (str, optional): The name of the subfolder within the instance folder where the screenshot will be saved. Defaults to None.
+
+    Returns:
+        str: The file path of the saved screenshot.
+
+    """
+    # get the base64 screen from target environment
+    screen_b64: str = get_screen()
+    return save_artifact(
+        filename=filename,
+        file_bytes=screen_b64.encode("utf-8"),
+        subfolder=sub_folder,
+    )
+
+
+def maybe_engage_operator_ui_action(details: Details) -> None:
+    """
+    Makes a call to the operator queue server to create an issue and waits for the allotted time for it to be resolved.
+    :param details: A dictionary containing the details of the exception raised (https://pypi.org/project/backoff/)
+    :returns: None
+    :raises: The exception raised by the action if the issue is not resolved within the allotted time
+    """
+    # Determine if the operator should be engaged
+    use_operator = strtobool(os.getenv("USE_OPERATOR", default="False"))
+    if not use_operator:
+        raise details["exception"]  # type: ignore
+
+    raise NotImplementedError("Feature not yet implemented")

clerk/gui_automation/ui_state_inspector/gui_vision.py

@@ -0,0 +1,184 @@
+from typing import Dict, Union, List, Tuple, Type, Literal
+from pydantic import BaseModel, Field
+
+from clerk.gui_automation.client import GUIVisionClerk
+
+from ..client_actor.client_actor import get_screen
+from .models import (
+    States,
+    BaseState,
+    ExpectedState,
+    TargetWithAnchor,
+    Answer,
+    ActionString,
+)
+
+
+class Vision(BaseModel):
+    """
+    Provides methods for interacting with a GUI for UI automation purposes. This class includes methods for finding
+    targets on the screen, verifying the GUI's state, answering questions about the screen, classifying the state of
+    the GUI, and generating action strings based on prompts.
+
+    Attributes:
+        response_models: A dictionary mapping task names to their corresponding response model classes.
+        use_ocr: A boolean indicating whether OCR should be included in the model call to increase precision with small details.
+        image_resolution: A parameter defining the resolution of the image used in the vision model.
+
+    Methods:
+        find_target(target_prompt: str, output_model: Type[TargetWithAnchor] = TargetWithAnchor) -> TargetWithAnchor:
+            Finds a target in the current screen based on the provided prompt. Limited to one-word targets.
+
+        verify_state(possible_states: States, output_model: Type[BaseState] = BaseState) -> BaseState:
+            Verifies the current state of the GUI against a set of possible states.
+
+        answer(question: str, output_model: Type[BaseModel] = Answer) -> Answer:
+            Answers a question about the current screen using the specified model for the response.
+
+        classify_state(possible_states: List[Dict[str, str]], output_model: Type[BaseState] = BaseState) -> Union[BaseModel, Tuple[str, str]]:
+            Classifies the current state of the GUI into one of the provided possible states. Returns either a model instance or a tuple of the ID and description.
+
+        write_action_string(action_prompt: str, output_model: Type[ActionString] = ActionString) -> ActionString:
+            Generates an action string based on the provided prompt.
+
+    Note: Each method that interacts with the screen can optionally include OCR data to improve accuracy, controlled by the `use_ocr` attribute.
+    """
+
+    response_models: Dict[str, Type[BaseModel]] = {
+        "find_target": TargetWithAnchor,
+        "answer": Answer,
+        "verify_state": BaseState,
+        "classify_state": BaseState,
+        "write_action_string": ActionString,
+    }
+    use_ocr: bool = Field(
+        default=False,
+        description="Whether OCR of the screen should be included with in the model call (increases precision with "
+        "small details).",
+    )
+    image_resolution: Literal["high", "low"] = "high"
+    clerk_client: GUIVisionClerk = GUIVisionClerk()
+
+    class Config:
+        arbitrary_types_allowed = True
+
+    @staticmethod
+    def _sort_into_state_class(
+        model_response: BaseState, possible_states: States
+    ) -> BaseState:
+        """
+        Sorts a model response into a corresponding state class.
+        Args:
+            model_response: The response from the model.
+            possible_states: A collection of possible states.
+        Returns:
+            A state class matching the model response.
+        """
+        for state_class, state_object in possible_states.possible_states.items():
+            if model_response.id == state_object.id:
+                return state_class(description=model_response.description)
+        # return expected by default
+        return ExpectedState(description=model_response.description)
+
+    def find_target(
+        self,
+        target_prompt: str,
+        output_model: Type[TargetWithAnchor] = TargetWithAnchor,
+    ) -> TargetWithAnchor:
+        """
+        Finds a target in the current screen. Currently limited to one word targets.
+        Args:
+            target_prompt: The prompt for the target to find.
+            output_model: The model to use for the response. If not provided, the default model for the task will be used.
+        Returns:
+            TargetWithAnchor object with the response from the model. Access the target with the "target" attribute.
+        """
+        screen_b64 = get_screen()
+
+        target = self.clerk_client.find_target(
+            screen_b64,
+            self.use_ocr,
+            target_prompt,
+        )
+        assert isinstance(target, output_model)
+        return target
+
+    def verify_state(
+        self, possible_states: States, output_model: Type[BaseState] = BaseState
+    ) -> BaseState:
+        """
+        Verifies the current state of the GUI.
+        Args:
+            possible_states: The possible states of the GUI (State class incl. screen examples).
+            output_model: The model to use for the response. If not provided, the default model for the task will be used.
+        Returns:
+            The current state of the GUI (BaseState or a subclass of BaseState)
+        """
+        screen_b64 = get_screen()
+        state = self.verify_state(
+            screen_b64,
+            self.use_ocr,
+            possible_states,
+        )
+        assert isinstance(state, BaseState)
+        sorted_state = self._sort_into_state_class(state, possible_states)
+        return sorted_state
+
+    def answer(
+        self, question: str, output_model: Type[BaseModel] = Answer
+    ) -> BaseModel:
+        """
+        Answers a question about the current screen.
+        Args:
+            question: The question to ask about the current screen.
+            output_model: The model to use for the response. If not provided, the default model for the task will be used.
+        Returns:
+            Answer object with the response from the model. Access the text with the "answer" attribute.
+        """
+        screen_b64 = get_screen()
+        answer = self.clerk_client.answer(
+            screen_b64, self.use_ocr, question, output_model
+        )
+        assert isinstance(answer, output_model)
+        return answer
+
+    def classify_state(
+        self,
+        possible_states: List[Dict[str, str]],
+        output_model: Type[BaseState] = BaseState,
+    ) -> Union[BaseModel, Tuple[str, str]]:
+        """
+        Classify the current state of the GUI into one of the provided classes.
+        Args:
+            possible_states: The possible states of the GUI.
+            output_model: The model to use for the response.
+        Returns:
+            The current state of the GUI (BaseState class if an output model was provided; access class key with the "id" attribute), otherwise Tuple of the id and description of the default model.
+        """
+        screen_b64 = get_screen()
+        state = self.clerk_client.classify_state(
+            screen_b64, self.use_ocr, possible_states
+        )
+        # if output_model is provided, return the model, otherwise return the id and description of the default model
+        if output_model is not None:
+            return state
+        assert isinstance(state, BaseState)
+        return state.id, state.description
+
+    def write_action_string(
+        self, action_prompt: str, output_model: Type[ActionString] = ActionString
+    ) -> ActionString:
+        """
+        Writes an action string based on the provided prompt.
+        Args:
+            action_prompt: The prompt for the action to write.
+            output_model: The model to use for the response.
+        Returns:
+            The action string.
+        """
+        screen_b64 = get_screen()
+        action_string = self.clerk_client.write_action_string(
+            screen_b64, self.use_ocr, action_prompt
+        )
+        assert isinstance(action_string, ActionString)
+        return action_string

clerk/gui_automation/ui_state_inspector/models.py

@@ -0,0 +1,184 @@
+from pydantic import BaseModel, field_validator, model_validator
+from typing import List, Dict, Optional
+
+
+class BaseState(BaseModel):
+    """
+    BaseState class represents a base state for an application.
+
+    Attributes:
+        id (str): The ID of the state.
+        description (str): The description of the state.
+        screenshots (List): A list of dictionaries representing the screenshots associated with the state.
+
+    Methods:
+        add_screenshot(bucket_name: str, file_name: str) -> None:
+            Adds a screenshot to the state.
+
+        get_screenshots_urls() -> List[str]:
+            Returns a list of presigned URLs for the screenshots associated with the state.
+    """
+
+    id: str
+    description: str
+    screenshots: List = []
+
+    def add_screenshot(self, bucket_name: str, file_name: str):
+        self.screenshots.append({"bucket_name": bucket_name, "file_name": file_name})
+
+
+class LoadingState(BaseState):
+    """
+    LoadingState class represents a loading state for an application.
+
+    Attributes:
+        id (str): The ID of the loading state.
+        description (str): The description of the loading state.
+
+    """
+
+    id: str = "loading"
+    description: str = (
+        "the application is loading. Typically indicated by a spinner or progress bar, greyed out UI"
+    )
+
+
+class ErrorState(BaseState):
+    """
+    ErrorState class represents an error state for an application.
+
+    Attributes:
+        id (str): The ID of the error state.
+        description (str): The description of the error state.
+
+    """
+
+    id: str = "error"
+    description: str = (
+        "the application is in an error state. Typically indicated by an error message, or a red banner"
+    )
+
+
+class ExpectedState(BaseState):
+    """
+    ExpectedState class represents an expected state for an application.
+
+    Attributes:
+        id (str): The ID of the state.
+        description (str): The description of the state.
+
+    """
+
+    id: str = "expected"
+    description: str = (
+        "the application is in an expected state, as in the provided screenshot"
+    )
+
+
+class States(BaseModel):
+    """
+    States class represents a collection of states for an application.
+
+    Attributes:
+        possible_states (Dict[type, BaseState]): A dictionary mapping state types to their corresponding instances.
+        bucket_name (str): The name of the bucket where screenshots are stored.
+        process_name (str): The name of the process associated with the states.
+
+    Methods:
+        add_screenshot(state: type[BaseState], file_name: str) -> None:
+            Adds a screenshot to the specified state.
+
+        add_description(state: type[BaseState], description: str) -> None:
+            Updates the description of the specified state.
+    """
+
+    possible_states: Dict[type, BaseState] = {
+        LoadingState: LoadingState(),
+        ErrorState: ErrorState(),
+        ExpectedState: ExpectedState(),
+    }
+    bucket_name: str
+    process_name: str
+
+    def add_screenshot(self, state_type: type[BaseState], file_name: str):
+        state = self.possible_states.get(state_type)
+        if state is None:
+            raise ValueError("state is not found in possible states")
+        state.add_screenshot(
+            bucket_name=f"{self.bucket_name}",
+            file_name=f"{self.process_name}/{file_name}",
+        )
+
+    def add_description(self, state_type: type[BaseState], description: str):
+        state = self.possible_states.get(state_type)
+        if state is None:
+            raise ValueError("state is not found in possible states")
+        state.description = description
+
+
+class TargetWithAnchor(BaseModel):
+    """
+    TargetWithAnchor class represents a target with an anchor for an application.
+
+    Attributes:
+        target (str): The target element or object.
+        anchor (str): The anchor element or object that the target is related to. Default is an empty string.
+        relation (str): The relation between the target and the anchor. Default is an empty string.
+
+    Methods:
+        retain_one_word(v: str) -> str:
+            Validator function that retains only the last word of the target string.
+
+    """
+
+    target: str
+    anchor: str = ""
+    relation: str = ""
+
+    @field_validator("target", mode="before")
+    @classmethod
+    def retain_one_word(cls, v):
+        return v.split(" ")[-1]
+
+
+class Answer(BaseModel):
+    """
+    Answer class represents the result of an operation or a response to a question.
+
+    Attributes:
+        answer (str): The answer or response.
+        success (bool): Indicates whether the operation was successful or not.
+
+    """
+
+    answer: str
+    success: bool
+
+
+class ActionString(BaseModel):
+    """
+    ActionString class represents a string that represents an action in an application.
+
+    Attributes:
+        action_string (str): The string representation of the action.
+        comment (str, optional): An optional comment or description for the action.
+
+    Methods:
+        ensure_format(v: str) -> str:
+            Validator function that ensures the action string has the correct format.
+
+    """
+
+    action_string: str
+    comment: Optional[str] = None
+
+    @field_validator("action_string", mode="before")
+    @classmethod
+    def ensure_format(cls, v):
+        if not isinstance(v, str):
+            raise ValueError("Action string must be a string")
+        if not v.startswith("LeftClick") and not v.startswith("NoAction"):
+            raise ValueError("Action string must start with 'LeftClick' or 'NoAction'")
+        if not v.endswith(".do()") and not v.startswith("NoAction"):
+            raise ValueError("Action string must end with '.do()'")
+        return v

clerk/gui_automation/ui_state_machine/__init__.py

@@ -0,0 +1,11 @@
+from .state_machine import ScreenPilot
+from .decorators import state, transition, rollback
+from .exceptions import (
+    BusinessException,
+    ScreenPilotException,
+    SuccessfulCompletion,
+    RollbackCompleted,
+    ScreenPilotOutcome,
+    CourseCorrectionImpossible,
+    complete_ui_automation,
+)