clerk-sdk 0.1.9__py3-none-any.whl → 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clerk/base.py +94 -0
- clerk/client.py +3 -104
- clerk/decorator/models.py +1 -0
- clerk/decorator/task_decorator.py +1 -0
- clerk/gui_automation/__init__.py +0 -0
- clerk/gui_automation/action_model/__init__.py +0 -0
- clerk/gui_automation/action_model/model.py +126 -0
- clerk/gui_automation/action_model/utils.py +26 -0
- clerk/gui_automation/client.py +144 -0
- clerk/gui_automation/client_actor/__init__.py +4 -0
- clerk/gui_automation/client_actor/client_actor.py +178 -0
- clerk/gui_automation/client_actor/exception.py +22 -0
- clerk/gui_automation/client_actor/model.py +192 -0
- clerk/gui_automation/decorators/__init__.py +1 -0
- clerk/gui_automation/decorators/gui_automation.py +109 -0
- clerk/gui_automation/exceptions/__init__.py +0 -0
- clerk/gui_automation/exceptions/modality/__init__.py +0 -0
- clerk/gui_automation/exceptions/modality/exc.py +46 -0
- clerk/gui_automation/exceptions/websocket.py +6 -0
- clerk/gui_automation/ui_actions/__init__.py +1 -0
- clerk/gui_automation/ui_actions/actions.py +781 -0
- clerk/gui_automation/ui_actions/base.py +200 -0
- clerk/gui_automation/ui_actions/support.py +68 -0
- clerk/gui_automation/ui_state_inspector/__init__.py +0 -0
- clerk/gui_automation/ui_state_inspector/gui_vision.py +184 -0
- clerk/gui_automation/ui_state_inspector/models.py +184 -0
- clerk/gui_automation/ui_state_machine/__init__.py +11 -0
- clerk/gui_automation/ui_state_machine/ai_recovery.py +110 -0
- clerk/gui_automation/ui_state_machine/decorators.py +71 -0
- clerk/gui_automation/ui_state_machine/exceptions.py +42 -0
- clerk/gui_automation/ui_state_machine/models.py +40 -0
- clerk/gui_automation/ui_state_machine/state_machine.py +838 -0
- clerk/models/remote_device.py +7 -0
- clerk/utils/__init__.py +0 -0
- clerk/utils/logger.py +118 -0
- clerk/utils/save_artifact.py +35 -0
- {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/METADATA +11 -1
- clerk_sdk-0.2.0.dist-info/RECORD +48 -0
- clerk_sdk-0.1.9.dist-info/RECORD +0 -15
- {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/WHEEL +0 -0
- {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/licenses/LICENSE +0 -0
- {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import networkx as nx # type: ignore
|
|
3
|
+
import functools
|
|
4
|
+
import inspect
|
|
5
|
+
from collections import deque
|
|
6
|
+
from pydantic import BaseModel, ValidationError
|
|
7
|
+
from typing import List, Tuple, Callable, Optional, Deque, Literal, Union
|
|
8
|
+
import traceback
|
|
9
|
+
from datetime import datetime
|
|
10
|
+
|
|
11
|
+
from ..ui_actions.actions import ForceCloseApplication
|
|
12
|
+
from ..ui_actions.support import save_screenshot
|
|
13
|
+
from .exceptions import (
|
|
14
|
+
BusinessException,
|
|
15
|
+
ScreenPilotException,
|
|
16
|
+
UnplannedTransitionsError,
|
|
17
|
+
RepeatTransitions,
|
|
18
|
+
RollbackCompleted,
|
|
19
|
+
RepeatStatesError,
|
|
20
|
+
ScreenPilotOutcome,
|
|
21
|
+
CourseCorrectionImpossible,
|
|
22
|
+
SuccessfulCompletion,
|
|
23
|
+
)
|
|
24
|
+
from ..ui_state_machine.models import ActionString
|
|
25
|
+
from .ai_recovery import CourseCorrector, course_corrector_v1
|
|
26
|
+
from ..client_actor.exception import PerformActionException
|
|
27
|
+
from ..ui_state_inspector.gui_vision import Vision
|
|
28
|
+
from ...client import Clerk
|
|
29
|
+
from ...utils import logger
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class ScreenPilot:
|
|
33
|
+
"""
|
|
34
|
+
A class representing a screen pilot state machine.
|
|
35
|
+
|
|
36
|
+
Attributes:
|
|
37
|
+
ai_recovery (bool): Whether to use AI recovery on errors. Default True.
|
|
38
|
+
ai_recovery_attempts (int): The number of AI recovery attempts. Default 5.
|
|
39
|
+
ai_recovery_agent_factory (Optional[Callable]): Factory for AI recovery agents. Default course_corrector_v1.
|
|
40
|
+
ai_recovery_instructions (Optional[str]): Custom instructions for the AI recovery agent. Default None.
|
|
41
|
+
state_eval_function (Callable): Function to evaluate the current state of the state machine. Default Vision().classify_state.
|
|
42
|
+
state_eval_output_model (BaseModel): Optional custom output model of the state evaluation function. Default None.
|
|
43
|
+
tolerate_unplanned_transitions (int): Number of unplanned transitions to tolerate before breaking the execution. Default 5.
|
|
44
|
+
tolerate_repeat_transitions (int): Number of repeated transitions to tolerate before breaking the execution. Default 5.
|
|
45
|
+
tolerate_repeat_states (int): Number of repeated states to tolerate before breaking the execution. Default 5.
|
|
46
|
+
enable_force_close_app_process (bool): If true, terminates the application process via `taskkill` command. Default False.
|
|
47
|
+
process_name (Optional[str]): Name of the process that needs to be closed (ie. process.exe). Required attribute if `enable_force_close_app_process` is True
|
|
48
|
+
|
|
49
|
+
Methods:
|
|
50
|
+
register_state(cls, state_cls): Register a state class in the state machine graph.
|
|
51
|
+
register_transition(cls, from_state, to_state): Register a transition function in the state machine graph.
|
|
52
|
+
configure(cls, **kwargs): Provide class parameters to configure the state machine.
|
|
53
|
+
run(cls, goal_function, **kwargs): Main loop of the state machine.
|
|
54
|
+
"""
|
|
55
|
+
|
|
56
|
+
ai_recovery: bool = True
|
|
57
|
+
ai_recovery_attempts: int = 5
|
|
58
|
+
ai_recovery_agent_factory: Optional[Callable] = course_corrector_v1
|
|
59
|
+
ai_recovery_instructions: Optional[str] = None
|
|
60
|
+
state_eval_function: Optional[Callable] = Vision().classify_state
|
|
61
|
+
state_eval_output_model: Optional[BaseModel] = None
|
|
62
|
+
tolerate_unplanned_transitions: int = 5
|
|
63
|
+
tolerate_repeat_transitions: int = 5
|
|
64
|
+
tolerate_repeat_states: int = 5
|
|
65
|
+
enable_force_close_app_process: bool = False
|
|
66
|
+
process_name: Optional[str] = None
|
|
67
|
+
_acted_since_state_eval: bool = False
|
|
68
|
+
_ai_recovery_agent: Optional[CourseCorrector] = None
|
|
69
|
+
_current_state: Optional[str] = None
|
|
70
|
+
_exit_reason: Optional[Union[ScreenPilotOutcome, ScreenPilotException]] = None
|
|
71
|
+
_final_state: Optional[str] = None
|
|
72
|
+
_graph = nx.MultiDiGraph()
|
|
73
|
+
_mode: Literal["planned", "rollback"] = "planned"
|
|
74
|
+
_next_target_state: Optional[str] = None
|
|
75
|
+
_runtime_error_details: Optional[Tuple[str, str, str]] = None
|
|
76
|
+
_state_history: Deque[str] = deque(maxlen=25)
|
|
77
|
+
_transition_history: Deque[Callable] = deque(maxlen=25)
|
|
78
|
+
_clerk_client: Clerk = Clerk(
|
|
79
|
+
base_url=os.getenv("CLERK_BASE_URL", "https://api.clerk-app.com")
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
@classmethod
|
|
83
|
+
def register_state(
|
|
84
|
+
cls, state_cls, start_allowed: bool = True, end_allowed: bool = True
|
|
85
|
+
):
|
|
86
|
+
"""
|
|
87
|
+
Register a state class in the state machine graph.
|
|
88
|
+
|
|
89
|
+
Parameters:
|
|
90
|
+
state_cls (class): The state class to be registered.
|
|
91
|
+
|
|
92
|
+
Returns:
|
|
93
|
+
class: The registered state class.
|
|
94
|
+
|
|
95
|
+
"""
|
|
96
|
+
cls._graph.add_node(
|
|
97
|
+
state_cls.__name__,
|
|
98
|
+
cls=state_cls,
|
|
99
|
+
description=state_cls.description,
|
|
100
|
+
start_allowed=start_allowed,
|
|
101
|
+
end_allowed=end_allowed,
|
|
102
|
+
)
|
|
103
|
+
return state_cls
|
|
104
|
+
|
|
105
|
+
@classmethod
|
|
106
|
+
def register_transition(
|
|
107
|
+
cls,
|
|
108
|
+
from_state: str,
|
|
109
|
+
to_state: str,
|
|
110
|
+
mode: Literal["planned", "rollback"] = "planned",
|
|
111
|
+
condition: Optional[Callable] = None,
|
|
112
|
+
):
|
|
113
|
+
"""
|
|
114
|
+
Register a transition function in the state machine graph.
|
|
115
|
+
|
|
116
|
+
Provides a decorator with logging and error handling.
|
|
117
|
+
|
|
118
|
+
Parameters:
|
|
119
|
+
from_state (str): The state from which the transition occurs.
|
|
120
|
+
to_state (str): The state to which the transition leads.
|
|
121
|
+
mode (Literal["planned", "rollback"]): The mode of the transition. Default "planned".
|
|
122
|
+
condition (Optional[Callable]): A condition function for the transition. Should return boolean. Default None.
|
|
123
|
+
|
|
124
|
+
Returns:
|
|
125
|
+
function: The decorated transition function.
|
|
126
|
+
|
|
127
|
+
Raises:
|
|
128
|
+
ValueError: If either the from_state or to_state is not a defined state in the state machine graph.
|
|
129
|
+
|
|
130
|
+
Example:
|
|
131
|
+
@register_transition("StateA", "StateB", mode="planned", condition=None)
|
|
132
|
+
def transition_func():
|
|
133
|
+
# Transition logic here
|
|
134
|
+
pass
|
|
135
|
+
"""
|
|
136
|
+
|
|
137
|
+
def decorator(transition_func):
|
|
138
|
+
@functools.wraps(transition_func)
|
|
139
|
+
def wrapper(*args, **kwargs):
|
|
140
|
+
# Argument filtering
|
|
141
|
+
sig = inspect.signature(transition_func)
|
|
142
|
+
supported_params = sig.parameters
|
|
143
|
+
filtered_kwargs = {
|
|
144
|
+
k: v for k, v in kwargs.items() if k in supported_params
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
try:
|
|
148
|
+
logger.debug(
|
|
149
|
+
f"Starting {transition_func.__name__}",
|
|
150
|
+
)
|
|
151
|
+
# Apply filtered_kwargs instead of kwargs
|
|
152
|
+
result = transition_func(*args, **filtered_kwargs)
|
|
153
|
+
logger.debug(
|
|
154
|
+
f"Finished {transition_func.__name__}",
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
return result
|
|
158
|
+
except BusinessException as e:
|
|
159
|
+
logger.info(f"Business exception: {e}\nExiting ScreenPilot.")
|
|
160
|
+
raise e
|
|
161
|
+
except RuntimeError as e:
|
|
162
|
+
cls._runtime_error_details = _action_line_from_exc()
|
|
163
|
+
logger.error(
|
|
164
|
+
f"Runtime error in {transition_func.__name__} at action {cls._runtime_error_details[1]}",
|
|
165
|
+
)
|
|
166
|
+
logger.debug(
|
|
167
|
+
f"Runtime error traceback: {cls._runtime_error_details[0]}",
|
|
168
|
+
)
|
|
169
|
+
screenshot_and_log("Runtime error")
|
|
170
|
+
logger.info(
|
|
171
|
+
"Proceeding to course correction, activating rollback mode.",
|
|
172
|
+
)
|
|
173
|
+
cls._attempt_ai_recovery(
|
|
174
|
+
scenario="runtime_error",
|
|
175
|
+
error_details=cls._runtime_error_details[1],
|
|
176
|
+
)
|
|
177
|
+
cls._mode = "rollback"
|
|
178
|
+
except RollbackCompleted as e:
|
|
179
|
+
logger.info("Rollback completed.")
|
|
180
|
+
raise type(e)(cls._runtime_error_details[0])
|
|
181
|
+
|
|
182
|
+
# Ensure that any duplicate transitions have a condition function
|
|
183
|
+
possible_transitions = cls._graph.out_edges(
|
|
184
|
+
from_state, keys=True, data=True
|
|
185
|
+
)
|
|
186
|
+
for start_state, end_state, key, data in possible_transitions:
|
|
187
|
+
if data.get("mode") == mode and (
|
|
188
|
+
not data.get("condition") or not condition
|
|
189
|
+
):
|
|
190
|
+
existing_transition_name = data.get("func").__name__
|
|
191
|
+
transition_type = "rollback" if mode == "rollback" else "transition"
|
|
192
|
+
raise ValueError(
|
|
193
|
+
f"Error while registering {wrapper.__name__}: {existing_transition_name} is already registered as {transition_type} from {start_state} to {end_state}. To add multiple transitions between the same states, provide a condition function for each of them."
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
# Register the wrapped function as a transition in the graph
|
|
197
|
+
if from_state in cls._graph.nodes and to_state in cls._graph.nodes:
|
|
198
|
+
key = f"{wrapper.__name__}_from_{from_state}_to_{to_state}_{mode}"
|
|
199
|
+
cls._graph.add_edge(
|
|
200
|
+
from_state,
|
|
201
|
+
to_state,
|
|
202
|
+
key=key,
|
|
203
|
+
func=wrapper,
|
|
204
|
+
condition=condition,
|
|
205
|
+
mode=mode,
|
|
206
|
+
)
|
|
207
|
+
else:
|
|
208
|
+
logger.error(
|
|
209
|
+
f"Error: Transition from {from_state} to {to_state} involves undefined state.",
|
|
210
|
+
)
|
|
211
|
+
return wrapper
|
|
212
|
+
|
|
213
|
+
return decorator
|
|
214
|
+
|
|
215
|
+
@classmethod
|
|
216
|
+
def _log_transition(cls, transition_func: Callable):
|
|
217
|
+
"""
|
|
218
|
+
Log a transition from one state to another.
|
|
219
|
+
|
|
220
|
+
Parameters:
|
|
221
|
+
from_state (str): The state from which the transition occurs.
|
|
222
|
+
to_state (str): The state to which the transition leads.
|
|
223
|
+
|
|
224
|
+
Returns:
|
|
225
|
+
None
|
|
226
|
+
|
|
227
|
+
"""
|
|
228
|
+
cls._transition_history.append(transition_func)
|
|
229
|
+
|
|
230
|
+
@classmethod
|
|
231
|
+
def _log_state(cls, state):
|
|
232
|
+
"""
|
|
233
|
+
Log a state in the state machine history.
|
|
234
|
+
|
|
235
|
+
Parameters:
|
|
236
|
+
state (str): The state to be logged.
|
|
237
|
+
|
|
238
|
+
Returns:
|
|
239
|
+
None
|
|
240
|
+
|
|
241
|
+
"""
|
|
242
|
+
cls._state_history.append(state)
|
|
243
|
+
|
|
244
|
+
@classmethod
|
|
245
|
+
def _evaluate_state(cls) -> None:
|
|
246
|
+
"""
|
|
247
|
+
Evaluate the current state of the state machine.
|
|
248
|
+
|
|
249
|
+
This method evaluates the current state of the state machine by calling the state evaluation function
|
|
250
|
+
provided during configuration. It retrieves the possible states from the state machine graph and passes
|
|
251
|
+
them along with the state evaluation output model to the state evaluation function. The current state and
|
|
252
|
+
its description are then updated based on the evaluation result. The current state is logged in the state
|
|
253
|
+
history.
|
|
254
|
+
|
|
255
|
+
If there is a previous state in the state history, an actual transition is added to the state machine graph,
|
|
256
|
+
connecting the previous state to the current state.
|
|
257
|
+
|
|
258
|
+
Raises:
|
|
259
|
+
ValueError: If the state evaluation function is not set or not callable.
|
|
260
|
+
"""
|
|
261
|
+
if not callable(cls.state_eval_function):
|
|
262
|
+
raise ValueError("State evaluation function is not set or not callable.")
|
|
263
|
+
|
|
264
|
+
possible_states = [
|
|
265
|
+
{state: cls._graph.nodes[state]["cls"].description}
|
|
266
|
+
for state in cls._graph.nodes
|
|
267
|
+
]
|
|
268
|
+
cls._current_state, cur_state_description = cls.state_eval_function(
|
|
269
|
+
possible_states, cls.state_eval_output_model
|
|
270
|
+
)
|
|
271
|
+
logger.debug(
|
|
272
|
+
f"Current state: {cls._current_state}\nDescription: {cur_state_description}",
|
|
273
|
+
)
|
|
274
|
+
cls._log_state(cls._current_state)
|
|
275
|
+
cls._acted_since_state_eval = False
|
|
276
|
+
try:
|
|
277
|
+
screenshot_and_log(f"State_{cls._current_state}")
|
|
278
|
+
except Exception as e:
|
|
279
|
+
logger.warning("Error occurred while taking and saving screenshot")
|
|
280
|
+
logger.warning(f"Details: {str(e)}")
|
|
281
|
+
|
|
282
|
+
if len(cls._state_history) > 1 and len(cls._transition_history) > 0:
|
|
283
|
+
key = f"{cls._transition_history[-1].__name__}_from_{cls._state_history[-2]}_to_{cls._current_state}_actual"
|
|
284
|
+
cls._graph.add_edge(
|
|
285
|
+
cls._state_history[-2], cls._current_state, key=key, mode="actual"
|
|
286
|
+
)
|
|
287
|
+
|
|
288
|
+
@staticmethod
|
|
289
|
+
def _find_unplanned_transitions(graph: nx.MultiDiGraph) -> List[Tuple[str, str]]:
|
|
290
|
+
"""
|
|
291
|
+
Identify transitions in the graph which were not originally planned.
|
|
292
|
+
|
|
293
|
+
Parameters:
|
|
294
|
+
graph (nx.MultiDiGraph): The graph representing the state machine.
|
|
295
|
+
|
|
296
|
+
Returns:
|
|
297
|
+
List[Tuple[str, str]]: A list of tuples representing the unplanned transitions. Each tuple contains the source state and the destination state of the transition.
|
|
298
|
+
|
|
299
|
+
"""
|
|
300
|
+
actual_edges = [
|
|
301
|
+
(u, v)
|
|
302
|
+
for u, v, k, data in graph.edges(keys=True, data=True)
|
|
303
|
+
if data.get("mode") == "actual"
|
|
304
|
+
]
|
|
305
|
+
planned_edges = set(
|
|
306
|
+
[
|
|
307
|
+
(u, v)
|
|
308
|
+
for u, v, k, data in graph.edges(keys=True, data=True)
|
|
309
|
+
if data.get("mode") == "planned"
|
|
310
|
+
]
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
# Filter out actual edges that have a corresponding planned edge
|
|
314
|
+
unplanned_transitions = [
|
|
315
|
+
edge for edge in actual_edges if edge not in planned_edges
|
|
316
|
+
]
|
|
317
|
+
|
|
318
|
+
if unplanned_transitions:
|
|
319
|
+
logger.info(f"Found unplanned transitions: {unplanned_transitions}")
|
|
320
|
+
|
|
321
|
+
return unplanned_transitions
|
|
322
|
+
|
|
323
|
+
@staticmethod
|
|
324
|
+
def _find_repeated_transitions(transition_history: deque) -> List[str]:
|
|
325
|
+
"""
|
|
326
|
+
Identify repeated transitions in the transition history.
|
|
327
|
+
|
|
328
|
+
This method takes in a deque object representing the transition history and identifies any repeated transitions. It iterates over each item in the transition history and checks if the count of that item is greater than 1. If a repeated transition is found, it is added to the list of repeated transitions.
|
|
329
|
+
|
|
330
|
+
Parameters:
|
|
331
|
+
transition_history (deque): A deque object representing the transition history.
|
|
332
|
+
|
|
333
|
+
Returns:
|
|
334
|
+
List[str]: A list of strings representing the repeated transitions.
|
|
335
|
+
|
|
336
|
+
"""
|
|
337
|
+
repeated_transitions = [
|
|
338
|
+
item.__name__
|
|
339
|
+
for item in transition_history
|
|
340
|
+
if transition_history.count(item) > 1
|
|
341
|
+
]
|
|
342
|
+
if repeated_transitions:
|
|
343
|
+
logger.info(f"Found repeated transitions: {repeated_transitions}")
|
|
344
|
+
return repeated_transitions
|
|
345
|
+
|
|
346
|
+
@staticmethod
|
|
347
|
+
def _find_repeated_states(state_history) -> List[str]:
|
|
348
|
+
"""
|
|
349
|
+
This method identifies repeated states in the state history.
|
|
350
|
+
Parameters:
|
|
351
|
+
state_history (deque): A deque object representing the state history.
|
|
352
|
+
Returns:
|
|
353
|
+
List[str]: A list of strings representing the repeated states.
|
|
354
|
+
"""
|
|
355
|
+
repeated_states = [
|
|
356
|
+
item for item in state_history if state_history.count(item) > 1
|
|
357
|
+
]
|
|
358
|
+
if repeated_states:
|
|
359
|
+
logger.info(f"Found repeated states: {repeated_states}")
|
|
360
|
+
return repeated_states
|
|
361
|
+
|
|
362
|
+
@classmethod
|
|
363
|
+
def _find_anti_patterns(cls) -> bool:
|
|
364
|
+
"""
|
|
365
|
+
Detect abnormal patterns in the state machine execution and take appropriate action.
|
|
366
|
+
|
|
367
|
+
Returns:
|
|
368
|
+
bool: Whether an anti-pattern was detected.
|
|
369
|
+
"""
|
|
370
|
+
# Detect deviation from planned transitions
|
|
371
|
+
unplanned_transitions: List = cls._find_unplanned_transitions(cls._graph)
|
|
372
|
+
if len(unplanned_transitions) > cls.tolerate_unplanned_transitions:
|
|
373
|
+
cls._exit_reason = UnplannedTransitionsError(
|
|
374
|
+
f"{len(unplanned_transitions)} unplanned transitions detected: {unplanned_transitions}."
|
|
375
|
+
)
|
|
376
|
+
raise cls._exit_reason
|
|
377
|
+
|
|
378
|
+
repeated_transitions: List = cls._find_repeated_transitions(
|
|
379
|
+
cls._transition_history
|
|
380
|
+
)
|
|
381
|
+
if (
|
|
382
|
+
len(repeated_transitions) > cls.tolerate_repeat_transitions + 1
|
|
383
|
+
): # +1 to account for the first transition
|
|
384
|
+
cls._exit_reason = RepeatTransitions(
|
|
385
|
+
f"{len(repeated_transitions)} repeated transitions detected: {repeated_transitions}."
|
|
386
|
+
)
|
|
387
|
+
raise cls._exit_reason
|
|
388
|
+
|
|
389
|
+
repeated_states: List = cls._find_repeated_states(cls._state_history)
|
|
390
|
+
if len(repeated_states) > cls.tolerate_repeat_states:
|
|
391
|
+
cls._exit_reason = RepeatStatesError(
|
|
392
|
+
f"{len(repeated_states)} repeated states detected: {repeated_states}."
|
|
393
|
+
)
|
|
394
|
+
if cls.enable_force_close_app_process and cls.process_name:
|
|
395
|
+
logger.info(
|
|
396
|
+
f"RepeatStatesError detected. Killing application with process name: {cls.process_name}",
|
|
397
|
+
)
|
|
398
|
+
try:
|
|
399
|
+
ForceCloseApplication(process_name=cls.process_name).do()
|
|
400
|
+
except Exception as e:
|
|
401
|
+
logger.warning("Killing application failed. Details")
|
|
402
|
+
logger.warning(f"{str(e)}")
|
|
403
|
+
elif cls.enable_force_close_app_process and not cls.process_name:
|
|
404
|
+
logger.warning(
|
|
405
|
+
"RepeatStatesError detected. Application cannot be killed as process_name was not provided.",
|
|
406
|
+
)
|
|
407
|
+
raise cls._exit_reason
|
|
408
|
+
|
|
409
|
+
# No anti-patterns detected
|
|
410
|
+
return False
|
|
411
|
+
|
|
412
|
+
@classmethod
|
|
413
|
+
def configure(
|
|
414
|
+
cls,
|
|
415
|
+
**kwargs,
|
|
416
|
+
) -> None:
|
|
417
|
+
"""
|
|
418
|
+
Configure the state machine.
|
|
419
|
+
|
|
420
|
+
Parameters:
|
|
421
|
+
**kwargs: Additional keyword arguments for the configuration.
|
|
422
|
+
|
|
423
|
+
Returns:
|
|
424
|
+
None
|
|
425
|
+
"""
|
|
426
|
+
for k, v in kwargs.items():
|
|
427
|
+
if hasattr(cls, k) and not k.startswith("_"):
|
|
428
|
+
setattr(cls, k, v)
|
|
429
|
+
else:
|
|
430
|
+
logger.error(f"Public attribute {k} not found in ScreenPilot.")
|
|
431
|
+
return None
|
|
432
|
+
|
|
433
|
+
@classmethod
|
|
434
|
+
def _evaluate_goal_function(cls, goal_function: Callable, **kwargs) -> None:
|
|
435
|
+
"""
|
|
436
|
+
Evaluate the success of the current state machine execution.
|
|
437
|
+
|
|
438
|
+
Args:
|
|
439
|
+
goal_function: A function that evaluates the completion and success of the current state machine execution.
|
|
440
|
+
Args: current_state (class), **kwargs.
|
|
441
|
+
Returns: Tuple[bool, bool].
|
|
442
|
+
**kwargs: Additional arguments for the goal function.
|
|
443
|
+
|
|
444
|
+
Returns:
|
|
445
|
+
None
|
|
446
|
+
"""
|
|
447
|
+
# Filter out unsupported kwargs
|
|
448
|
+
sig = inspect.signature(goal_function)
|
|
449
|
+
supported_params = sig.parameters
|
|
450
|
+
filtered_kwargs = {k: v for k, v in kwargs.items() if k in supported_params}
|
|
451
|
+
goal_function(cls._current_state, **filtered_kwargs)
|
|
452
|
+
return None
|
|
453
|
+
|
|
454
|
+
@classmethod
|
|
455
|
+
def _act_on_start(cls) -> None:
|
|
456
|
+
"""
|
|
457
|
+
Actions to be taken when the state machine execution is started.
|
|
458
|
+
|
|
459
|
+
Returns:
|
|
460
|
+
None
|
|
461
|
+
"""
|
|
462
|
+
cls._exit_reason = None
|
|
463
|
+
cls._current_state = None
|
|
464
|
+
cls._final_state = None
|
|
465
|
+
logger.info("State machine execution started.")
|
|
466
|
+
|
|
467
|
+
@classmethod
|
|
468
|
+
def _act_on_completed(cls) -> None:
|
|
469
|
+
"""
|
|
470
|
+
Actions to be taken when the state machine execution is completed.
|
|
471
|
+
|
|
472
|
+
This method performs the following actions:
|
|
473
|
+
- Sets the final state of the state machine to the current state.
|
|
474
|
+
- Clears the transition history and state history.
|
|
475
|
+
- Removes the actual transitions from the state machine graph.
|
|
476
|
+
- Resets the remaining AI recovery attempts to the initial value.
|
|
477
|
+
- Logs a message indicating that the state machine execution is completed and cleanup has been performed.
|
|
478
|
+
|
|
479
|
+
Returns:
|
|
480
|
+
None
|
|
481
|
+
"""
|
|
482
|
+
try:
|
|
483
|
+
cls._ensure_allowed_end_state()
|
|
484
|
+
except Exception as e:
|
|
485
|
+
logger.error(f"Error during end state check: {e}")
|
|
486
|
+
raise e
|
|
487
|
+
finally:
|
|
488
|
+
cls._final_state = cls._current_state
|
|
489
|
+
cls._transition_history.clear()
|
|
490
|
+
cls._state_history.clear()
|
|
491
|
+
# remove actual transitions from the graph
|
|
492
|
+
cls._graph.remove_edges_from(
|
|
493
|
+
[
|
|
494
|
+
(u, v, k)
|
|
495
|
+
for u, v, k, data in cls._graph.edges(keys=True, data=True)
|
|
496
|
+
if data.get("mode") == "actual"
|
|
497
|
+
]
|
|
498
|
+
)
|
|
499
|
+
cls._mode = "planned"
|
|
500
|
+
cls._runtime_error_details = None
|
|
501
|
+
cls._ai_recovery_agent = None
|
|
502
|
+
cls._next_target_state = None
|
|
503
|
+
|
|
504
|
+
logger.info("State machine execution completed, cleanup performed.")
|
|
505
|
+
return None
|
|
506
|
+
|
|
507
|
+
@classmethod
|
|
508
|
+
def _write_ai_recovery_prompt(
|
|
509
|
+
cls,
|
|
510
|
+
scenario: Literal["exit", "runtime_error", "unexpected_state"],
|
|
511
|
+
error_details: Optional[str] = None,
|
|
512
|
+
) -> str:
|
|
513
|
+
# If no transition history is available, we are dealing with an error from the previous instance
|
|
514
|
+
if scenario == "unexpected_state" and not cls._transition_history:
|
|
515
|
+
start_nodes = ", ".join(
|
|
516
|
+
[
|
|
517
|
+
f"{node} ({data.get('description')})"
|
|
518
|
+
for node, data in cls._graph.nodes(data=True)
|
|
519
|
+
if data.get("start_allowed") == True
|
|
520
|
+
]
|
|
521
|
+
)
|
|
522
|
+
action_prompt = f"The process just started but the system is in an unexpected state. We need to return to one of the known states: {start_nodes}."
|
|
523
|
+
# If the ScreenPilot is exiting, we just need to clean up the system for the next instance
|
|
524
|
+
elif scenario == "exit":
|
|
525
|
+
end_nodes = ", ".join(
|
|
526
|
+
[
|
|
527
|
+
f"{node} ({data.get('description')})"
|
|
528
|
+
for node, data in cls._graph.nodes(data=True)
|
|
529
|
+
if data.get("end_allowed") == True
|
|
530
|
+
]
|
|
531
|
+
)
|
|
532
|
+
action_prompt = f"The process has completed and we need to bring the system back to a state where we can exit safely: {end_nodes}."
|
|
533
|
+
# If we have error details, we assume an error during a transition
|
|
534
|
+
elif scenario == "runtime_error":
|
|
535
|
+
attempted_transition = cls._transition_history[-1]
|
|
536
|
+
attempted_transition_description = attempted_transition.__doc__
|
|
537
|
+
previous_state = cls._state_history[-1]
|
|
538
|
+
previous_state_description = next(
|
|
539
|
+
(
|
|
540
|
+
data.get("description")
|
|
541
|
+
for node, data in cls._graph.nodes(data=True)
|
|
542
|
+
if node == previous_state
|
|
543
|
+
),
|
|
544
|
+
"", # Default value if no description is found
|
|
545
|
+
)
|
|
546
|
+
action_prompt = f"We tried performing '{attempted_transition.__name__}' ({attempted_transition_description}) but ran into an error: {error_details}. Could you try and remove possible causes of the error? It could be e.g. a popup, or window focus. We want to return to the previous state, {previous_state} ({previous_state_description})."
|
|
547
|
+
# Otherewise we assume atransition has completed but the resulting screen is not as expected
|
|
548
|
+
else:
|
|
549
|
+
attempted_transition = cls._transition_history[-1]
|
|
550
|
+
attempted_transition_description = attempted_transition.__doc__
|
|
551
|
+
target_state = cls._next_target_state
|
|
552
|
+
target_state_description = next(
|
|
553
|
+
(
|
|
554
|
+
data.get("description")
|
|
555
|
+
for node, data in cls._graph.nodes(data=True)
|
|
556
|
+
if node == target_state
|
|
557
|
+
),
|
|
558
|
+
"", # Default value if no description is found
|
|
559
|
+
)
|
|
560
|
+
action_prompt = f"We tried performing '{attempted_transition.__name__}' ({attempted_transition_description}) and expected to see state '{target_state}' ({target_state_description}) but the screen does not match the expected state."
|
|
561
|
+
return action_prompt
|
|
562
|
+
|
|
563
|
+
@classmethod
|
|
564
|
+
def _attempt_ai_recovery(
|
|
565
|
+
cls,
|
|
566
|
+
scenario: Literal["exit", "runtime_error", "unexpected_state"],
|
|
567
|
+
attempts: int = 5,
|
|
568
|
+
error_details: Optional[str] = None,
|
|
569
|
+
) -> None:
|
|
570
|
+
"""
|
|
571
|
+
Attempt AI recovery in case of an error state.
|
|
572
|
+
Args:
|
|
573
|
+
attempts (int, optional): The number of attempts to recover using AI.
|
|
574
|
+
error_details (str, optional): Details of the error that occurred.
|
|
575
|
+
Returns:
|
|
576
|
+
None
|
|
577
|
+
"""
|
|
578
|
+
# import actions used in AI recovery
|
|
579
|
+
from ..ui_actions import (
|
|
580
|
+
LeftClick,
|
|
581
|
+
DoubleClick,
|
|
582
|
+
SendKeys,
|
|
583
|
+
PressKeys,
|
|
584
|
+
ActivateWindow,
|
|
585
|
+
CloseWindow,
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
# Initialize CourseCorrector, if not already initialized
|
|
589
|
+
if not cls._ai_recovery_agent and cls.ai_recovery_agent_factory is not None:
|
|
590
|
+
action_prompt = cls._write_ai_recovery_prompt(scenario, error_details)
|
|
591
|
+
cls._ai_recovery_agent = cls.ai_recovery_agent_factory(
|
|
592
|
+
goal=action_prompt, custom_instructions=cls.ai_recovery_instructions
|
|
593
|
+
)
|
|
594
|
+
if cls._ai_recovery_agent is not None:
|
|
595
|
+
logger.debug(
|
|
596
|
+
f"AI recovery agent {cls._ai_recovery_agent.name} initialized with prompt: '{action_prompt}'",
|
|
597
|
+
)
|
|
598
|
+
if cls._ai_recovery_agent is not None:
|
|
599
|
+
attempts_made = 0
|
|
600
|
+
|
|
601
|
+
while attempts_made < attempts:
|
|
602
|
+
attempts_made += 1
|
|
603
|
+
logger.info(
|
|
604
|
+
f"AI recovery attempt {attempts_made} / {attempts}",
|
|
605
|
+
)
|
|
606
|
+
action: Optional[ActionString] = None
|
|
607
|
+
try:
|
|
608
|
+
# Get corrective actions from the course corrector and execute the first one
|
|
609
|
+
action = cls._ai_recovery_agent.get_corrective_actions()[0]
|
|
610
|
+
logger.info(f"Corrective action: {action}")
|
|
611
|
+
|
|
612
|
+
# If no action is suggested, break the loop and return None
|
|
613
|
+
if action.action_string.startswith("NoAction"):
|
|
614
|
+
return None
|
|
615
|
+
|
|
616
|
+
try:
|
|
617
|
+
eval(action.action_string)
|
|
618
|
+
cls._acted_since_state_eval = True
|
|
619
|
+
break
|
|
620
|
+
except (
|
|
621
|
+
AttributeError,
|
|
622
|
+
SyntaxError,
|
|
623
|
+
ValidationError,
|
|
624
|
+
NameError,
|
|
625
|
+
) as e:
|
|
626
|
+
cls._ai_recovery_agent.add_feedback(str(e))
|
|
627
|
+
logger.error(f"Error during AI recovery: {e}")
|
|
628
|
+
logger.info(
|
|
629
|
+
f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
|
|
630
|
+
)
|
|
631
|
+
continue
|
|
632
|
+
except RuntimeError:
|
|
633
|
+
cls._ai_recovery_agent.add_feedback(
|
|
634
|
+
"The target could not be uniquely identified. Try using different anchors or target."
|
|
635
|
+
)
|
|
636
|
+
logger.error("Runtime error during AI recovery.")
|
|
637
|
+
logger.info(
|
|
638
|
+
f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
|
|
639
|
+
)
|
|
640
|
+
continue
|
|
641
|
+
except PerformActionException:
|
|
642
|
+
cls._ai_recovery_agent.add_feedback(
|
|
643
|
+
"The action could not be performed."
|
|
644
|
+
)
|
|
645
|
+
logger.error("PerformActionException during AI recovery.")
|
|
646
|
+
logger.info(
|
|
647
|
+
f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
|
|
648
|
+
)
|
|
649
|
+
continue
|
|
650
|
+
|
|
651
|
+
except Exception as e:
|
|
652
|
+
logger.error(f"Unexpected error during AI recovery: {e}")
|
|
653
|
+
logger.info("Interrupting AI recovery.")
|
|
654
|
+
break
|
|
655
|
+
finally:
|
|
656
|
+
# Interrupt the process if the action is impossible to process
|
|
657
|
+
if not action:
|
|
658
|
+
raise CourseCorrectionImpossible(
|
|
659
|
+
"No course correction action could be generated, see logs for details."
|
|
660
|
+
)
|
|
661
|
+
elif action.interrupt_process:
|
|
662
|
+
raise CourseCorrectionImpossible(action.observation)
|
|
663
|
+
screenshot_and_log("Completed AI recovery")
|
|
664
|
+
cls._ai_recovery_agent.reset_feedback()
|
|
665
|
+
|
|
666
|
+
@classmethod
|
|
667
|
+
def _get_next_transition(cls, **kwargs) -> Tuple[Callable, str]:
|
|
668
|
+
"""
|
|
669
|
+
Get the next transition function based on the current state.
|
|
670
|
+
Args:
|
|
671
|
+
**kwargs: Additional arguments for the condition functions.
|
|
672
|
+
|
|
673
|
+
Returns:
|
|
674
|
+
Callable: The next transition function.
|
|
675
|
+
str: The next state.
|
|
676
|
+
"""
|
|
677
|
+
possible_transitions = cls._graph.out_edges(
|
|
678
|
+
cls._current_state, keys=True, data=True
|
|
679
|
+
)
|
|
680
|
+
if not possible_transitions:
|
|
681
|
+
logger.error(
|
|
682
|
+
f"No available {cls._mode} transitions from {cls._current_state}.",
|
|
683
|
+
)
|
|
684
|
+
for _, to_state, key, data in possible_transitions:
|
|
685
|
+
if data.get("mode") == cls._mode:
|
|
686
|
+
# Execute the first transition which has no condition or has a condition that evaluates to True
|
|
687
|
+
condition: Optional[Callable] = data.get("condition", None)
|
|
688
|
+
if not condition:
|
|
689
|
+
return data.get("func"), to_state
|
|
690
|
+
else:
|
|
691
|
+
# Argument filtering
|
|
692
|
+
sig = inspect.signature(condition)
|
|
693
|
+
supported_params = sig.parameters
|
|
694
|
+
filtered_kwargs = {
|
|
695
|
+
k: v for k, v in kwargs.items() if k in supported_params
|
|
696
|
+
}
|
|
697
|
+
if condition(**filtered_kwargs):
|
|
698
|
+
return data.get("func"), to_state
|
|
699
|
+
|
|
700
|
+
# raise an error to prevent failing in run()
|
|
701
|
+
err_msg = f"Cannot get a valid transition from {cls._current_state} in {cls._mode} mode. {len(possible_transitions)} transitions available, but none of them match the mode and conditions."
|
|
702
|
+
logger.error(err_msg)
|
|
703
|
+
raise ValueError(err_msg)
|
|
704
|
+
|
|
705
|
+
@staticmethod
|
|
706
|
+
def _default_goal_function(current_state: str) -> None:
|
|
707
|
+
"""
|
|
708
|
+
Default goal function for the state machine.
|
|
709
|
+
|
|
710
|
+
Args:
|
|
711
|
+
current_state (str): The current state of the state machine.
|
|
712
|
+
**kwargs: Additional arguments for the goal function.
|
|
713
|
+
|
|
714
|
+
Returns:
|
|
715
|
+
None
|
|
716
|
+
"""
|
|
717
|
+
return None
|
|
718
|
+
|
|
719
|
+
@classmethod
|
|
720
|
+
def run(
|
|
721
|
+
cls, goal_function: Optional[Callable] = None, **kwargs
|
|
722
|
+
) -> Union[ScreenPilotOutcome, ScreenPilotException]:
|
|
723
|
+
"""
|
|
724
|
+
Main loop of the state machine.
|
|
725
|
+
|
|
726
|
+
Args:
|
|
727
|
+
goal_function (Optional[Callable]): Function to evaluate the success of the current state machine execution. If no goal function is provided, the complete_ui_automation() method must be called from a transition.
|
|
728
|
+
Args: current_state (str), **kwargs.
|
|
729
|
+
Returns: bool.
|
|
730
|
+
**kwargs: Additional arguments for the goal function and the transition functions (will be dynamically filtered as needed).
|
|
731
|
+
|
|
732
|
+
Returns:
|
|
733
|
+
- Exit reason (ScreenPilotOutcome): Exception class that interrupted the state machine execution.
|
|
734
|
+
"""
|
|
735
|
+
if not goal_function:
|
|
736
|
+
goal_function = cls._default_goal_function
|
|
737
|
+
cls._act_on_start()
|
|
738
|
+
try:
|
|
739
|
+
while not cls._exit_reason:
|
|
740
|
+
# Detect anti-patterns (raises exceptions if detected)
|
|
741
|
+
cls._find_anti_patterns()
|
|
742
|
+
|
|
743
|
+
# Evaluate the current state and check for completion
|
|
744
|
+
cls._evaluate_state()
|
|
745
|
+
cls._evaluate_goal_function(goal_function, **kwargs)
|
|
746
|
+
|
|
747
|
+
# Attempt AI recovery if required
|
|
748
|
+
if cls._current_state == "Error" and cls.ai_recovery:
|
|
749
|
+
cls._attempt_ai_recovery(
|
|
750
|
+
scenario="unexpected_state", attempts=cls.ai_recovery_attempts
|
|
751
|
+
)
|
|
752
|
+
continue
|
|
753
|
+
# Reset course corrector if the state is not Error
|
|
754
|
+
else:
|
|
755
|
+
cls._ai_recovery_agent = None
|
|
756
|
+
|
|
757
|
+
# Find possible transitions from the current state
|
|
758
|
+
if cls._current_state not in cls._graph:
|
|
759
|
+
err_msg: str = (
|
|
760
|
+
f"Current state {cls._current_state} is not recognized."
|
|
761
|
+
)
|
|
762
|
+
logger.error(err_msg)
|
|
763
|
+
raise ValueError(err_msg)
|
|
764
|
+
next_transition, cls._next_target_state = cls._get_next_transition(
|
|
765
|
+
**kwargs
|
|
766
|
+
)
|
|
767
|
+
|
|
768
|
+
# Execute the transition function
|
|
769
|
+
cls._log_transition(next_transition)
|
|
770
|
+
cls._acted_since_state_eval = True
|
|
771
|
+
next_transition(**kwargs) # Execute the transition function
|
|
772
|
+
|
|
773
|
+
except (BusinessException, SuccessfulCompletion) as e:
|
|
774
|
+
cls._exit_reason = e
|
|
775
|
+
except Exception as e:
|
|
776
|
+
logger.error(traceback.format_exc())
|
|
777
|
+
raise e
|
|
778
|
+
finally:
|
|
779
|
+
cls._act_on_completed()
|
|
780
|
+
return cls._exit_reason
|
|
781
|
+
|
|
782
|
+
@classmethod
|
|
783
|
+
def _ensure_allowed_end_state(cls):
|
|
784
|
+
# If actions have been performed since the last state evaluation, evaluate the state again
|
|
785
|
+
if cls._acted_since_state_eval:
|
|
786
|
+
cls._evaluate_state()
|
|
787
|
+
end_states = [
|
|
788
|
+
state
|
|
789
|
+
for state, data in cls._graph.nodes(data=True)
|
|
790
|
+
if data.get("end_allowed") == True
|
|
791
|
+
]
|
|
792
|
+
if cls._current_state not in end_states and cls.ai_recovery:
|
|
793
|
+
cls._attempt_ai_recovery(scenario="exit", attempts=cls.ai_recovery_attempts)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _action_line_from_exc() -> Tuple[str, str, str]:
|
|
797
|
+
"""
|
|
798
|
+
Format the current exception to extract the action line and action string.
|
|
799
|
+
Returns:
|
|
800
|
+
Tuple[str, str, str]: Tuple with the full traceback, line with action, and action string.
|
|
801
|
+
"""
|
|
802
|
+
import sys
|
|
803
|
+
|
|
804
|
+
(
|
|
805
|
+
exc_type,
|
|
806
|
+
exc_value,
|
|
807
|
+
exc_traceback,
|
|
808
|
+
) = sys.exc_info() # Capture the current exception info
|
|
809
|
+
tb_lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
|
|
810
|
+
output = (
|
|
811
|
+
"\n".join(tb_lines),
|
|
812
|
+
"",
|
|
813
|
+
"",
|
|
814
|
+
) # Tuple: (full traceback, line with action, action)
|
|
815
|
+
for line in tb_lines:
|
|
816
|
+
if ".do(" in line:
|
|
817
|
+
import re
|
|
818
|
+
|
|
819
|
+
# Match anything after an action type and before do()
|
|
820
|
+
match = re.search(
|
|
821
|
+
r"(LeftClick|PressKeys|SendKeys|DoubleClick|WaitFor|RightClick|OpenApplication|MaximizeWindow|CloseWindow|ActivateWindow|SaveFiles|DeleteFiles|\(.*?\)\.do\((.*)\))",
|
|
822
|
+
line,
|
|
823
|
+
)
|
|
824
|
+
if match:
|
|
825
|
+
output = (output[0], line, match.group(0))
|
|
826
|
+
break
|
|
827
|
+
return output
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def screenshot_and_log(message: str, screenshot_folder: str = "screenshots") -> None:
|
|
831
|
+
logger.info(message)
|
|
832
|
+
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
|
|
833
|
+
screenshot_name = f"{message.replace(' ', '_')}.png"
|
|
834
|
+
save_screenshot(f"{timestamp}_{screenshot_name}", screenshot_folder)
|
|
835
|
+
logger.info(
|
|
836
|
+
f"Saved screenshot {timestamp}_{screenshot_name} in {screenshot_folder}."
|
|
837
|
+
)
|
|
838
|
+
return None
|