clerk-sdk 0.1.9__py3-none-any.whl → 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. clerk/base.py +94 -0
  2. clerk/client.py +3 -104
  3. clerk/decorator/models.py +1 -0
  4. clerk/decorator/task_decorator.py +1 -0
  5. clerk/gui_automation/__init__.py +0 -0
  6. clerk/gui_automation/action_model/__init__.py +0 -0
  7. clerk/gui_automation/action_model/model.py +126 -0
  8. clerk/gui_automation/action_model/utils.py +26 -0
  9. clerk/gui_automation/client.py +144 -0
  10. clerk/gui_automation/client_actor/__init__.py +4 -0
  11. clerk/gui_automation/client_actor/client_actor.py +178 -0
  12. clerk/gui_automation/client_actor/exception.py +22 -0
  13. clerk/gui_automation/client_actor/model.py +192 -0
  14. clerk/gui_automation/decorators/__init__.py +1 -0
  15. clerk/gui_automation/decorators/gui_automation.py +109 -0
  16. clerk/gui_automation/exceptions/__init__.py +0 -0
  17. clerk/gui_automation/exceptions/modality/__init__.py +0 -0
  18. clerk/gui_automation/exceptions/modality/exc.py +46 -0
  19. clerk/gui_automation/exceptions/websocket.py +6 -0
  20. clerk/gui_automation/ui_actions/__init__.py +1 -0
  21. clerk/gui_automation/ui_actions/actions.py +781 -0
  22. clerk/gui_automation/ui_actions/base.py +200 -0
  23. clerk/gui_automation/ui_actions/support.py +68 -0
  24. clerk/gui_automation/ui_state_inspector/__init__.py +0 -0
  25. clerk/gui_automation/ui_state_inspector/gui_vision.py +184 -0
  26. clerk/gui_automation/ui_state_inspector/models.py +184 -0
  27. clerk/gui_automation/ui_state_machine/__init__.py +11 -0
  28. clerk/gui_automation/ui_state_machine/ai_recovery.py +110 -0
  29. clerk/gui_automation/ui_state_machine/decorators.py +71 -0
  30. clerk/gui_automation/ui_state_machine/exceptions.py +42 -0
  31. clerk/gui_automation/ui_state_machine/models.py +40 -0
  32. clerk/gui_automation/ui_state_machine/state_machine.py +838 -0
  33. clerk/models/remote_device.py +7 -0
  34. clerk/utils/__init__.py +0 -0
  35. clerk/utils/logger.py +118 -0
  36. clerk/utils/save_artifact.py +35 -0
  37. {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/METADATA +11 -1
  38. clerk_sdk-0.2.0.dist-info/RECORD +48 -0
  39. clerk_sdk-0.1.9.dist-info/RECORD +0 -15
  40. {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/WHEEL +0 -0
  41. {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/licenses/LICENSE +0 -0
  42. {clerk_sdk-0.1.9.dist-info → clerk_sdk-0.2.0.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,838 @@
1
+ import os
2
+ import networkx as nx # type: ignore
3
+ import functools
4
+ import inspect
5
+ from collections import deque
6
+ from pydantic import BaseModel, ValidationError
7
+ from typing import List, Tuple, Callable, Optional, Deque, Literal, Union
8
+ import traceback
9
+ from datetime import datetime
10
+
11
+ from ..ui_actions.actions import ForceCloseApplication
12
+ from ..ui_actions.support import save_screenshot
13
+ from .exceptions import (
14
+ BusinessException,
15
+ ScreenPilotException,
16
+ UnplannedTransitionsError,
17
+ RepeatTransitions,
18
+ RollbackCompleted,
19
+ RepeatStatesError,
20
+ ScreenPilotOutcome,
21
+ CourseCorrectionImpossible,
22
+ SuccessfulCompletion,
23
+ )
24
+ from ..ui_state_machine.models import ActionString
25
+ from .ai_recovery import CourseCorrector, course_corrector_v1
26
+ from ..client_actor.exception import PerformActionException
27
+ from ..ui_state_inspector.gui_vision import Vision
28
+ from ...client import Clerk
29
+ from ...utils import logger
30
+
31
+
32
+ class ScreenPilot:
33
+ """
34
+ A class representing a screen pilot state machine.
35
+
36
+ Attributes:
37
+ ai_recovery (bool): Whether to use AI recovery on errors. Default True.
38
+ ai_recovery_attempts (int): The number of AI recovery attempts. Default 5.
39
+ ai_recovery_agent_factory (Optional[Callable]): Factory for AI recovery agents. Default course_corrector_v1.
40
+ ai_recovery_instructions (Optional[str]): Custom instructions for the AI recovery agent. Default None.
41
+ state_eval_function (Callable): Function to evaluate the current state of the state machine. Default Vision().classify_state.
42
+ state_eval_output_model (BaseModel): Optional custom output model of the state evaluation function. Default None.
43
+ tolerate_unplanned_transitions (int): Number of unplanned transitions to tolerate before breaking the execution. Default 5.
44
+ tolerate_repeat_transitions (int): Number of repeated transitions to tolerate before breaking the execution. Default 5.
45
+ tolerate_repeat_states (int): Number of repeated states to tolerate before breaking the execution. Default 5.
46
+ enable_force_close_app_process (bool): If true, terminates the application process via `taskkill` command. Default False.
47
+ process_name (Optional[str]): Name of the process that needs to be closed (ie. process.exe). Required attribute if `enable_force_close_app_process` is True
48
+
49
+ Methods:
50
+ register_state(cls, state_cls): Register a state class in the state machine graph.
51
+ register_transition(cls, from_state, to_state): Register a transition function in the state machine graph.
52
+ configure(cls, **kwargs): Provide class parameters to configure the state machine.
53
+ run(cls, goal_function, **kwargs): Main loop of the state machine.
54
+ """
55
+
56
+ ai_recovery: bool = True
57
+ ai_recovery_attempts: int = 5
58
+ ai_recovery_agent_factory: Optional[Callable] = course_corrector_v1
59
+ ai_recovery_instructions: Optional[str] = None
60
+ state_eval_function: Optional[Callable] = Vision().classify_state
61
+ state_eval_output_model: Optional[BaseModel] = None
62
+ tolerate_unplanned_transitions: int = 5
63
+ tolerate_repeat_transitions: int = 5
64
+ tolerate_repeat_states: int = 5
65
+ enable_force_close_app_process: bool = False
66
+ process_name: Optional[str] = None
67
+ _acted_since_state_eval: bool = False
68
+ _ai_recovery_agent: Optional[CourseCorrector] = None
69
+ _current_state: Optional[str] = None
70
+ _exit_reason: Optional[Union[ScreenPilotOutcome, ScreenPilotException]] = None
71
+ _final_state: Optional[str] = None
72
+ _graph = nx.MultiDiGraph()
73
+ _mode: Literal["planned", "rollback"] = "planned"
74
+ _next_target_state: Optional[str] = None
75
+ _runtime_error_details: Optional[Tuple[str, str, str]] = None
76
+ _state_history: Deque[str] = deque(maxlen=25)
77
+ _transition_history: Deque[Callable] = deque(maxlen=25)
78
+ _clerk_client: Clerk = Clerk(
79
+ base_url=os.getenv("CLERK_BASE_URL", "https://api.clerk-app.com")
80
+ )
81
+
82
+ @classmethod
83
+ def register_state(
84
+ cls, state_cls, start_allowed: bool = True, end_allowed: bool = True
85
+ ):
86
+ """
87
+ Register a state class in the state machine graph.
88
+
89
+ Parameters:
90
+ state_cls (class): The state class to be registered.
91
+
92
+ Returns:
93
+ class: The registered state class.
94
+
95
+ """
96
+ cls._graph.add_node(
97
+ state_cls.__name__,
98
+ cls=state_cls,
99
+ description=state_cls.description,
100
+ start_allowed=start_allowed,
101
+ end_allowed=end_allowed,
102
+ )
103
+ return state_cls
104
+
105
+ @classmethod
106
+ def register_transition(
107
+ cls,
108
+ from_state: str,
109
+ to_state: str,
110
+ mode: Literal["planned", "rollback"] = "planned",
111
+ condition: Optional[Callable] = None,
112
+ ):
113
+ """
114
+ Register a transition function in the state machine graph.
115
+
116
+ Provides a decorator with logging and error handling.
117
+
118
+ Parameters:
119
+ from_state (str): The state from which the transition occurs.
120
+ to_state (str): The state to which the transition leads.
121
+ mode (Literal["planned", "rollback"]): The mode of the transition. Default "planned".
122
+ condition (Optional[Callable]): A condition function for the transition. Should return boolean. Default None.
123
+
124
+ Returns:
125
+ function: The decorated transition function.
126
+
127
+ Raises:
128
+ ValueError: If either the from_state or to_state is not a defined state in the state machine graph.
129
+
130
+ Example:
131
+ @register_transition("StateA", "StateB", mode="planned", condition=None)
132
+ def transition_func():
133
+ # Transition logic here
134
+ pass
135
+ """
136
+
137
+ def decorator(transition_func):
138
+ @functools.wraps(transition_func)
139
+ def wrapper(*args, **kwargs):
140
+ # Argument filtering
141
+ sig = inspect.signature(transition_func)
142
+ supported_params = sig.parameters
143
+ filtered_kwargs = {
144
+ k: v for k, v in kwargs.items() if k in supported_params
145
+ }
146
+
147
+ try:
148
+ logger.debug(
149
+ f"Starting {transition_func.__name__}",
150
+ )
151
+ # Apply filtered_kwargs instead of kwargs
152
+ result = transition_func(*args, **filtered_kwargs)
153
+ logger.debug(
154
+ f"Finished {transition_func.__name__}",
155
+ )
156
+
157
+ return result
158
+ except BusinessException as e:
159
+ logger.info(f"Business exception: {e}\nExiting ScreenPilot.")
160
+ raise e
161
+ except RuntimeError as e:
162
+ cls._runtime_error_details = _action_line_from_exc()
163
+ logger.error(
164
+ f"Runtime error in {transition_func.__name__} at action {cls._runtime_error_details[1]}",
165
+ )
166
+ logger.debug(
167
+ f"Runtime error traceback: {cls._runtime_error_details[0]}",
168
+ )
169
+ screenshot_and_log("Runtime error")
170
+ logger.info(
171
+ "Proceeding to course correction, activating rollback mode.",
172
+ )
173
+ cls._attempt_ai_recovery(
174
+ scenario="runtime_error",
175
+ error_details=cls._runtime_error_details[1],
176
+ )
177
+ cls._mode = "rollback"
178
+ except RollbackCompleted as e:
179
+ logger.info("Rollback completed.")
180
+ raise type(e)(cls._runtime_error_details[0])
181
+
182
+ # Ensure that any duplicate transitions have a condition function
183
+ possible_transitions = cls._graph.out_edges(
184
+ from_state, keys=True, data=True
185
+ )
186
+ for start_state, end_state, key, data in possible_transitions:
187
+ if data.get("mode") == mode and (
188
+ not data.get("condition") or not condition
189
+ ):
190
+ existing_transition_name = data.get("func").__name__
191
+ transition_type = "rollback" if mode == "rollback" else "transition"
192
+ raise ValueError(
193
+ f"Error while registering {wrapper.__name__}: {existing_transition_name} is already registered as {transition_type} from {start_state} to {end_state}. To add multiple transitions between the same states, provide a condition function for each of them."
194
+ )
195
+
196
+ # Register the wrapped function as a transition in the graph
197
+ if from_state in cls._graph.nodes and to_state in cls._graph.nodes:
198
+ key = f"{wrapper.__name__}_from_{from_state}_to_{to_state}_{mode}"
199
+ cls._graph.add_edge(
200
+ from_state,
201
+ to_state,
202
+ key=key,
203
+ func=wrapper,
204
+ condition=condition,
205
+ mode=mode,
206
+ )
207
+ else:
208
+ logger.error(
209
+ f"Error: Transition from {from_state} to {to_state} involves undefined state.",
210
+ )
211
+ return wrapper
212
+
213
+ return decorator
214
+
215
+ @classmethod
216
+ def _log_transition(cls, transition_func: Callable):
217
+ """
218
+ Log a transition from one state to another.
219
+
220
+ Parameters:
221
+ from_state (str): The state from which the transition occurs.
222
+ to_state (str): The state to which the transition leads.
223
+
224
+ Returns:
225
+ None
226
+
227
+ """
228
+ cls._transition_history.append(transition_func)
229
+
230
+ @classmethod
231
+ def _log_state(cls, state):
232
+ """
233
+ Log a state in the state machine history.
234
+
235
+ Parameters:
236
+ state (str): The state to be logged.
237
+
238
+ Returns:
239
+ None
240
+
241
+ """
242
+ cls._state_history.append(state)
243
+
244
+ @classmethod
245
+ def _evaluate_state(cls) -> None:
246
+ """
247
+ Evaluate the current state of the state machine.
248
+
249
+ This method evaluates the current state of the state machine by calling the state evaluation function
250
+ provided during configuration. It retrieves the possible states from the state machine graph and passes
251
+ them along with the state evaluation output model to the state evaluation function. The current state and
252
+ its description are then updated based on the evaluation result. The current state is logged in the state
253
+ history.
254
+
255
+ If there is a previous state in the state history, an actual transition is added to the state machine graph,
256
+ connecting the previous state to the current state.
257
+
258
+ Raises:
259
+ ValueError: If the state evaluation function is not set or not callable.
260
+ """
261
+ if not callable(cls.state_eval_function):
262
+ raise ValueError("State evaluation function is not set or not callable.")
263
+
264
+ possible_states = [
265
+ {state: cls._graph.nodes[state]["cls"].description}
266
+ for state in cls._graph.nodes
267
+ ]
268
+ cls._current_state, cur_state_description = cls.state_eval_function(
269
+ possible_states, cls.state_eval_output_model
270
+ )
271
+ logger.debug(
272
+ f"Current state: {cls._current_state}\nDescription: {cur_state_description}",
273
+ )
274
+ cls._log_state(cls._current_state)
275
+ cls._acted_since_state_eval = False
276
+ try:
277
+ screenshot_and_log(f"State_{cls._current_state}")
278
+ except Exception as e:
279
+ logger.warning("Error occurred while taking and saving screenshot")
280
+ logger.warning(f"Details: {str(e)}")
281
+
282
+ if len(cls._state_history) > 1 and len(cls._transition_history) > 0:
283
+ key = f"{cls._transition_history[-1].__name__}_from_{cls._state_history[-2]}_to_{cls._current_state}_actual"
284
+ cls._graph.add_edge(
285
+ cls._state_history[-2], cls._current_state, key=key, mode="actual"
286
+ )
287
+
288
+ @staticmethod
289
+ def _find_unplanned_transitions(graph: nx.MultiDiGraph) -> List[Tuple[str, str]]:
290
+ """
291
+ Identify transitions in the graph which were not originally planned.
292
+
293
+ Parameters:
294
+ graph (nx.MultiDiGraph): The graph representing the state machine.
295
+
296
+ Returns:
297
+ List[Tuple[str, str]]: A list of tuples representing the unplanned transitions. Each tuple contains the source state and the destination state of the transition.
298
+
299
+ """
300
+ actual_edges = [
301
+ (u, v)
302
+ for u, v, k, data in graph.edges(keys=True, data=True)
303
+ if data.get("mode") == "actual"
304
+ ]
305
+ planned_edges = set(
306
+ [
307
+ (u, v)
308
+ for u, v, k, data in graph.edges(keys=True, data=True)
309
+ if data.get("mode") == "planned"
310
+ ]
311
+ )
312
+
313
+ # Filter out actual edges that have a corresponding planned edge
314
+ unplanned_transitions = [
315
+ edge for edge in actual_edges if edge not in planned_edges
316
+ ]
317
+
318
+ if unplanned_transitions:
319
+ logger.info(f"Found unplanned transitions: {unplanned_transitions}")
320
+
321
+ return unplanned_transitions
322
+
323
+ @staticmethod
324
+ def _find_repeated_transitions(transition_history: deque) -> List[str]:
325
+ """
326
+ Identify repeated transitions in the transition history.
327
+
328
+ This method takes in a deque object representing the transition history and identifies any repeated transitions. It iterates over each item in the transition history and checks if the count of that item is greater than 1. If a repeated transition is found, it is added to the list of repeated transitions.
329
+
330
+ Parameters:
331
+ transition_history (deque): A deque object representing the transition history.
332
+
333
+ Returns:
334
+ List[str]: A list of strings representing the repeated transitions.
335
+
336
+ """
337
+ repeated_transitions = [
338
+ item.__name__
339
+ for item in transition_history
340
+ if transition_history.count(item) > 1
341
+ ]
342
+ if repeated_transitions:
343
+ logger.info(f"Found repeated transitions: {repeated_transitions}")
344
+ return repeated_transitions
345
+
346
+ @staticmethod
347
+ def _find_repeated_states(state_history) -> List[str]:
348
+ """
349
+ This method identifies repeated states in the state history.
350
+ Parameters:
351
+ state_history (deque): A deque object representing the state history.
352
+ Returns:
353
+ List[str]: A list of strings representing the repeated states.
354
+ """
355
+ repeated_states = [
356
+ item for item in state_history if state_history.count(item) > 1
357
+ ]
358
+ if repeated_states:
359
+ logger.info(f"Found repeated states: {repeated_states}")
360
+ return repeated_states
361
+
362
+ @classmethod
363
+ def _find_anti_patterns(cls) -> bool:
364
+ """
365
+ Detect abnormal patterns in the state machine execution and take appropriate action.
366
+
367
+ Returns:
368
+ bool: Whether an anti-pattern was detected.
369
+ """
370
+ # Detect deviation from planned transitions
371
+ unplanned_transitions: List = cls._find_unplanned_transitions(cls._graph)
372
+ if len(unplanned_transitions) > cls.tolerate_unplanned_transitions:
373
+ cls._exit_reason = UnplannedTransitionsError(
374
+ f"{len(unplanned_transitions)} unplanned transitions detected: {unplanned_transitions}."
375
+ )
376
+ raise cls._exit_reason
377
+
378
+ repeated_transitions: List = cls._find_repeated_transitions(
379
+ cls._transition_history
380
+ )
381
+ if (
382
+ len(repeated_transitions) > cls.tolerate_repeat_transitions + 1
383
+ ): # +1 to account for the first transition
384
+ cls._exit_reason = RepeatTransitions(
385
+ f"{len(repeated_transitions)} repeated transitions detected: {repeated_transitions}."
386
+ )
387
+ raise cls._exit_reason
388
+
389
+ repeated_states: List = cls._find_repeated_states(cls._state_history)
390
+ if len(repeated_states) > cls.tolerate_repeat_states:
391
+ cls._exit_reason = RepeatStatesError(
392
+ f"{len(repeated_states)} repeated states detected: {repeated_states}."
393
+ )
394
+ if cls.enable_force_close_app_process and cls.process_name:
395
+ logger.info(
396
+ f"RepeatStatesError detected. Killing application with process name: {cls.process_name}",
397
+ )
398
+ try:
399
+ ForceCloseApplication(process_name=cls.process_name).do()
400
+ except Exception as e:
401
+ logger.warning("Killing application failed. Details")
402
+ logger.warning(f"{str(e)}")
403
+ elif cls.enable_force_close_app_process and not cls.process_name:
404
+ logger.warning(
405
+ "RepeatStatesError detected. Application cannot be killed as process_name was not provided.",
406
+ )
407
+ raise cls._exit_reason
408
+
409
+ # No anti-patterns detected
410
+ return False
411
+
412
+ @classmethod
413
+ def configure(
414
+ cls,
415
+ **kwargs,
416
+ ) -> None:
417
+ """
418
+ Configure the state machine.
419
+
420
+ Parameters:
421
+ **kwargs: Additional keyword arguments for the configuration.
422
+
423
+ Returns:
424
+ None
425
+ """
426
+ for k, v in kwargs.items():
427
+ if hasattr(cls, k) and not k.startswith("_"):
428
+ setattr(cls, k, v)
429
+ else:
430
+ logger.error(f"Public attribute {k} not found in ScreenPilot.")
431
+ return None
432
+
433
+ @classmethod
434
+ def _evaluate_goal_function(cls, goal_function: Callable, **kwargs) -> None:
435
+ """
436
+ Evaluate the success of the current state machine execution.
437
+
438
+ Args:
439
+ goal_function: A function that evaluates the completion and success of the current state machine execution.
440
+ Args: current_state (class), **kwargs.
441
+ Returns: Tuple[bool, bool].
442
+ **kwargs: Additional arguments for the goal function.
443
+
444
+ Returns:
445
+ None
446
+ """
447
+ # Filter out unsupported kwargs
448
+ sig = inspect.signature(goal_function)
449
+ supported_params = sig.parameters
450
+ filtered_kwargs = {k: v for k, v in kwargs.items() if k in supported_params}
451
+ goal_function(cls._current_state, **filtered_kwargs)
452
+ return None
453
+
454
+ @classmethod
455
+ def _act_on_start(cls) -> None:
456
+ """
457
+ Actions to be taken when the state machine execution is started.
458
+
459
+ Returns:
460
+ None
461
+ """
462
+ cls._exit_reason = None
463
+ cls._current_state = None
464
+ cls._final_state = None
465
+ logger.info("State machine execution started.")
466
+
467
+ @classmethod
468
+ def _act_on_completed(cls) -> None:
469
+ """
470
+ Actions to be taken when the state machine execution is completed.
471
+
472
+ This method performs the following actions:
473
+ - Sets the final state of the state machine to the current state.
474
+ - Clears the transition history and state history.
475
+ - Removes the actual transitions from the state machine graph.
476
+ - Resets the remaining AI recovery attempts to the initial value.
477
+ - Logs a message indicating that the state machine execution is completed and cleanup has been performed.
478
+
479
+ Returns:
480
+ None
481
+ """
482
+ try:
483
+ cls._ensure_allowed_end_state()
484
+ except Exception as e:
485
+ logger.error(f"Error during end state check: {e}")
486
+ raise e
487
+ finally:
488
+ cls._final_state = cls._current_state
489
+ cls._transition_history.clear()
490
+ cls._state_history.clear()
491
+ # remove actual transitions from the graph
492
+ cls._graph.remove_edges_from(
493
+ [
494
+ (u, v, k)
495
+ for u, v, k, data in cls._graph.edges(keys=True, data=True)
496
+ if data.get("mode") == "actual"
497
+ ]
498
+ )
499
+ cls._mode = "planned"
500
+ cls._runtime_error_details = None
501
+ cls._ai_recovery_agent = None
502
+ cls._next_target_state = None
503
+
504
+ logger.info("State machine execution completed, cleanup performed.")
505
+ return None
506
+
507
+ @classmethod
508
+ def _write_ai_recovery_prompt(
509
+ cls,
510
+ scenario: Literal["exit", "runtime_error", "unexpected_state"],
511
+ error_details: Optional[str] = None,
512
+ ) -> str:
513
+ # If no transition history is available, we are dealing with an error from the previous instance
514
+ if scenario == "unexpected_state" and not cls._transition_history:
515
+ start_nodes = ", ".join(
516
+ [
517
+ f"{node} ({data.get('description')})"
518
+ for node, data in cls._graph.nodes(data=True)
519
+ if data.get("start_allowed") == True
520
+ ]
521
+ )
522
+ action_prompt = f"The process just started but the system is in an unexpected state. We need to return to one of the known states: {start_nodes}."
523
+ # If the ScreenPilot is exiting, we just need to clean up the system for the next instance
524
+ elif scenario == "exit":
525
+ end_nodes = ", ".join(
526
+ [
527
+ f"{node} ({data.get('description')})"
528
+ for node, data in cls._graph.nodes(data=True)
529
+ if data.get("end_allowed") == True
530
+ ]
531
+ )
532
+ action_prompt = f"The process has completed and we need to bring the system back to a state where we can exit safely: {end_nodes}."
533
+ # If we have error details, we assume an error during a transition
534
+ elif scenario == "runtime_error":
535
+ attempted_transition = cls._transition_history[-1]
536
+ attempted_transition_description = attempted_transition.__doc__
537
+ previous_state = cls._state_history[-1]
538
+ previous_state_description = next(
539
+ (
540
+ data.get("description")
541
+ for node, data in cls._graph.nodes(data=True)
542
+ if node == previous_state
543
+ ),
544
+ "", # Default value if no description is found
545
+ )
546
+ action_prompt = f"We tried performing '{attempted_transition.__name__}' ({attempted_transition_description}) but ran into an error: {error_details}. Could you try and remove possible causes of the error? It could be e.g. a popup, or window focus. We want to return to the previous state, {previous_state} ({previous_state_description})."
547
+ # Otherewise we assume atransition has completed but the resulting screen is not as expected
548
+ else:
549
+ attempted_transition = cls._transition_history[-1]
550
+ attempted_transition_description = attempted_transition.__doc__
551
+ target_state = cls._next_target_state
552
+ target_state_description = next(
553
+ (
554
+ data.get("description")
555
+ for node, data in cls._graph.nodes(data=True)
556
+ if node == target_state
557
+ ),
558
+ "", # Default value if no description is found
559
+ )
560
+ action_prompt = f"We tried performing '{attempted_transition.__name__}' ({attempted_transition_description}) and expected to see state '{target_state}' ({target_state_description}) but the screen does not match the expected state."
561
+ return action_prompt
562
+
563
+ @classmethod
564
+ def _attempt_ai_recovery(
565
+ cls,
566
+ scenario: Literal["exit", "runtime_error", "unexpected_state"],
567
+ attempts: int = 5,
568
+ error_details: Optional[str] = None,
569
+ ) -> None:
570
+ """
571
+ Attempt AI recovery in case of an error state.
572
+ Args:
573
+ attempts (int, optional): The number of attempts to recover using AI.
574
+ error_details (str, optional): Details of the error that occurred.
575
+ Returns:
576
+ None
577
+ """
578
+ # import actions used in AI recovery
579
+ from ..ui_actions import (
580
+ LeftClick,
581
+ DoubleClick,
582
+ SendKeys,
583
+ PressKeys,
584
+ ActivateWindow,
585
+ CloseWindow,
586
+ )
587
+
588
+ # Initialize CourseCorrector, if not already initialized
589
+ if not cls._ai_recovery_agent and cls.ai_recovery_agent_factory is not None:
590
+ action_prompt = cls._write_ai_recovery_prompt(scenario, error_details)
591
+ cls._ai_recovery_agent = cls.ai_recovery_agent_factory(
592
+ goal=action_prompt, custom_instructions=cls.ai_recovery_instructions
593
+ )
594
+ if cls._ai_recovery_agent is not None:
595
+ logger.debug(
596
+ f"AI recovery agent {cls._ai_recovery_agent.name} initialized with prompt: '{action_prompt}'",
597
+ )
598
+ if cls._ai_recovery_agent is not None:
599
+ attempts_made = 0
600
+
601
+ while attempts_made < attempts:
602
+ attempts_made += 1
603
+ logger.info(
604
+ f"AI recovery attempt {attempts_made} / {attempts}",
605
+ )
606
+ action: Optional[ActionString] = None
607
+ try:
608
+ # Get corrective actions from the course corrector and execute the first one
609
+ action = cls._ai_recovery_agent.get_corrective_actions()[0]
610
+ logger.info(f"Corrective action: {action}")
611
+
612
+ # If no action is suggested, break the loop and return None
613
+ if action.action_string.startswith("NoAction"):
614
+ return None
615
+
616
+ try:
617
+ eval(action.action_string)
618
+ cls._acted_since_state_eval = True
619
+ break
620
+ except (
621
+ AttributeError,
622
+ SyntaxError,
623
+ ValidationError,
624
+ NameError,
625
+ ) as e:
626
+ cls._ai_recovery_agent.add_feedback(str(e))
627
+ logger.error(f"Error during AI recovery: {e}")
628
+ logger.info(
629
+ f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
630
+ )
631
+ continue
632
+ except RuntimeError:
633
+ cls._ai_recovery_agent.add_feedback(
634
+ "The target could not be uniquely identified. Try using different anchors or target."
635
+ )
636
+ logger.error("Runtime error during AI recovery.")
637
+ logger.info(
638
+ f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
639
+ )
640
+ continue
641
+ except PerformActionException:
642
+ cls._ai_recovery_agent.add_feedback(
643
+ "The action could not be performed."
644
+ )
645
+ logger.error("PerformActionException during AI recovery.")
646
+ logger.info(
647
+ f"Retrying with feedback to course corrector: {cls._ai_recovery_agent.get_latest_feedback()}",
648
+ )
649
+ continue
650
+
651
+ except Exception as e:
652
+ logger.error(f"Unexpected error during AI recovery: {e}")
653
+ logger.info("Interrupting AI recovery.")
654
+ break
655
+ finally:
656
+ # Interrupt the process if the action is impossible to process
657
+ if not action:
658
+ raise CourseCorrectionImpossible(
659
+ "No course correction action could be generated, see logs for details."
660
+ )
661
+ elif action.interrupt_process:
662
+ raise CourseCorrectionImpossible(action.observation)
663
+ screenshot_and_log("Completed AI recovery")
664
+ cls._ai_recovery_agent.reset_feedback()
665
+
666
+ @classmethod
667
+ def _get_next_transition(cls, **kwargs) -> Tuple[Callable, str]:
668
+ """
669
+ Get the next transition function based on the current state.
670
+ Args:
671
+ **kwargs: Additional arguments for the condition functions.
672
+
673
+ Returns:
674
+ Callable: The next transition function.
675
+ str: The next state.
676
+ """
677
+ possible_transitions = cls._graph.out_edges(
678
+ cls._current_state, keys=True, data=True
679
+ )
680
+ if not possible_transitions:
681
+ logger.error(
682
+ f"No available {cls._mode} transitions from {cls._current_state}.",
683
+ )
684
+ for _, to_state, key, data in possible_transitions:
685
+ if data.get("mode") == cls._mode:
686
+ # Execute the first transition which has no condition or has a condition that evaluates to True
687
+ condition: Optional[Callable] = data.get("condition", None)
688
+ if not condition:
689
+ return data.get("func"), to_state
690
+ else:
691
+ # Argument filtering
692
+ sig = inspect.signature(condition)
693
+ supported_params = sig.parameters
694
+ filtered_kwargs = {
695
+ k: v for k, v in kwargs.items() if k in supported_params
696
+ }
697
+ if condition(**filtered_kwargs):
698
+ return data.get("func"), to_state
699
+
700
+ # raise an error to prevent failing in run()
701
+ err_msg = f"Cannot get a valid transition from {cls._current_state} in {cls._mode} mode. {len(possible_transitions)} transitions available, but none of them match the mode and conditions."
702
+ logger.error(err_msg)
703
+ raise ValueError(err_msg)
704
+
705
+ @staticmethod
706
+ def _default_goal_function(current_state: str) -> None:
707
+ """
708
+ Default goal function for the state machine.
709
+
710
+ Args:
711
+ current_state (str): The current state of the state machine.
712
+ **kwargs: Additional arguments for the goal function.
713
+
714
+ Returns:
715
+ None
716
+ """
717
+ return None
718
+
719
+ @classmethod
720
+ def run(
721
+ cls, goal_function: Optional[Callable] = None, **kwargs
722
+ ) -> Union[ScreenPilotOutcome, ScreenPilotException]:
723
+ """
724
+ Main loop of the state machine.
725
+
726
+ Args:
727
+ goal_function (Optional[Callable]): Function to evaluate the success of the current state machine execution. If no goal function is provided, the complete_ui_automation() method must be called from a transition.
728
+ Args: current_state (str), **kwargs.
729
+ Returns: bool.
730
+ **kwargs: Additional arguments for the goal function and the transition functions (will be dynamically filtered as needed).
731
+
732
+ Returns:
733
+ - Exit reason (ScreenPilotOutcome): Exception class that interrupted the state machine execution.
734
+ """
735
+ if not goal_function:
736
+ goal_function = cls._default_goal_function
737
+ cls._act_on_start()
738
+ try:
739
+ while not cls._exit_reason:
740
+ # Detect anti-patterns (raises exceptions if detected)
741
+ cls._find_anti_patterns()
742
+
743
+ # Evaluate the current state and check for completion
744
+ cls._evaluate_state()
745
+ cls._evaluate_goal_function(goal_function, **kwargs)
746
+
747
+ # Attempt AI recovery if required
748
+ if cls._current_state == "Error" and cls.ai_recovery:
749
+ cls._attempt_ai_recovery(
750
+ scenario="unexpected_state", attempts=cls.ai_recovery_attempts
751
+ )
752
+ continue
753
+ # Reset course corrector if the state is not Error
754
+ else:
755
+ cls._ai_recovery_agent = None
756
+
757
+ # Find possible transitions from the current state
758
+ if cls._current_state not in cls._graph:
759
+ err_msg: str = (
760
+ f"Current state {cls._current_state} is not recognized."
761
+ )
762
+ logger.error(err_msg)
763
+ raise ValueError(err_msg)
764
+ next_transition, cls._next_target_state = cls._get_next_transition(
765
+ **kwargs
766
+ )
767
+
768
+ # Execute the transition function
769
+ cls._log_transition(next_transition)
770
+ cls._acted_since_state_eval = True
771
+ next_transition(**kwargs) # Execute the transition function
772
+
773
+ except (BusinessException, SuccessfulCompletion) as e:
774
+ cls._exit_reason = e
775
+ except Exception as e:
776
+ logger.error(traceback.format_exc())
777
+ raise e
778
+ finally:
779
+ cls._act_on_completed()
780
+ return cls._exit_reason
781
+
782
+ @classmethod
783
+ def _ensure_allowed_end_state(cls):
784
+ # If actions have been performed since the last state evaluation, evaluate the state again
785
+ if cls._acted_since_state_eval:
786
+ cls._evaluate_state()
787
+ end_states = [
788
+ state
789
+ for state, data in cls._graph.nodes(data=True)
790
+ if data.get("end_allowed") == True
791
+ ]
792
+ if cls._current_state not in end_states and cls.ai_recovery:
793
+ cls._attempt_ai_recovery(scenario="exit", attempts=cls.ai_recovery_attempts)
794
+
795
+
796
+ def _action_line_from_exc() -> Tuple[str, str, str]:
797
+ """
798
+ Format the current exception to extract the action line and action string.
799
+ Returns:
800
+ Tuple[str, str, str]: Tuple with the full traceback, line with action, and action string.
801
+ """
802
+ import sys
803
+
804
+ (
805
+ exc_type,
806
+ exc_value,
807
+ exc_traceback,
808
+ ) = sys.exc_info() # Capture the current exception info
809
+ tb_lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
810
+ output = (
811
+ "\n".join(tb_lines),
812
+ "",
813
+ "",
814
+ ) # Tuple: (full traceback, line with action, action)
815
+ for line in tb_lines:
816
+ if ".do(" in line:
817
+ import re
818
+
819
+ # Match anything after an action type and before do()
820
+ match = re.search(
821
+ r"(LeftClick|PressKeys|SendKeys|DoubleClick|WaitFor|RightClick|OpenApplication|MaximizeWindow|CloseWindow|ActivateWindow|SaveFiles|DeleteFiles|\(.*?\)\.do\((.*)\))",
822
+ line,
823
+ )
824
+ if match:
825
+ output = (output[0], line, match.group(0))
826
+ break
827
+ return output
828
+
829
+
830
+ def screenshot_and_log(message: str, screenshot_folder: str = "screenshots") -> None:
831
+ logger.info(message)
832
+ timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
833
+ screenshot_name = f"{message.replace(' ', '_')}.png"
834
+ save_screenshot(f"{timestamp}_{screenshot_name}", screenshot_folder)
835
+ logger.info(
836
+ f"Saved screenshot {timestamp}_{screenshot_name} in {screenshot_folder}."
837
+ )
838
+ return None