cua-agent 0.4.22__py3-none-any.whl → 0.4.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cua-agent might be problematic. Click here for more details.
- agent/callbacks/image_retention.py +38 -87
- agent/callbacks/operator_validator.py +32 -32
- agent/human_tool/ui.py +121 -105
- {cua_agent-0.4.22.dist-info → cua_agent-0.4.24.dist-info}/METADATA +1 -1
- {cua_agent-0.4.22.dist-info → cua_agent-0.4.24.dist-info}/RECORD +7 -7
- {cua_agent-0.4.22.dist-info → cua_agent-0.4.24.dist-info}/WHEEL +0 -0
- {cua_agent-0.4.22.dist-info → cua_agent-0.4.24.dist-info}/entry_points.txt +0 -0
|
@@ -50,90 +50,41 @@ class ImageRetentionCallback(AsyncCallbackHandler):
|
|
|
50
50
|
"""
|
|
51
51
|
if self.only_n_most_recent_images is None:
|
|
52
52
|
return messages
|
|
53
|
-
|
|
54
|
-
#
|
|
55
|
-
|
|
56
|
-
for
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
for
|
|
90
|
-
|
|
91
|
-
call_id = msg.get("call_id")
|
|
92
|
-
|
|
93
|
-
# Remove old computer_call items
|
|
94
|
-
if msg_type == "computer_call" and call_id not in keep_call_ids:
|
|
95
|
-
# Check if this call_id corresponds to an image call
|
|
96
|
-
has_image_output = any(
|
|
97
|
-
m.get("type") == "computer_call_output" and
|
|
98
|
-
m.get("call_id") == call_id and
|
|
99
|
-
isinstance(m.get("output"), dict) and
|
|
100
|
-
"image_url" in m.get("output", {})
|
|
101
|
-
for m in messages_with_call_ids
|
|
102
|
-
)
|
|
103
|
-
if has_image_output:
|
|
104
|
-
continue # Skip this computer_call
|
|
105
|
-
|
|
106
|
-
# Remove old computer_call_output items with images
|
|
107
|
-
if (msg_type == "computer_call_output" and
|
|
108
|
-
call_id not in keep_call_ids and
|
|
109
|
-
isinstance(msg.get("output"), dict) and
|
|
110
|
-
"image_url" in msg.get("output", {})):
|
|
111
|
-
continue # Skip this computer_call_output
|
|
112
|
-
|
|
113
|
-
# Remove old reasoning items that are paired with removed computer calls
|
|
114
|
-
if (msg_type == "reasoning" and
|
|
115
|
-
call_id and call_id not in keep_call_ids):
|
|
116
|
-
# Check if this call_id corresponds to an image call that's being removed
|
|
117
|
-
has_image_output = any(
|
|
118
|
-
m.get("type") == "computer_call_output" and
|
|
119
|
-
m.get("call_id") == call_id and
|
|
120
|
-
isinstance(m.get("output"), dict) and
|
|
121
|
-
"image_url" in m.get("output", {})
|
|
122
|
-
for m in messages_with_call_ids
|
|
123
|
-
)
|
|
124
|
-
if has_image_output:
|
|
125
|
-
continue # Skip this reasoning item
|
|
126
|
-
|
|
127
|
-
filtered_messages.append(msg)
|
|
128
|
-
|
|
129
|
-
# Clean up: Remove call_id from reasoning items before returning
|
|
130
|
-
final_messages = []
|
|
131
|
-
for msg in filtered_messages:
|
|
132
|
-
if msg.get("type") == "reasoning" and "call_id" in msg:
|
|
133
|
-
# Create a copy without call_id for reasoning items
|
|
134
|
-
cleaned_msg = {k: v for k, v in msg.items() if k != "call_id"}
|
|
135
|
-
final_messages.append(cleaned_msg)
|
|
136
|
-
else:
|
|
137
|
-
final_messages.append(msg)
|
|
138
|
-
|
|
139
|
-
return final_messages
|
|
53
|
+
|
|
54
|
+
# Gather indices of all computer_call_output messages that contain an image_url
|
|
55
|
+
output_indices: List[int] = []
|
|
56
|
+
for idx, msg in enumerate(messages):
|
|
57
|
+
if msg.get("type") == "computer_call_output":
|
|
58
|
+
out = msg.get("output")
|
|
59
|
+
if isinstance(out, dict) and ("image_url" in out):
|
|
60
|
+
output_indices.append(idx)
|
|
61
|
+
|
|
62
|
+
# Nothing to trim
|
|
63
|
+
if len(output_indices) <= self.only_n_most_recent_images:
|
|
64
|
+
return messages
|
|
65
|
+
|
|
66
|
+
# Determine which outputs to keep (most recent N)
|
|
67
|
+
keep_output_indices = set(output_indices[-self.only_n_most_recent_images :])
|
|
68
|
+
|
|
69
|
+
# Build set of indices to remove in one pass
|
|
70
|
+
to_remove: set[int] = set()
|
|
71
|
+
|
|
72
|
+
for idx in output_indices:
|
|
73
|
+
if idx in keep_output_indices:
|
|
74
|
+
continue # keep this screenshot and its context
|
|
75
|
+
|
|
76
|
+
to_remove.add(idx) # remove the computer_call_output itself
|
|
77
|
+
|
|
78
|
+
# Remove the immediately preceding computer_call with matching call_id (if present)
|
|
79
|
+
call_id = messages[idx].get("call_id")
|
|
80
|
+
prev_idx = idx - 1
|
|
81
|
+
if prev_idx >= 0 and messages[prev_idx].get("type") == "computer_call" and messages[prev_idx].get("call_id") == call_id:
|
|
82
|
+
to_remove.add(prev_idx)
|
|
83
|
+
# Check a single reasoning immediately before that computer_call
|
|
84
|
+
r_idx = prev_idx - 1
|
|
85
|
+
if r_idx >= 0 and messages[r_idx].get("type") == "reasoning":
|
|
86
|
+
to_remove.add(r_idx)
|
|
87
|
+
|
|
88
|
+
# Construct filtered list
|
|
89
|
+
filtered = [m for i, m in enumerate(messages) if i not in to_remove]
|
|
90
|
+
return filtered
|
|
@@ -102,37 +102,37 @@ class OperatorNormalizerCallback(AsyncCallbackHandler):
|
|
|
102
102
|
_keep_keys(action, keep)
|
|
103
103
|
|
|
104
104
|
|
|
105
|
-
# Second pass: if an assistant message is immediately followed by a computer_call,
|
|
106
|
-
# replace the assistant message itself with a reasoning message with summary text.
|
|
107
|
-
if isinstance(output, list):
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
105
|
+
# # Second pass: if an assistant message is immediately followed by a computer_call,
|
|
106
|
+
# # replace the assistant message itself with a reasoning message with summary text.
|
|
107
|
+
# if isinstance(output, list):
|
|
108
|
+
# for i, item in enumerate(output):
|
|
109
|
+
# # AssistantMessage shape: { type: 'message', role: 'assistant', content: OutputContent[] }
|
|
110
|
+
# if item.get("type") == "message" and item.get("role") == "assistant":
|
|
111
|
+
# next_idx = i + 1
|
|
112
|
+
# if next_idx >= len(output):
|
|
113
|
+
# continue
|
|
114
|
+
# next_item = output[next_idx]
|
|
115
|
+
# if not isinstance(next_item, dict):
|
|
116
|
+
# continue
|
|
117
|
+
# if next_item.get("type") != "computer_call":
|
|
118
|
+
# continue
|
|
119
|
+
# contents = item.get("content") or []
|
|
120
|
+
# # Extract text from OutputContent[]
|
|
121
|
+
# text_parts: List[str] = []
|
|
122
|
+
# if isinstance(contents, list):
|
|
123
|
+
# for c in contents:
|
|
124
|
+
# if isinstance(c, dict) and c.get("type") == "output_text" and isinstance(c.get("text"), str):
|
|
125
|
+
# text_parts.append(c["text"])
|
|
126
|
+
# text_content = "\n".join(text_parts).strip()
|
|
127
|
+
# # Replace assistant message with reasoning message
|
|
128
|
+
# output[i] = {
|
|
129
|
+
# "type": "reasoning",
|
|
130
|
+
# "summary": [
|
|
131
|
+
# {
|
|
132
|
+
# "type": "summary_text",
|
|
133
|
+
# "text": text_content,
|
|
134
|
+
# }
|
|
135
|
+
# ],
|
|
136
|
+
# }
|
|
137
137
|
|
|
138
138
|
return output
|
agent/human_tool/ui.py
CHANGED
|
@@ -196,7 +196,9 @@ class HumanCompletionUI:
|
|
|
196
196
|
gr.update(choices=["latest"], value="latest"), # dropdown
|
|
197
197
|
gr.update(value=None), # image (no image)
|
|
198
198
|
gr.update(value=[]), # chatbot (empty messages)
|
|
199
|
-
gr.update(interactive=False) # submit button
|
|
199
|
+
gr.update(interactive=False), # submit button
|
|
200
|
+
gr.update(visible=False), # click_actions_group hidden
|
|
201
|
+
gr.update(visible=False), # actions_group hidden
|
|
200
202
|
)
|
|
201
203
|
|
|
202
204
|
# Sort pending calls by created_at to get oldest first
|
|
@@ -237,7 +239,9 @@ class HumanCompletionUI:
|
|
|
237
239
|
gr.update(choices=choices, value="latest"),
|
|
238
240
|
gr.update(value=self.last_image),
|
|
239
241
|
gr.update(value=conversation),
|
|
240
|
-
gr.update(interactive=bool(choices))
|
|
242
|
+
gr.update(interactive=bool(choices)),
|
|
243
|
+
gr.update(visible=True), # click_actions_group visible when there is a call
|
|
244
|
+
gr.update(visible=True), # actions_group visible when there is a call
|
|
241
245
|
)
|
|
242
246
|
|
|
243
247
|
def on_call_selected(self, selected_choice):
|
|
@@ -246,7 +250,9 @@ class HumanCompletionUI:
|
|
|
246
250
|
return (
|
|
247
251
|
gr.update(value=None), # no image
|
|
248
252
|
gr.update(value=[]), # empty chatbot
|
|
249
|
-
gr.update(interactive=False)
|
|
253
|
+
gr.update(interactive=False),
|
|
254
|
+
gr.update(visible=False), # click_actions_group hidden
|
|
255
|
+
gr.update(visible=False), # actions_group hidden
|
|
250
256
|
)
|
|
251
257
|
|
|
252
258
|
pending_calls = self.get_pending_calls()
|
|
@@ -254,7 +260,9 @@ class HumanCompletionUI:
|
|
|
254
260
|
return (
|
|
255
261
|
gr.update(value=None), # no image
|
|
256
262
|
gr.update(value=[]), # empty chatbot
|
|
257
|
-
gr.update(interactive=False)
|
|
263
|
+
gr.update(interactive=False),
|
|
264
|
+
gr.update(visible=False), # click_actions_group hidden
|
|
265
|
+
gr.update(visible=False), # actions_group hidden
|
|
258
266
|
)
|
|
259
267
|
|
|
260
268
|
# Handle "latest" option
|
|
@@ -286,7 +294,9 @@ class HumanCompletionUI:
|
|
|
286
294
|
return (
|
|
287
295
|
gr.update(value=None), # no image
|
|
288
296
|
gr.update(value=[]), # empty chatbot
|
|
289
|
-
gr.update(interactive=False)
|
|
297
|
+
gr.update(interactive=False),
|
|
298
|
+
gr.update(visible=False), # click_actions_group hidden
|
|
299
|
+
gr.update(visible=False), # actions_group hidden
|
|
290
300
|
)
|
|
291
301
|
|
|
292
302
|
conversation = self.format_messages_for_chatbot(selected_call.get("messages", []))
|
|
@@ -297,7 +307,9 @@ class HumanCompletionUI:
|
|
|
297
307
|
return (
|
|
298
308
|
gr.update(value=self.last_image),
|
|
299
309
|
gr.update(value=conversation),
|
|
300
|
-
gr.update(interactive=True)
|
|
310
|
+
gr.update(interactive=True),
|
|
311
|
+
gr.update(visible=True), # click_actions_group visible
|
|
312
|
+
gr.update(visible=True), # actions_group visible
|
|
301
313
|
)
|
|
302
314
|
|
|
303
315
|
def submit_response(self, response_text: str):
|
|
@@ -368,6 +380,10 @@ class HumanCompletionUI:
|
|
|
368
380
|
"""Submit a hotkey action."""
|
|
369
381
|
return self.submit_action("keypress", keys=keys)
|
|
370
382
|
|
|
383
|
+
def submit_wait_action(self) -> str:
|
|
384
|
+
"""Submit a wait action with no kwargs."""
|
|
385
|
+
return self.submit_action("wait")
|
|
386
|
+
|
|
371
387
|
def submit_description_click(self, description: str, action_type: str = "click", button: str = "left") -> str:
|
|
372
388
|
"""Submit a description-based action."""
|
|
373
389
|
if action_type == "click":
|
|
@@ -407,7 +423,7 @@ def create_ui():
|
|
|
407
423
|
"""Create the Gradio interface."""
|
|
408
424
|
ui_handler = HumanCompletionUI()
|
|
409
425
|
|
|
410
|
-
with gr.Blocks(title="Human-in-the-Loop Agent Tool") as demo:
|
|
426
|
+
with gr.Blocks(title="Human-in-the-Loop Agent Tool", fill_width=True) as demo:
|
|
411
427
|
gr.Markdown("# 🤖 Human-in-the-Loop Agent Tool")
|
|
412
428
|
gr.Markdown("Review AI conversation requests and provide human responses.")
|
|
413
429
|
|
|
@@ -415,29 +431,30 @@ def create_ui():
|
|
|
415
431
|
with gr.Column(scale=2):
|
|
416
432
|
with gr.Group():
|
|
417
433
|
screenshot_image = gr.Image(
|
|
418
|
-
label="Screenshot",
|
|
434
|
+
label="Interactive Screenshot",
|
|
419
435
|
interactive=False,
|
|
420
436
|
height=600
|
|
421
437
|
)
|
|
422
438
|
|
|
423
|
-
# Action type selection for image clicks
|
|
424
|
-
with gr.
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
439
|
+
# Action type selection for image clicks (wrapped for visibility control)
|
|
440
|
+
with gr.Group(visible=False) as click_actions_group:
|
|
441
|
+
with gr.Row():
|
|
442
|
+
action_type_radio = gr.Dropdown(
|
|
443
|
+
label="Action",
|
|
444
|
+
choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
|
|
445
|
+
value="click",
|
|
446
|
+
scale=2
|
|
447
|
+
)
|
|
448
|
+
action_button_radio = gr.Dropdown(
|
|
449
|
+
label="Button",
|
|
450
|
+
choices=["left", "right", "wheel", "back", "forward"],
|
|
451
|
+
value="left",
|
|
452
|
+
visible=True,
|
|
453
|
+
scale=1
|
|
454
|
+
)
|
|
438
455
|
|
|
439
456
|
conversation_chatbot = gr.Chatbot(
|
|
440
|
-
label="
|
|
457
|
+
label="Conversation",
|
|
441
458
|
type="messages",
|
|
442
459
|
height=500,
|
|
443
460
|
show_copy_button=True
|
|
@@ -446,91 +463,83 @@ def create_ui():
|
|
|
446
463
|
with gr.Column(scale=1):
|
|
447
464
|
with gr.Group():
|
|
448
465
|
call_dropdown = gr.Dropdown(
|
|
449
|
-
label="Select a pending
|
|
466
|
+
label="Select a pending conversation request",
|
|
450
467
|
choices=["latest"],
|
|
451
468
|
interactive=True,
|
|
452
469
|
value="latest"
|
|
453
470
|
)
|
|
454
471
|
refresh_btn = gr.Button("🔄 Refresh", variant="secondary")
|
|
472
|
+
status_display = gr.Textbox(
|
|
473
|
+
label="Status",
|
|
474
|
+
interactive=False,
|
|
475
|
+
value="Ready to receive requests..."
|
|
476
|
+
)
|
|
455
477
|
|
|
456
478
|
with gr.Group():
|
|
457
479
|
response_text = gr.Textbox(
|
|
458
|
-
label="
|
|
480
|
+
label="Message",
|
|
459
481
|
lines=3,
|
|
460
|
-
placeholder="Enter your
|
|
482
|
+
placeholder="Enter your message here..."
|
|
461
483
|
)
|
|
462
|
-
submit_btn = gr.Button("📤 Submit
|
|
484
|
+
submit_btn = gr.Button("📤 Submit Message", variant="primary", interactive=False)
|
|
463
485
|
|
|
464
|
-
# Action Accordions
|
|
465
|
-
with gr.
|
|
466
|
-
with gr.
|
|
467
|
-
with gr.
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
)
|
|
511
|
-
description_button = gr.Radio(
|
|
512
|
-
label="Button (for click only)",
|
|
513
|
-
choices=["left", "right", "wheel", "back", "forward"],
|
|
514
|
-
value="left"
|
|
515
|
-
)
|
|
516
|
-
description_submit_btn = gr.Button("Submit Description Action")
|
|
517
|
-
|
|
518
|
-
status_display = gr.Textbox(
|
|
519
|
-
label="Status",
|
|
520
|
-
interactive=False,
|
|
521
|
-
value="Ready to receive calls..."
|
|
522
|
-
)
|
|
486
|
+
# Action Accordions (wrapped for visibility control)
|
|
487
|
+
with gr.Group(visible=False) as actions_group:
|
|
488
|
+
with gr.Tabs():
|
|
489
|
+
with gr.Tab("🖱️ Click Actions"):
|
|
490
|
+
with gr.Group():
|
|
491
|
+
description_text = gr.Textbox(
|
|
492
|
+
label="Element Description",
|
|
493
|
+
placeholder="e.g., 'Privacy and security option in left sidebar'"
|
|
494
|
+
)
|
|
495
|
+
with gr.Row():
|
|
496
|
+
description_action_type = gr.Dropdown(
|
|
497
|
+
label="Action",
|
|
498
|
+
choices=["click", "double_click", "move", "left_mouse_up", "left_mouse_down"],
|
|
499
|
+
value="click"
|
|
500
|
+
)
|
|
501
|
+
description_button = gr.Dropdown(
|
|
502
|
+
label="Button",
|
|
503
|
+
choices=["left", "right", "wheel", "back", "forward"],
|
|
504
|
+
value="left"
|
|
505
|
+
)
|
|
506
|
+
description_submit_btn = gr.Button("Submit Click Action")
|
|
507
|
+
|
|
508
|
+
with gr.Tab("📝 Type Action"):
|
|
509
|
+
with gr.Group():
|
|
510
|
+
type_text = gr.Textbox(
|
|
511
|
+
label="Text to Type",
|
|
512
|
+
placeholder="Enter text to type..."
|
|
513
|
+
)
|
|
514
|
+
type_submit_btn = gr.Button("Submit Type")
|
|
515
|
+
|
|
516
|
+
with gr.Tab("⌨️ Keypress Action"):
|
|
517
|
+
with gr.Group():
|
|
518
|
+
keypress_text = gr.Textbox(
|
|
519
|
+
label="Keys",
|
|
520
|
+
placeholder="e.g., ctrl+c, alt+tab"
|
|
521
|
+
)
|
|
522
|
+
keypress_submit_btn = gr.Button("Submit Keypress")
|
|
523
|
+
|
|
524
|
+
with gr.Tab("🧰 Misc Actions"):
|
|
525
|
+
with gr.Group():
|
|
526
|
+
misc_action_dropdown = gr.Dropdown(
|
|
527
|
+
label="Action",
|
|
528
|
+
choices=["wait"],
|
|
529
|
+
value="wait"
|
|
530
|
+
)
|
|
531
|
+
misc_submit_btn = gr.Button("Submit Action")
|
|
523
532
|
|
|
524
533
|
# Event handlers
|
|
525
534
|
refresh_btn.click(
|
|
526
535
|
fn=ui_handler.refresh_pending_calls,
|
|
527
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
536
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
528
537
|
)
|
|
529
538
|
|
|
530
539
|
call_dropdown.change(
|
|
531
540
|
fn=ui_handler.on_call_selected,
|
|
532
541
|
inputs=[call_dropdown],
|
|
533
|
-
outputs=[screenshot_image, conversation_chatbot, submit_btn]
|
|
542
|
+
outputs=[screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
534
543
|
)
|
|
535
544
|
|
|
536
545
|
def handle_image_click(evt: gr.SelectData):
|
|
@@ -548,7 +557,7 @@ def create_ui():
|
|
|
548
557
|
outputs=[status_display]
|
|
549
558
|
).then(
|
|
550
559
|
fn=ui_handler.wait_for_pending_calls,
|
|
551
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
560
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
552
561
|
)
|
|
553
562
|
|
|
554
563
|
# Response submission
|
|
@@ -558,7 +567,7 @@ def create_ui():
|
|
|
558
567
|
outputs=[response_text, status_display]
|
|
559
568
|
).then(
|
|
560
569
|
fn=ui_handler.refresh_pending_calls,
|
|
561
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
570
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
562
571
|
)
|
|
563
572
|
|
|
564
573
|
# Toggle button radio visibility based on action type
|
|
@@ -570,16 +579,6 @@ def create_ui():
|
|
|
570
579
|
inputs=[action_type_radio],
|
|
571
580
|
outputs=[action_button_radio]
|
|
572
581
|
)
|
|
573
|
-
|
|
574
|
-
# Action accordion handlers
|
|
575
|
-
click_submit_btn.click(
|
|
576
|
-
fn=ui_handler.submit_click_action,
|
|
577
|
-
inputs=[click_x, click_y, click_action_type, click_button],
|
|
578
|
-
outputs=[status_display]
|
|
579
|
-
).then(
|
|
580
|
-
fn=ui_handler.wait_for_pending_calls,
|
|
581
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
582
|
-
)
|
|
583
582
|
|
|
584
583
|
type_submit_btn.click(
|
|
585
584
|
fn=ui_handler.submit_type_action,
|
|
@@ -587,7 +586,7 @@ def create_ui():
|
|
|
587
586
|
outputs=[status_display]
|
|
588
587
|
).then(
|
|
589
588
|
fn=ui_handler.wait_for_pending_calls,
|
|
590
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
589
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
591
590
|
)
|
|
592
591
|
|
|
593
592
|
keypress_submit_btn.click(
|
|
@@ -596,7 +595,7 @@ def create_ui():
|
|
|
596
595
|
outputs=[status_display]
|
|
597
596
|
).then(
|
|
598
597
|
fn=ui_handler.wait_for_pending_calls,
|
|
599
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
598
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
600
599
|
)
|
|
601
600
|
|
|
602
601
|
def handle_description_submit(description, action_type, button):
|
|
@@ -612,13 +611,30 @@ def create_ui():
|
|
|
612
611
|
outputs=[status_display]
|
|
613
612
|
).then(
|
|
614
613
|
fn=ui_handler.wait_for_pending_calls,
|
|
615
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
614
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
615
|
+
)
|
|
616
|
+
|
|
617
|
+
# Misc action handler
|
|
618
|
+
def handle_misc_submit(selected_action):
|
|
619
|
+
if selected_action == "wait":
|
|
620
|
+
result = ui_handler.submit_wait_action()
|
|
621
|
+
ui_handler.wait_for_pending_calls()
|
|
622
|
+
return result
|
|
623
|
+
return f"Unsupported misc action: {selected_action}"
|
|
624
|
+
|
|
625
|
+
misc_submit_btn.click(
|
|
626
|
+
fn=handle_misc_submit,
|
|
627
|
+
inputs=[misc_action_dropdown],
|
|
628
|
+
outputs=[status_display]
|
|
629
|
+
).then(
|
|
630
|
+
fn=ui_handler.wait_for_pending_calls,
|
|
631
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
616
632
|
)
|
|
617
633
|
|
|
618
634
|
# Load initial data
|
|
619
635
|
demo.load(
|
|
620
636
|
fn=ui_handler.refresh_pending_calls,
|
|
621
|
-
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn]
|
|
637
|
+
outputs=[call_dropdown, screenshot_image, conversation_chatbot, submit_btn, click_actions_group, actions_group]
|
|
622
638
|
)
|
|
623
639
|
|
|
624
640
|
return demo
|
|
@@ -8,9 +8,9 @@ agent/agent.py,sha256=ao3SKnZoX5_P6mTzEg0hgOUam6bNRwpbitzlyvwI3bg,29826
|
|
|
8
8
|
agent/callbacks/__init__.py,sha256=et6pNfX_AiJqhVzUfCvcjzFbDhfLoHazKCXN5sqwxaM,631
|
|
9
9
|
agent/callbacks/base.py,sha256=UnnnYlh6XCm6HKZZsAPaT_Eyo9LUYLyjyNwF-QRm6Ns,4691
|
|
10
10
|
agent/callbacks/budget_manager.py,sha256=RyKM-7iXQcDotYvrw3eURzeEHEXvQjID-NobtvQWE7k,1832
|
|
11
|
-
agent/callbacks/image_retention.py,sha256=
|
|
11
|
+
agent/callbacks/image_retention.py,sha256=8MeLo5-Y7cACpsNk2p_bvnZIYKpW6XgyukmdYGX23rE,3588
|
|
12
12
|
agent/callbacks/logging.py,sha256=OOxU97EzrxlnUAtiEnvy9FB7SwCUK90-rdpDFA2Ae4E,10921
|
|
13
|
-
agent/callbacks/operator_validator.py,sha256=
|
|
13
|
+
agent/callbacks/operator_validator.py,sha256=T5tp62pkShkcdHu2rgREUGdk8fryL_ziJsItXsfgYUQ,6494
|
|
14
14
|
agent/callbacks/pii_anonymization.py,sha256=NEkUTUjQBi82nqus7kT-1E4RaeQ2hQrY7YCnKndLhP8,3272
|
|
15
15
|
agent/callbacks/telemetry.py,sha256=RbUDhE41mTi8g9hNre0EpltK_NUZkLj8buJLWBzs0Ek,7363
|
|
16
16
|
agent/callbacks/trajectory_saver.py,sha256=rslgg4Ak7JHSNmmJgANRQ5TsUYWGuUJDZ6amureaz_o,15963
|
|
@@ -23,7 +23,7 @@ agent/decorators.py,sha256=n8VvMsififWkmuk75Q7HIpo0xAA2yAeQ6J-OOiwbAKc,1836
|
|
|
23
23
|
agent/human_tool/__init__.py,sha256=3m5_g-Fo_0yX5vi7eg-A92oTqO0N3aY929Ajp78HKsE,771
|
|
24
24
|
agent/human_tool/__main__.py,sha256=VsW2BAghlonOuqZbP_xuCsaec9bemA1I_ibnDcED9D4,1068
|
|
25
25
|
agent/human_tool/server.py,sha256=ceuL5kw_RjgAi8fueLU3nTjyzOLE25Shv1oTJnSHsoQ,7964
|
|
26
|
-
agent/human_tool/ui.py,sha256=
|
|
26
|
+
agent/human_tool/ui.py,sha256=c5IbzVbj6dTtrswK3KKo6svjatQhQHHwzFA848U2Cw0,28130
|
|
27
27
|
agent/integrations/hud/__init__.py,sha256=q0QEyJZSrcjiN2sRi_hoX-ePmLyYm9CpAIvA0xMxGJI,8360
|
|
28
28
|
agent/integrations/hud/proxy.py,sha256=yA7C2jeXnrpI5HS0VgCvn0BflVbAORZynIfyE27rvBg,7782
|
|
29
29
|
agent/loops/__init__.py,sha256=Ef8aj07l3osibwDk-DTo80PrpL4_GdKRTP1ikl_b-BQ,328
|
|
@@ -45,7 +45,7 @@ agent/ui/__main__.py,sha256=vudWXYvGM0aNT5aZ94HPtGW8YXOZ4cLXepHyhUM_k1g,73
|
|
|
45
45
|
agent/ui/gradio/__init__.py,sha256=yv4Mrfo-Sj2U5sVn_UJHAuwYCezo-5O4ItR2C9jzNko,145
|
|
46
46
|
agent/ui/gradio/app.py,sha256=Ol97YEbwREZZQ9_PMjVHlfOcu9BGsawxgAGAm79hT80,9117
|
|
47
47
|
agent/ui/gradio/ui_components.py,sha256=dJUvKDmc1oSejtoR_gU_oWWYwxaOOQyPloSYRGMrUCQ,36068
|
|
48
|
-
cua_agent-0.4.
|
|
49
|
-
cua_agent-0.4.
|
|
50
|
-
cua_agent-0.4.
|
|
51
|
-
cua_agent-0.4.
|
|
48
|
+
cua_agent-0.4.24.dist-info/METADATA,sha256=-yvFHUziugRMdDqtf_NDVnQfcNbHKut_rr-yswIDYkM,12712
|
|
49
|
+
cua_agent-0.4.24.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90
|
|
50
|
+
cua_agent-0.4.24.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34
|
|
51
|
+
cua_agent-0.4.24.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|