screenhand 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +165 -446
- package/bin/darwin-arm64/macos-bridge +0 -0
- package/dist/mcp-desktop.js +3615 -400
- package/dist/scripts/export-help-center.js +112 -0
- package/dist/scripts/marketing-loop.js +117 -0
- package/dist/scripts/observer-daemon.js +288 -0
- package/dist/scripts/orchestrator-daemon.js +399 -0
- package/dist/scripts/threads-campaign.js +208 -0
- package/dist/src/community/fetcher.js +109 -0
- package/dist/src/community/index.js +6 -0
- package/dist/src/community/publisher.js +191 -0
- package/dist/src/community/remote-api.js +121 -0
- package/dist/src/community/types.js +3 -0
- package/dist/src/community/validator.js +95 -0
- package/dist/src/context-tracker.js +489 -0
- package/dist/src/ingestion/coverage-auditor.js +233 -0
- package/dist/src/ingestion/doc-parser.js +164 -0
- package/dist/src/ingestion/index.js +8 -0
- package/dist/src/ingestion/menu-scanner.js +152 -0
- package/dist/src/ingestion/reference-merger.js +186 -0
- package/dist/src/ingestion/shortcut-extractor.js +180 -0
- package/dist/src/ingestion/tutorial-extractor.js +170 -0
- package/dist/src/ingestion/types.js +3 -0
- package/dist/src/jobs/manager.js +82 -14
- package/dist/src/jobs/runner.js +138 -15
- package/dist/src/learning/engine.js +356 -0
- package/dist/src/learning/index.js +9 -0
- package/dist/src/learning/locator-policy.js +120 -0
- package/dist/src/learning/pattern-policy.js +89 -0
- package/dist/src/learning/recovery-policy.js +116 -0
- package/dist/src/learning/sensor-policy.js +115 -0
- package/dist/src/learning/timing-model.js +204 -0
- package/dist/src/learning/topology-policy.js +90 -0
- package/dist/src/learning/types.js +9 -0
- package/dist/src/logging/timeline-logger.js +4 -1
- package/dist/src/memory/playbook-seeds.js +200 -0
- package/dist/src/memory/recall.js +60 -8
- package/dist/src/memory/service.js +30 -5
- package/dist/src/memory/store.js +34 -5
- package/dist/src/native/bridge-client.js +253 -31
- package/dist/src/observer/state.js +199 -0
- package/dist/src/observer/types.js +43 -0
- package/dist/src/orchestrator/state.js +68 -0
- package/dist/src/orchestrator/types.js +22 -0
- package/dist/src/perception/ax-source.js +162 -0
- package/dist/src/perception/cdp-source.js +162 -0
- package/dist/src/perception/coordinator.js +771 -0
- package/dist/src/perception/frame-differ.js +287 -0
- package/dist/src/perception/index.js +22 -0
- package/dist/src/perception/manager.js +199 -0
- package/dist/src/perception/types.js +47 -0
- package/dist/src/perception/vision-source.js +399 -0
- package/dist/src/planner/deterministic.js +298 -0
- package/dist/src/planner/executor.js +870 -0
- package/dist/src/planner/goal-store.js +92 -0
- package/dist/src/planner/index.js +21 -0
- package/dist/src/planner/planner.js +520 -0
- package/dist/src/planner/tool-registry.js +71 -0
- package/dist/src/planner/types.js +22 -0
- package/dist/src/platform/explorer.js +213 -0
- package/dist/src/platform/help-center-markdown.js +527 -0
- package/dist/src/platform/learner.js +257 -0
- package/dist/src/playbook/engine.js +296 -11
- package/dist/src/playbook/mcp-recorder.js +204 -0
- package/dist/src/playbook/recorder.js +3 -2
- package/dist/src/playbook/runner.js +1 -1
- package/dist/src/playbook/store.js +139 -10
- package/dist/src/recovery/detectors.js +156 -0
- package/dist/src/recovery/engine.js +327 -0
- package/dist/src/recovery/index.js +20 -0
- package/dist/src/recovery/strategies.js +274 -0
- package/dist/src/recovery/types.js +20 -0
- package/dist/src/runtime/accessibility-adapter.js +55 -18
- package/dist/src/runtime/applescript-adapter.js +8 -2
- package/dist/src/runtime/cdp-chrome-adapter.js +1 -1
- package/dist/src/runtime/executor.js +23 -3
- package/dist/src/runtime/locator-cache.js +24 -2
- package/dist/src/runtime/service.js +59 -15
- package/dist/src/runtime/session-manager.js +4 -1
- package/dist/src/runtime/vision-adapter.js +2 -1
- package/dist/src/state/app-map-types.js +72 -0
- package/dist/src/state/app-map.js +1974 -0
- package/dist/src/state/entity-tracker.js +108 -0
- package/dist/src/state/fusion.js +96 -0
- package/dist/src/state/index.js +21 -0
- package/dist/src/state/ladder-generator.js +236 -0
- package/dist/src/state/persistence.js +156 -0
- package/dist/src/state/types.js +17 -0
- package/dist/src/state/world-model.js +1456 -0
- package/dist/src/util/atomic-write.js +19 -4
- package/dist/src/util/sanitize.js +146 -0
- package/dist-app-maps/com.figma.Desktop.json +959 -0
- package/dist-app-maps/com.hnc.Discord.json +1146 -0
- package/dist-app-maps/notion.id.json +2831 -0
- package/dist-playbooks/canva-screenhand-carousel.json +445 -0
- package/dist-playbooks/codex-desktop.json +76 -0
- package/dist-playbooks/competitor-research-stack.json +122 -0
- package/dist-playbooks/davinci-color-grade.json +153 -0
- package/dist-playbooks/davinci-edit-timeline.json +162 -0
- package/dist-playbooks/davinci-render.json +114 -0
- package/dist-playbooks/devto.json +52 -0
- package/dist-playbooks/discord.json +41 -0
- package/dist-playbooks/google-flow-create-project.json +59 -0
- package/dist-playbooks/google-flow-edit-image.json +90 -0
- package/dist-playbooks/google-flow-edit-video.json +90 -0
- package/dist-playbooks/google-flow-generate-image.json +68 -0
- package/dist-playbooks/google-flow-generate-video.json +191 -0
- package/dist-playbooks/google-flow-open-project.json +48 -0
- package/dist-playbooks/google-flow-open-scenebuilder.json +64 -0
- package/dist-playbooks/google-flow-search-assets.json +64 -0
- package/dist-playbooks/instagram.json +57 -0
- package/dist-playbooks/linkedin.json +52 -0
- package/dist-playbooks/n8n.json +43 -0
- package/dist-playbooks/reddit.json +52 -0
- package/dist-playbooks/threads.json +59 -0
- package/dist-playbooks/x-twitter.json +59 -0
- package/dist-playbooks/youtube.json +59 -0
- package/dist-references/canva.json +646 -0
- package/dist-references/codex-desktop.json +305 -0
- package/dist-references/davinci-resolve-keyboard.json +594 -0
- package/dist-references/davinci-resolve-menu-map.json +1139 -0
- package/dist-references/davinci-resolve-menus-batch1.json +116 -0
- package/dist-references/davinci-resolve-menus-batch2.json +372 -0
- package/dist-references/davinci-resolve-menus-batch3.json +330 -0
- package/dist-references/davinci-resolve-menus-batch4.json +297 -0
- package/dist-references/davinci-resolve-shortcuts.json +333 -0
- package/dist-references/devpost.json +186 -0
- package/dist-references/devto.json +317 -0
- package/dist-references/discord.json +549 -0
- package/dist-references/figma.json +1186 -0
- package/dist-references/finder.json +146 -0
- package/dist-references/google-ads-transparency.json +95 -0
- package/dist-references/google-flow.json +649 -0
- package/dist-references/instagram.json +341 -0
- package/dist-references/linkedin.json +324 -0
- package/dist-references/meta-ad-library.json +86 -0
- package/dist-references/n8n.json +387 -0
- package/dist-references/notes.json +27 -0
- package/dist-references/notion.json +163 -0
- package/dist-references/reddit.json +341 -0
- package/dist-references/threads.json +337 -0
- package/dist-references/x-twitter.json +403 -0
- package/dist-references/youtube.json +373 -0
- package/native/macos-bridge/Package.swift +22 -0
- package/native/macos-bridge/Sources/AccessibilityBridge.swift +482 -0
- package/native/macos-bridge/Sources/AppManagement.swift +339 -0
- package/native/macos-bridge/Sources/CoreGraphicsBridge.swift +537 -0
- package/native/macos-bridge/Sources/ObserverBridge.swift +120 -0
- package/native/macos-bridge/Sources/StreamCapture.swift +136 -0
- package/native/macos-bridge/Sources/VisionBridge.swift +238 -0
- package/native/macos-bridge/Sources/main.swift +498 -0
- package/native/windows-bridge/AppManagement.cs +234 -0
- package/native/windows-bridge/InputBridge.cs +436 -0
- package/native/windows-bridge/Program.cs +270 -0
- package/native/windows-bridge/ScreenCapture.cs +453 -0
- package/native/windows-bridge/UIAutomationBridge.cs +571 -0
- package/native/windows-bridge/WindowsBridge.csproj +17 -0
- package/package.json +12 -1
- package/scripts/postinstall.cjs +127 -0
- package/dist/.audit-log.jsonl +0 -55
- package/dist/.screenhand/memory/.lock +0 -1
- package/dist/.screenhand/memory/actions.jsonl +0 -85
- package/dist/.screenhand/memory/errors.jsonl +0 -5
- package/dist/.screenhand/memory/errors.jsonl.bak +0 -4
- package/dist/.screenhand/memory/state.json +0 -35
- package/dist/.screenhand/memory/state.json.bak +0 -35
- package/dist/.screenhand/memory/strategies.jsonl +0 -12
- package/dist/agent/cli.js +0 -73
- package/dist/agent/loop.js +0 -258
- package/dist/config.js +0 -9
- package/dist/index.js +0 -56
- package/dist/logging/timeline-logger.js +0 -29
- package/dist/mcp/mcp-stdio-server.js +0 -448
- package/dist/mcp/server.js +0 -347
- package/dist/mcp-entry.js +0 -59
- package/dist/memory/recall.js +0 -160
- package/dist/memory/research.js +0 -98
- package/dist/memory/seeds.js +0 -89
- package/dist/memory/session.js +0 -161
- package/dist/memory/store.js +0 -391
- package/dist/memory/types.js +0 -4
- package/dist/monitor/codex-monitor.js +0 -377
- package/dist/monitor/task-queue.js +0 -84
- package/dist/monitor/types.js +0 -49
- package/dist/native/bridge-client.js +0 -174
- package/dist/native/macos-bridge-client.js +0 -5
- package/dist/npm-publish-helper.js +0 -117
- package/dist/npm-token-cdp.js +0 -113
- package/dist/npm-token-create.js +0 -135
- package/dist/npm-token-finish.js +0 -126
- package/dist/playbook/engine.js +0 -193
- package/dist/playbook/index.js +0 -4
- package/dist/playbook/recorder.js +0 -519
- package/dist/playbook/runner.js +0 -392
- package/dist/playbook/store.js +0 -166
- package/dist/playbook/types.js +0 -4
- package/dist/runtime/accessibility-adapter.js +0 -377
- package/dist/runtime/app-adapter.js +0 -48
- package/dist/runtime/applescript-adapter.js +0 -283
- package/dist/runtime/ax-role-map.js +0 -80
- package/dist/runtime/browser-adapter.js +0 -36
- package/dist/runtime/cdp-chrome-adapter.js +0 -505
- package/dist/runtime/composite-adapter.js +0 -205
- package/dist/runtime/executor.js +0 -250
- package/dist/runtime/locator-cache.js +0 -12
- package/dist/runtime/planning-loop.js +0 -47
- package/dist/runtime/service.js +0 -372
- package/dist/runtime/session-manager.js +0 -28
- package/dist/runtime/state-observer.js +0 -105
- package/dist/runtime/vision-adapter.js +0 -208
- package/dist/test-mcp-protocol.js +0 -138
- package/dist/types.js +0 -1
|
@@ -0,0 +1,649 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "google-flow",
|
|
3
|
+
"name": "Google Flow Automation",
|
|
4
|
+
"description": "Curated automation reference for Google Flow as observed live in Chrome on March 11, 2026. Combines the live dashboard, project grid, image editor, and video editor with official Google Flow help guidance for generation, image editing, clip editing, Scenebuilder, collections, and model support, plus a verified state-level prompt submission path through Flow's internal Slate store.",
|
|
5
|
+
"platform": "google-flow",
|
|
6
|
+
"version": "2026-03-11",
|
|
7
|
+
"urlPatterns": [
|
|
8
|
+
"*labs.google/fx/tools/flow*",
|
|
9
|
+
"*labs.google/fx/tools/flow/project/*",
|
|
10
|
+
"*labs.google/flow/about*",
|
|
11
|
+
"*support.google.com/flow*",
|
|
12
|
+
"*support.google.com/labs*"
|
|
13
|
+
],
|
|
14
|
+
"tags": [
|
|
15
|
+
"google-flow",
|
|
16
|
+
"flow",
|
|
17
|
+
"video",
|
|
18
|
+
"image",
|
|
19
|
+
"browser",
|
|
20
|
+
"cdp",
|
|
21
|
+
"google-labs"
|
|
22
|
+
],
|
|
23
|
+
"successCount": 0,
|
|
24
|
+
"failCount": 0,
|
|
25
|
+
"urls": {
|
|
26
|
+
"home": "https://labs.google/fx/tools/flow",
|
|
27
|
+
"project": "https://labs.google/fx/tools/flow/project/{projectId}",
|
|
28
|
+
"about": "https://labs.google/flow/about",
|
|
29
|
+
"faq": "https://labs.google/fx/tools/flow/faq",
|
|
30
|
+
"help_center": "https://support.google.com/flow#topic=16908930",
|
|
31
|
+
"help_get_started": "https://support.google.com/flow/answer/16353333?hl=en",
|
|
32
|
+
"help_create_videos": "https://support.google.com/flow/answer/16353334?hl=en",
|
|
33
|
+
"help_edit_videos": "https://support.google.com/flow/answer/16935718?hl=en",
|
|
34
|
+
"help_edit_images": "https://support.google.com/labs/answer/16729550?hl=en",
|
|
35
|
+
"help_projects_assets": "https://support.google.com/flow/answer/16935308?hl=en",
|
|
36
|
+
"help_models": "https://support.google.com/flow/answer/16352836?hl=en",
|
|
37
|
+
"flow_tv": "https://labs.google/flow/tv"
|
|
38
|
+
},
|
|
39
|
+
"selectors": {
|
|
40
|
+
"dashboard": {
|
|
41
|
+
"home_link": "a[href='/fx/tools/flow']",
|
|
42
|
+
"flow_tv_link": "a[href='https://labs.google/flow/tv']",
|
|
43
|
+
"volume_button": "button[aria-label='Adjust volume']",
|
|
44
|
+
"project_cards": "a[href*='/fx/tools/flow/project/']",
|
|
45
|
+
"new_project_button": "visible button whose text includes 'New project'",
|
|
46
|
+
"edit_project_button": "visible button whose text includes 'Edit project'",
|
|
47
|
+
"delete_project_button": "visible button whose text includes 'Delete project'",
|
|
48
|
+
"help_center_button": "visible button whose text includes 'Flow Help Center'",
|
|
49
|
+
"_project_card_note": "Project cards are reliable anchors because they use real links to /fx/tools/flow/project/{id}. The dashboard card controls are easier to resolve by visible text than by generated class names.",
|
|
50
|
+
"_new_project_note": "The current dashboard DOM does not expose a stable aria-label or data-testid for New project. Prefer JS text lookup over styled-components class names."
|
|
51
|
+
},
|
|
52
|
+
"project_header": {
|
|
53
|
+
"title_input": "input[aria-label='Editable text']",
|
|
54
|
+
"search_input": "form input[type='text']",
|
|
55
|
+
"search_button": "form button",
|
|
56
|
+
"sort_filter_button": "visible button with text 'Sort & Filter'",
|
|
57
|
+
"add_media_button": "visible button with text 'Add Media'",
|
|
58
|
+
"scenebuilder_button": "visible button with text 'Scenebuilder'",
|
|
59
|
+
"grid_settings_button": "visible button with text 'View Tile Grid Settings'",
|
|
60
|
+
"page_menu_button": "visible button with text 'More'",
|
|
61
|
+
"view_dashboard_button": "visible button with text 'View full dashboard'",
|
|
62
|
+
"view_images_button": "visible button with text 'View images'",
|
|
63
|
+
"view_videos_button": "visible button with text 'View videos'",
|
|
64
|
+
"_radix_menu_note": "Flow uses Radix menu triggers with runtime ids such as radix-:r16:. Do not hardcode those ids; resolve each menu button by visible text."
|
|
65
|
+
},
|
|
66
|
+
"prompt_panel": {
|
|
67
|
+
"prompt_box": "[role='textbox'][contenteditable='true'][data-slate-editor='true']",
|
|
68
|
+
"swap_frames_button": "visible button with text 'Swap first and last frames'",
|
|
69
|
+
"generation_mode_button": "the prompt tray settings button whose text includes crop_16_9 or crop_9_16 plus x1/x2/x3/x4",
|
|
70
|
+
"create_button": "the rightmost prompt tray button with the arrow_forward icon and accessible text 'Create'",
|
|
71
|
+
"_prompt_box_note": "The prompt editor is a Slate contenteditable div backed by an internal promptBoxStore and a Slate textEditor instance. Direct .value assignment does nothing, and naive DOM text injection can leave the prompt inside the placeholder/zero-width DOM without updating Flow's real state.",
|
|
72
|
+
"_prompt_store_note": "Verified live on March 11, 2026: promptBoxStore.getState() exposes mode, aspectRatio, outputsPerPrompt, imageModelFamilyId, videoModelFamilyId, inputs.textEditor, and actions such as setMode, setAspectRatio, setOutputsPerPrompt, setImageModelFamily, setVideoModelFamily, setImageEditOptions, and setVideoEditOptions.",
|
|
73
|
+
"_prompt_insert_note": "UPDATED March 11, 2026: The Slate API insertText path (promptBoxStore.getState().inputs.textEditor) can desync from the DOM. The RELIABLE path is: browser_click on the Slate editor to focus it, then browser_type with CDP keyboard events. This keeps Slate and DOM in sync and allows the Create button click to actually submit.",
|
|
74
|
+
"_submit_note": "UPDATED March 11, 2026: The programmatic onSubmit() handler from the React fiber tree fires without error but silently does nothing for video generation. The RELIABLE submit path is: assign a unique ID to the Create button via browser_js, then use browser_human_click on that ID. This was verified in a live video generation run.",
|
|
75
|
+
"_create_label_note": "Official help articles still say 'Generate', but the current project UI observed on March 11, 2026 labels the submit button 'Create'."
|
|
76
|
+
},
|
|
77
|
+
"settings_panel": {
|
|
78
|
+
"_open_note": "Opened by clicking the settings button in the prompt tray (text includes crop_16_9 or crop_9_16 plus x1/x2/x3/x4). Closed by clicking the same button again.",
|
|
79
|
+
"image_video_tabs": {
|
|
80
|
+
"image_tab": "[role='tab'] with text 'imageImage' (icon + label)",
|
|
81
|
+
"video_tab": "[role='tab'] with text 'videocamVideo' (icon + label)",
|
|
82
|
+
"_note": "Use aria-selected='true' to check current selection"
|
|
83
|
+
},
|
|
84
|
+
"generation_mode_tabs": {
|
|
85
|
+
"frames_tab": "[role='tab'] with text 'crop_freeFrames' — for image-to-video workflows using start/end frames",
|
|
86
|
+
"ingredients_tab": "[role='tab'] with text 'chrome_extensionIngredients' — for referencing uploaded media assets",
|
|
87
|
+
"_note": "Frames is preferred for creating video from images. Ingredients lets you @ reference uploaded assets."
|
|
88
|
+
},
|
|
89
|
+
"aspect_ratio_tabs": {
|
|
90
|
+
"landscape_tab": "[role='tab'] with text 'crop_16_9Landscape'",
|
|
91
|
+
"portrait_tab": "[role='tab'] with text 'crop_9_16Portrait'",
|
|
92
|
+
"_note": "Use aria-selected='true' to check current selection"
|
|
93
|
+
},
|
|
94
|
+
"output_count_tabs": {
|
|
95
|
+
"x1_tab": "[role='tab'] with text 'x1'",
|
|
96
|
+
"x2_tab": "[role='tab'] with text 'x2'",
|
|
97
|
+
"x3_tab": "[role='tab'] with text 'x3'",
|
|
98
|
+
"x4_tab": "[role='tab'] with text 'x4'",
|
|
99
|
+
"_note": "More outputs = more credits. Video generation at x1 costs 20 credits on Veo 3.1 Fast."
|
|
100
|
+
},
|
|
101
|
+
"model_dropdown": {
|
|
102
|
+
"trigger": "button with text including 'arrow_drop_down' and 'Veo'",
|
|
103
|
+
"options_observed": ["Veo 3.1 - Fast", "Veo 3.1 - Quality", "Veo 2 - Fast"],
|
|
104
|
+
"_note": "Veo 3.1 Fast is the default and most cost-effective option."
|
|
105
|
+
}
|
|
106
|
+
},
|
|
107
|
+
"project_assets": {
|
|
108
|
+
"search_input": "form input[type='text']",
|
|
109
|
+
"filters_button": "visible button with text 'Sort & Filter'",
|
|
110
|
+
"collection_menu": "Add Media menu, then 'Create Collection'",
|
|
111
|
+
"asset_tiles": "[id^='fe_id_'], a[href*='/edit/']",
|
|
112
|
+
"image_edit_links": "visible a[href*='/edit/'] tiles whose container does not include a video element",
|
|
113
|
+
"video_edit_links": "visible a[href*='/edit/'] tiles whose container includes a video element or play_circle overlay",
|
|
114
|
+
"clip_more_menu": "hovered asset card action with text 'More'",
|
|
115
|
+
"selected_asset_favorite_button": "visible button with text 'Favorite' after selecting a successful asset tile",
|
|
116
|
+
"selected_asset_reuse_prompt_button": "visible button with text 'Reuse prompt' after selecting a successful asset tile",
|
|
117
|
+
"selected_asset_more_button": "visible button with text 'More' next to Favorite and Reuse prompt after selecting a successful asset tile",
|
|
118
|
+
"_asset_note": "Many asset actions are hover-only. Expect to use browser_human_click or OCR-assisted hover when the media grid is densely packed."
|
|
119
|
+
},
|
|
120
|
+
"image_editor": {
|
|
121
|
+
"back_button": "visible button with text 'Back'",
|
|
122
|
+
"media_info_button": "visible button with text 'Get more info about this media'",
|
|
123
|
+
"download_button": "visible button with text 'Download'",
|
|
124
|
+
"history_button": "visible button with text 'Hide history'",
|
|
125
|
+
"done_button": "visible button with text 'Done'",
|
|
126
|
+
"crop_tool_button": "visible button with text 'Crop'",
|
|
127
|
+
"select_tool_button": "visible button with text 'Select'",
|
|
128
|
+
"draw_tool_button": "visible button with text 'Draw'",
|
|
129
|
+
"prompt_box": "[role='textbox'][contenteditable='true'][data-slate-editor='true']",
|
|
130
|
+
"create_button": "the image edit prompt tray button with accessible text 'Create'",
|
|
131
|
+
"model_button": "the edit model button that shows a Nano Banana label and aspect ratio",
|
|
132
|
+
"reuse_prompt_button": "visible button with text 'Reuse text prompt'",
|
|
133
|
+
"expand_prompt_button": "visible button with text 'Expand prompt'",
|
|
134
|
+
"crop_landscape_button": "visible button with text 'Landscape (16:9)'",
|
|
135
|
+
"crop_portrait_button": "visible button with text 'Portrait (9:16)'",
|
|
136
|
+
"crop_square_button": "visible button with text 'Square (1:1)'",
|
|
137
|
+
"crop_freeform_button": "visible button with text 'Freeform'",
|
|
138
|
+
"crop_drag_handles": ".ReactCrop__drag-handle",
|
|
139
|
+
"select_box_button": "visible button with text 'Box'",
|
|
140
|
+
"select_lasso_button": "visible button with text 'Lasso'",
|
|
141
|
+
"draw_freehand_button": "visible button whose icon text includes 'draw'",
|
|
142
|
+
"draw_text_button": "visible button whose icon text includes 'text_fields'",
|
|
143
|
+
"draw_rectangle_button": "visible button whose icon text includes 'rectangle'",
|
|
144
|
+
"_image_edit_note": "Live image edit view exposes Crop, Select, and Draw as left-rail tools. Crop uses draggable handles, Select offers Box and Lasso, and Draw exposes a small overlay toolbar with draw/text/rectangle plus undo, redo, refresh, and size controls."
|
|
145
|
+
},
|
|
146
|
+
"video_editor": {
|
|
147
|
+
"back_button": "visible button with text 'Back'",
|
|
148
|
+
"media_info_button": "visible button with text 'Get more info about this media'",
|
|
149
|
+
"download_button": "visible button with text 'Download'",
|
|
150
|
+
"history_button": "visible button with text 'Hide history'",
|
|
151
|
+
"done_button": "visible button with text 'Done'",
|
|
152
|
+
"extend_button": "visible button with text 'Extend'",
|
|
153
|
+
"insert_button": "visible button with text 'Insert'",
|
|
154
|
+
"remove_button": "visible button with text 'Remove'",
|
|
155
|
+
"camera_button": "visible button with text 'Camera'",
|
|
156
|
+
"camera_motion_tab": "visible button with text 'Camera motion'",
|
|
157
|
+
"camera_position_tab": "visible button with text 'Camera position'",
|
|
158
|
+
"camera_motion_dolly_in_button": "visible button with text 'Dolly in'",
|
|
159
|
+
"camera_motion_dolly_out_button": "visible button with text 'Dolly out'",
|
|
160
|
+
"camera_motion_orbit_left_button": "visible button with text 'Orbit left'",
|
|
161
|
+
"camera_motion_orbit_right_button": "visible button with text 'Orbit right'",
|
|
162
|
+
"camera_motion_orbit_up_button": "visible button with text 'Orbit up'",
|
|
163
|
+
"camera_motion_orbit_low_button": "visible button with text 'Orbit low'",
|
|
164
|
+
"camera_motion_dolly_in_zoom_out_button": "visible button with text 'Dolly in zoom out'",
|
|
165
|
+
"camera_motion_dolly_out_zoom_in_button": "visible button with text 'Dolly out zoom in'",
|
|
166
|
+
"camera_position_center_button": "visible button with text 'Center'",
|
|
167
|
+
"camera_position_left_button": "visible button with text 'Left'",
|
|
168
|
+
"camera_position_right_button": "visible button with text 'Right'",
|
|
169
|
+
"camera_position_high_button": "visible button with text 'High'",
|
|
170
|
+
"camera_position_low_button": "visible button with text 'Low'",
|
|
171
|
+
"camera_position_closer_button": "visible button with text 'Closer'",
|
|
172
|
+
"camera_position_further_button": "visible button with text 'Further'",
|
|
173
|
+
"prompt_box": "[role='textbox'][contenteditable='true'][data-slate-editor='true']",
|
|
174
|
+
"create_button": "the video edit prompt tray button with accessible text 'Create'",
|
|
175
|
+
"reuse_prompt_button": "visible button with text 'Reuse text prompt'",
|
|
176
|
+
"_video_edit_note": "Live video edit view exposes Extend, Insert, Remove, and Camera. Insert and Remove display region-selection instructions. Camera exposes motion and position tabs with preset buttons."
|
|
177
|
+
},
|
|
178
|
+
"help_center": {
|
|
179
|
+
"search_button": "button[aria-label='Search']",
|
|
180
|
+
"topic_link_create_videos": "a[href*='/flow/answer/16353334']",
|
|
181
|
+
"topic_link_edit_videos": "a[href*='/flow/answer/16935718']",
|
|
182
|
+
"topic_link_edit_images": "a[href*='/labs/answer/16729550']",
|
|
183
|
+
"topic_link_projects_assets": "a[href*='/flow/answer/16935308']",
|
|
184
|
+
"topic_link_models": "a[href*='/flow/answer/16352836']"
|
|
185
|
+
}
|
|
186
|
+
},
|
|
187
|
+
"keyboard_shortcuts": {
|
|
188
|
+
"documented": [],
|
|
189
|
+
"notes": "No official Google Flow keyboard shortcut list was surfaced on the Flow about page or help center during this pass. Treat Flow as a pointer-first browser app and automate primarily through DOM selectors, browser_js, hover interactions, and screenshots."
|
|
190
|
+
},
|
|
191
|
+
"detection": {
|
|
192
|
+
"is_dashboard": "location.hostname === 'labs.google' && location.pathname === '/fx/tools/flow' && document.body.innerText.includes('New project')",
|
|
193
|
+
"is_project": "location.hostname === 'labs.google' && location.pathname.includes('/fx/tools/flow/project/') && !location.pathname.includes('/edit/')",
|
|
194
|
+
"has_project_cards": "document.querySelectorAll(\"a[href*='/fx/tools/flow/project/']\").length > 0",
|
|
195
|
+
"has_prompt_box": "!!document.querySelector(\"[role='textbox'][contenteditable='true'][data-slate-editor='true']\")",
|
|
196
|
+
"has_title_input": "!!document.querySelector(\"input[aria-label='Editable text']\")",
|
|
197
|
+
"is_image_editor": "location.hostname === 'labs.google' && location.pathname.includes('/edit/') && document.body.innerText.includes('What do you want to change?') && document.body.innerText.includes('Crop')",
|
|
198
|
+
"is_video_editor": "location.hostname === 'labs.google' && location.pathname.includes('/edit/') && document.body.innerText.includes('Extend') && document.body.innerText.includes('Insert') && document.body.innerText.includes('Remove')",
|
|
199
|
+
"is_help_center": "location.hostname === 'support.google.com' && (location.pathname.startsWith('/flow') || location.pathname.startsWith('/labs'))",
|
|
200
|
+
"is_about_page": "location.hostname === 'labs.google' && location.pathname === '/flow/about'"
|
|
201
|
+
},
|
|
202
|
+
"flows": {
|
|
203
|
+
"open_existing_project": {
|
|
204
|
+
"guards": [
|
|
205
|
+
"Must already be signed into an eligible Google account.",
|
|
206
|
+
"Project cards must be visible on https://labs.google/fx/tools/flow."
|
|
207
|
+
],
|
|
208
|
+
"why": "Project cards are real anchor tags to /project/{id}, which is more reliable than chasing hover-only card chrome.",
|
|
209
|
+
"verification_text_patterns": [
|
|
210
|
+
"Create",
|
|
211
|
+
"Scenebuilder",
|
|
212
|
+
"Add Media"
|
|
213
|
+
],
|
|
214
|
+
"steps": [
|
|
215
|
+
"Navigate to https://labs.google/fx/tools/flow or reuse the dashboard tab.",
|
|
216
|
+
"Find the desired project card via a[href*='/fx/tools/flow/project/'] and click the anchor itself, not the overlay controls.",
|
|
217
|
+
"Wait for the URL to include /fx/tools/flow/project/.",
|
|
218
|
+
"Verify that input[aria-label='Editable text'] and the Slate prompt box are visible before continuing."
|
|
219
|
+
]
|
|
220
|
+
},
|
|
221
|
+
"create_project": {
|
|
222
|
+
"guards": [
|
|
223
|
+
"Dashboard must be loaded and show the New project button."
|
|
224
|
+
],
|
|
225
|
+
"why": "The dashboard is the stable entry point for all new Flow workspaces.",
|
|
226
|
+
"verification_text_patterns": [
|
|
227
|
+
"Create",
|
|
228
|
+
"What do you want to create?",
|
|
229
|
+
"Scenebuilder"
|
|
230
|
+
],
|
|
231
|
+
"steps": [
|
|
232
|
+
"Open https://labs.google/fx/tools/flow.",
|
|
233
|
+
"Resolve the visible New project button by text and click it.",
|
|
234
|
+
"Wait for a new /project/{id} URL to load.",
|
|
235
|
+
"Optional: rename the generated title through input[aria-label='Editable text'] once the editor is ready."
|
|
236
|
+
]
|
|
237
|
+
},
|
|
238
|
+
"rename_project_in_editor": {
|
|
239
|
+
"guards": [
|
|
240
|
+
"Must already be inside a project editor."
|
|
241
|
+
],
|
|
242
|
+
"why": "The title field is a normal input, so renaming is simpler and more reliable here than through hover-only dashboard controls.",
|
|
243
|
+
"verification_text_patterns": [
|
|
244
|
+
"Editable text"
|
|
245
|
+
],
|
|
246
|
+
"steps": [
|
|
247
|
+
"Focus input[aria-label='Editable text'].",
|
|
248
|
+
"Replace the auto-generated project title with browser_fill_form or a native input setter.",
|
|
249
|
+
"Commit with Enter or blur the field.",
|
|
250
|
+
"Verify the input value persists after focus leaves the field."
|
|
251
|
+
]
|
|
252
|
+
},
|
|
253
|
+
"generate_video_from_text_prompt": {
|
|
254
|
+
"guards": [
|
|
255
|
+
"Must already be in a project editor.",
|
|
256
|
+
"This action consumes AI credits."
|
|
257
|
+
],
|
|
258
|
+
"why": "The current editor is only partially DOM-addressable. The reliable path is state-level automation through Flow's internal promptBoxStore and Slate textEditor, then calling the submit component's internal onSubmit handler.",
|
|
259
|
+
"verification_text_patterns": [
|
|
260
|
+
"Create",
|
|
261
|
+
"Video",
|
|
262
|
+
"Flow can make mistakes, so double check it"
|
|
263
|
+
],
|
|
264
|
+
"steps": [
|
|
265
|
+
"Resolve promptBoxStore from the prompt box React fiber, then use promptBoxStore.getState().actions.setMode('VIDEO') and the related settings actions instead of clicking the settings menu by class name.",
|
|
266
|
+
"Insert the prompt text into promptBoxStore.getState().inputs.textEditor with Slate methods such as select, deleteFragment, insertText, and onChange.",
|
|
267
|
+
"Invoke the internal onSubmit() handler from the prompt tray submit component instead of relying on DOM .click().",
|
|
268
|
+
"Poll for a new generation API call such as video:batchAsyncGenerateVideoText or for a new tile/workflow to appear in the grid.",
|
|
269
|
+
"Verify the resulting clip visually before download, edit, or reuse."
|
|
270
|
+
]
|
|
271
|
+
},
|
|
272
|
+
"generate_image_from_text_prompt": {
|
|
273
|
+
"guards": [
|
|
274
|
+
"Must already be in a project editor.",
|
|
275
|
+
"Image mode is the most reliable live-tested automation path from this pass."
|
|
276
|
+
],
|
|
277
|
+
"why": "Image generation was verified live in the project editor by setting mode through the promptBoxStore, inserting prompt text through the Slate textEditor, then calling the internal onSubmit handler. That run created a new ScreenHand image tile in the grid.",
|
|
278
|
+
"verification_text_patterns": [
|
|
279
|
+
"Create",
|
|
280
|
+
"🍌 Nano Banana 2",
|
|
281
|
+
"Flow can make mistakes, so double check it"
|
|
282
|
+
],
|
|
283
|
+
"steps": [
|
|
284
|
+
"Resolve promptBoxStore from the prompt box React fiber.",
|
|
285
|
+
"Set mode to IMAGE and update aspect ratio and outputsPerPrompt through promptBoxStore.getState().actions.",
|
|
286
|
+
"Insert the prompt text into promptBoxStore.getState().inputs.textEditor with Slate methods such as select, deleteFragment, insertText, and onChange.",
|
|
287
|
+
"Invoke the prompt tray submit component's internal onSubmit() handler.",
|
|
288
|
+
"Poll for flowMedia:batchGenerateImages or a new tile id in the asset grid, then verify the resulting image visually."
|
|
289
|
+
]
|
|
290
|
+
},
|
|
291
|
+
"open_image_editor": {
|
|
292
|
+
"guards": [
|
|
293
|
+
"Must already be inside a project with at least one image asset."
|
|
294
|
+
],
|
|
295
|
+
"why": "Image tiles expose stable edit URLs even when the tile chrome itself has little text.",
|
|
296
|
+
"verification_text_patterns": [
|
|
297
|
+
"What do you want to change?",
|
|
298
|
+
"Crop",
|
|
299
|
+
"Select",
|
|
300
|
+
"Draw"
|
|
301
|
+
],
|
|
302
|
+
"steps": [
|
|
303
|
+
"Optionally click View images to narrow the project grid to image assets.",
|
|
304
|
+
"Open the image tile via its visible a[href*='/edit/'] link.",
|
|
305
|
+
"Verify the image editor shows Crop, Select, Draw, and the prompt tray with Create.",
|
|
306
|
+
"Use the prompt tray or the left-rail tools to continue editing."
|
|
307
|
+
]
|
|
308
|
+
},
|
|
309
|
+
"edit_image_from_prompt": {
|
|
310
|
+
"guards": [
|
|
311
|
+
"Must already be in an image editor view.",
|
|
312
|
+
"This action consumes AI credits."
|
|
313
|
+
],
|
|
314
|
+
"why": "The image editor reuses the same promptBoxStore and Slate textEditor pattern as the main generation tray, so prompt injection and submit can be automated reliably without generic typing.",
|
|
315
|
+
"verification_text_patterns": [
|
|
316
|
+
"What do you want to change?",
|
|
317
|
+
"Create",
|
|
318
|
+
"Reuse text prompt"
|
|
319
|
+
],
|
|
320
|
+
"steps": [
|
|
321
|
+
"Stay in the default image prompt mode or switch to Select or Draw first if the edit should be localized.",
|
|
322
|
+
"Insert the edit prompt into promptBoxStore.getState().inputs.textEditor with Slate methods such as select, deleteFragment, insertText, and onChange.",
|
|
323
|
+
"Invoke the internal onSubmit() handler from the image edit prompt tray.",
|
|
324
|
+
"Poll for an image generation request or a new version in History, then verify the output visually."
|
|
325
|
+
]
|
|
326
|
+
},
|
|
327
|
+
"crop_image": {
|
|
328
|
+
"guards": [
|
|
329
|
+
"Must already be in an image editor view."
|
|
330
|
+
],
|
|
331
|
+
"why": "Crop exposes real aspect buttons and ReactCrop handles in the live DOM.",
|
|
332
|
+
"verification_text_patterns": [
|
|
333
|
+
"Landscape (16:9)",
|
|
334
|
+
"Portrait (9:16)",
|
|
335
|
+
"Square (1:1)",
|
|
336
|
+
"Freeform"
|
|
337
|
+
],
|
|
338
|
+
"steps": [
|
|
339
|
+
"Click Crop in the left rail.",
|
|
340
|
+
"Choose Landscape (16:9), Portrait (9:16), Square (1:1), or Freeform.",
|
|
341
|
+
"Drag the crop handles to the desired bounds.",
|
|
342
|
+
"Click Crop to confirm the framing change."
|
|
343
|
+
]
|
|
344
|
+
},
|
|
345
|
+
"select_region_for_image_edit": {
|
|
346
|
+
"guards": [
|
|
347
|
+
"Must already be in an image editor view."
|
|
348
|
+
],
|
|
349
|
+
"why": "Select is the most direct way to localize an image edit before prompting a change.",
|
|
350
|
+
"verification_text_patterns": [
|
|
351
|
+
"Select",
|
|
352
|
+
"Box",
|
|
353
|
+
"Lasso"
|
|
354
|
+
],
|
|
355
|
+
"steps": [
|
|
356
|
+
"Click Select in the left rail.",
|
|
357
|
+
"Choose Box or Lasso.",
|
|
358
|
+
"Mark the region to be changed on the image.",
|
|
359
|
+
"Describe the change in the prompt box and click Create."
|
|
360
|
+
]
|
|
361
|
+
},
|
|
362
|
+
"draw_guided_image_edit": {
|
|
363
|
+
"guards": [
|
|
364
|
+
"Must already be in an image editor view."
|
|
365
|
+
],
|
|
366
|
+
"why": "Draw mode exposes a small annotation toolbar that can guide the model toward specific regions or overlays.",
|
|
367
|
+
"verification_text_patterns": [
|
|
368
|
+
"Draw"
|
|
369
|
+
],
|
|
370
|
+
"steps": [
|
|
371
|
+
"Click Draw in the left rail.",
|
|
372
|
+
"Use the draw, text, or rectangle overlay tools to mark the image.",
|
|
373
|
+
"Adjust size if needed and use undo, redo, or refresh when marks are incorrect.",
|
|
374
|
+
"Describe the intended change in the prompt box and click Create."
|
|
375
|
+
]
|
|
376
|
+
},
|
|
377
|
+
"reuse_prompt_from_existing_asset": {
|
|
378
|
+
"guards": [
|
|
379
|
+
"Must already have at least one successful asset tile in the current project."
|
|
380
|
+
],
|
|
381
|
+
"why": "Selecting a successful asset exposes Favorite, Reuse prompt, and More actions, which are more stable than reconstructing old prompts from hidden state.",
|
|
382
|
+
"verification_text_patterns": [
|
|
383
|
+
"Favorite",
|
|
384
|
+
"Reuse prompt",
|
|
385
|
+
"More"
|
|
386
|
+
],
|
|
387
|
+
"steps": [
|
|
388
|
+
"Select a successful asset tile in the grid.",
|
|
389
|
+
"Click Reuse prompt to pull the asset's prompt back into the active prompt tray.",
|
|
390
|
+
"Adjust the prompt in the internal Slate editor if needed.",
|
|
391
|
+
"Submit again with the internal onSubmit() handler."
|
|
392
|
+
]
|
|
393
|
+
},
|
|
394
|
+
"add_ingredients_or_frames": {
|
|
395
|
+
"guards": [
|
|
396
|
+
"Must already be in a project editor.",
|
|
397
|
+
"Feature availability varies by model and region."
|
|
398
|
+
],
|
|
399
|
+
"why": "The official Flow help flow supports both uploaded media and project assets referenced inline with @ mentions.",
|
|
400
|
+
"verification_text_patterns": [
|
|
401
|
+
"Add Media",
|
|
402
|
+
"Video",
|
|
403
|
+
"Create"
|
|
404
|
+
],
|
|
405
|
+
"steps": [
|
|
406
|
+
"Use Add Media to upload reference images or videos into the current project, or type @ in the prompt box to reference existing uploaded assets by name.",
|
|
407
|
+
"Open the current Video mode/settings menu and switch into the ingredients or frames path required by the prompt.",
|
|
408
|
+
"For frame-driven generations, add a start frame, an end frame, or both, then describe the transition in the prompt.",
|
|
409
|
+
"For ingredient-driven generations, describe how each ingredient should appear or behave in the clip.",
|
|
410
|
+
"Click Create after confirming the active model supports the requested feature."
|
|
411
|
+
]
|
|
412
|
+
},
|
|
413
|
+
"search_filter_assets_and_collections": {
|
|
414
|
+
"guards": [
|
|
415
|
+
"Must already be inside a project with existing assets."
|
|
416
|
+
],
|
|
417
|
+
"why": "Search, filters, and collections are the main organization features called out in the official project management help.",
|
|
418
|
+
"verification_text_patterns": [
|
|
419
|
+
"Search",
|
|
420
|
+
"Sort & Filter",
|
|
421
|
+
"Add Media"
|
|
422
|
+
],
|
|
423
|
+
"steps": [
|
|
424
|
+
"Type the asset name or keyword into the project search bar.",
|
|
425
|
+
"Open Sort & Filter to narrow the visible grid.",
|
|
426
|
+
"Use Add Media and the Create Collection action to create a collection when you need persistent grouping.",
|
|
427
|
+
"Drag one asset onto another or into a collection when pointer automation is acceptable; nested collections are supported."
|
|
428
|
+
]
|
|
429
|
+
},
|
|
430
|
+
"open_video_editor": {
|
|
431
|
+
"guards": [
|
|
432
|
+
"Must already be inside a project with at least one video asset."
|
|
433
|
+
],
|
|
434
|
+
"why": "Video tiles expose stable edit URLs, and the edit surface is the entry point for Extend, Insert, Remove, and Camera.",
|
|
435
|
+
"verification_text_patterns": [
|
|
436
|
+
"Extend",
|
|
437
|
+
"Insert",
|
|
438
|
+
"Remove",
|
|
439
|
+
"Camera"
|
|
440
|
+
],
|
|
441
|
+
"steps": [
|
|
442
|
+
"Optionally click View videos to narrow the project grid to clips.",
|
|
443
|
+
"Open the video tile via its visible a[href*='/edit/'] link.",
|
|
444
|
+
"Verify the bottom action bar shows Extend, Insert, Remove, and Camera.",
|
|
445
|
+
"Choose the desired edit mode before prompting or downloading."
|
|
446
|
+
]
|
|
447
|
+
},
|
|
448
|
+
"extend_existing_video_clip": {
|
|
449
|
+
"guards": [
|
|
450
|
+
"Must already be in a video editor view.",
|
|
451
|
+
"This action consumes AI credits."
|
|
452
|
+
],
|
|
453
|
+
"why": "Extend is the simplest post-generation video edit because it is prompt-only and does not require on-frame region selection.",
|
|
454
|
+
"verification_text_patterns": [
|
|
455
|
+
"Extend",
|
|
456
|
+
"What happens next?",
|
|
457
|
+
"Create"
|
|
458
|
+
],
|
|
459
|
+
"steps": [
|
|
460
|
+
"Click Extend in the bottom action bar.",
|
|
461
|
+
"Describe how the action should continue in the prompt box.",
|
|
462
|
+
"Invoke the internal onSubmit() handler from the prompt tray.",
|
|
463
|
+
"Poll for a new video generation workflow and then review the result in History."
|
|
464
|
+
]
|
|
465
|
+
},
|
|
466
|
+
"insert_object_into_video": {
|
|
467
|
+
"guards": [
|
|
468
|
+
"Must already be in a video editor view."
|
|
469
|
+
],
|
|
470
|
+
"why": "Insert combines a prompt with an optional frame-region selection so the added object appears in the intended place.",
|
|
471
|
+
"verification_text_patterns": [
|
|
472
|
+
"Insert",
|
|
473
|
+
"Describe what you'd like to add"
|
|
474
|
+
],
|
|
475
|
+
"steps": [
|
|
476
|
+
"Click Insert in the bottom action bar.",
|
|
477
|
+
"Optionally click and drag on the frame to mark the insertion location.",
|
|
478
|
+
"Describe the object to add in the prompt box.",
|
|
479
|
+
"Click Create."
|
|
480
|
+
]
|
|
481
|
+
},
|
|
482
|
+
"remove_object_from_video": {
|
|
483
|
+
"guards": [
|
|
484
|
+
"Must already be in a video editor view."
|
|
485
|
+
],
|
|
486
|
+
"why": "Remove is region-first and does not depend on text entry unless you want extra context.",
|
|
487
|
+
"verification_text_patterns": [
|
|
488
|
+
"Remove",
|
|
489
|
+
"Click-and-drag to fully select what you want to remove from the video."
|
|
490
|
+
],
|
|
491
|
+
"steps": [
|
|
492
|
+
"Click Remove in the bottom action bar.",
|
|
493
|
+
"Draw a box around the object to remove from the video.",
|
|
494
|
+
"Click Create to generate the edited clip."
|
|
495
|
+
]
|
|
496
|
+
},
|
|
497
|
+
"change_video_camera_motion_or_position": {
|
|
498
|
+
"guards": [
|
|
499
|
+
"Must already be in a video editor view.",
|
|
500
|
+
"This action consumes AI credits."
|
|
501
|
+
],
|
|
502
|
+
"why": "Camera is selector-addressable in the current UI because the motion and position presets render as real buttons.",
|
|
503
|
+
"verification_text_patterns": [
|
|
504
|
+
"Camera motion",
|
|
505
|
+
"Camera position",
|
|
506
|
+
"Dolly in"
|
|
507
|
+
],
|
|
508
|
+
"steps": [
|
|
509
|
+
"Click Camera in the bottom action bar.",
|
|
510
|
+
"Choose Camera motion or Camera position.",
|
|
511
|
+
"Select a preset such as Dolly in, Orbit right, Center, High, or Closer.",
|
|
512
|
+
"Click Create to generate the edited clip."
|
|
513
|
+
]
|
|
514
|
+
},
|
|
515
|
+
"review_or_restore_edit_history": {
|
|
516
|
+
"guards": [
|
|
517
|
+
"Must already be in an image or video editor view."
|
|
518
|
+
],
|
|
519
|
+
"why": "History is the main rollback path because Flow keeps the original media and prior prompt-driven variants.",
|
|
520
|
+
"verification_text_patterns": [
|
|
521
|
+
"Hide history",
|
|
522
|
+
"Done"
|
|
523
|
+
],
|
|
524
|
+
"steps": [
|
|
525
|
+
"Use the History panel to inspect previous versions and prompts.",
|
|
526
|
+
"Hover the version you want to keep or reuse.",
|
|
527
|
+
"Save that version back into the project when you want to reuse it in the main clip grid or Scenebuilder.",
|
|
528
|
+
"Use Download if you want the current version locally instead."
|
|
529
|
+
]
|
|
530
|
+
},
|
|
531
|
+
"edit_clip_and_build_scene": {
|
|
532
|
+
"guards": [
|
|
533
|
+
"Must already have generated or uploaded clips in the project.",
|
|
534
|
+
"Insert, Remove, and Camera edits require visual box or point placement on the frame."
|
|
535
|
+
],
|
|
536
|
+
"why": "Clip editing and Scenebuilder are Flow's main post-generation workflows, but some steps are still inherently coordinate-driven.",
|
|
537
|
+
"verification_text_patterns": [
|
|
538
|
+
"Scenebuilder",
|
|
539
|
+
"Arrange",
|
|
540
|
+
"Done"
|
|
541
|
+
],
|
|
542
|
+
"steps": [
|
|
543
|
+
"Select a clip in the project grid to expose the bottom edit actions.",
|
|
544
|
+
"Use Extend to continue an existing clip from a prompt-only edit.",
|
|
545
|
+
"Use Insert, Remove, or Camera when you can also provide the required on-frame box or position via pointer automation.",
|
|
546
|
+
"Hover a clip, open More, and choose Add to Scene to send it to Scenebuilder.",
|
|
547
|
+
"Open Scenebuilder, click Arrange, drag clips into order, and click Done when the sequence is correct."
|
|
548
|
+
]
|
|
549
|
+
},
|
|
550
|
+
"download_video_or_gif": {
|
|
551
|
+
"guards": [
|
|
552
|
+
"Must already have a generated asset tile in the project."
|
|
553
|
+
],
|
|
554
|
+
"why": "Downloads are exposed from the clip-level More menu instead of a dedicated toolbar.",
|
|
555
|
+
"verification_text_patterns": [
|
|
556
|
+
"More",
|
|
557
|
+
"Download"
|
|
558
|
+
],
|
|
559
|
+
"steps": [
|
|
560
|
+
"Hover the target asset tile until its action buttons appear.",
|
|
561
|
+
"Open the More menu for that asset.",
|
|
562
|
+
"Choose Download and then the desired file type.",
|
|
563
|
+
"Use 270p when you want a GIF export, or choose a video file option for normal downloads."
|
|
564
|
+
]
|
|
565
|
+
}
|
|
566
|
+
},
|
|
567
|
+
"errors": [
|
|
568
|
+
{
|
|
569
|
+
"error": "Flow access is unavailable even though the site loads",
|
|
570
|
+
"context": "Account, region, age verification, or subscription eligibility mismatch",
|
|
571
|
+
"solution": "As of March 11, 2026, official help says Flow requires age verification, a supported region, and Google AI Pro, Google AI Ultra, or a qualifying Workspace plan.",
|
|
572
|
+
"severity": "high"
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
"error": "A feature silently switches models or becomes unavailable",
|
|
576
|
+
"context": "Model-specific feature mismatch",
|
|
577
|
+
"solution": "Check the active model and credit cost before generation. Official help says unsupported Veo 3.1 features may automatically fall back to Veo 2.",
|
|
578
|
+
"severity": "medium"
|
|
579
|
+
},
|
|
580
|
+
{
|
|
581
|
+
"error": "Prompt text appears in the DOM but Flow still treats the prompt as empty",
|
|
582
|
+
"context": "The main prompt editor is Slate-backed and the DOM can diverge from the real textEditor state",
|
|
583
|
+
"solution": "Do not trust the visible contenteditable DOM alone. Resolve promptBoxStore from the prompt box React fiber, then use promptBoxStore.getState().inputs.textEditor with Slate methods such as select, deleteFragment, insertText, and onChange.",
|
|
584
|
+
"severity": "medium"
|
|
585
|
+
},
|
|
586
|
+
{
|
|
587
|
+
"error": "Clicking Create does nothing after prompt injection",
|
|
588
|
+
"context": "Prompt DOM was mutated, but Flow's real submit state was not updated",
|
|
589
|
+
"solution": "Populate the internal Slate textEditor first, then call the submit component's internal onSubmit() handler. Plain DOM .click() and browser_human_click are not reliable when the prompt state only changed in the DOM.",
|
|
590
|
+
"severity": "medium"
|
|
591
|
+
},
|
|
592
|
+
{
|
|
593
|
+
"error": "browser_type or browser_fill_form crashes the Flow editor or causes a client-side exception",
|
|
594
|
+
"context": "Flow's Slate prompt surface is sensitive to generic CDP typing helpers",
|
|
595
|
+
"solution": "Reload the project URL and avoid browser_type/browser_fill_form on the prompt box. Use browser_js against the internal Slate textEditor instead.",
|
|
596
|
+
"severity": "medium"
|
|
597
|
+
},
|
|
598
|
+
{
|
|
599
|
+
"error": "Crop, Select, Draw, Insert, or Remove cannot be completed selector-only",
|
|
600
|
+
"context": "These image and video edit modes depend on crop handles, drawn regions, or on-frame selections",
|
|
601
|
+
"solution": "Stage the UI with browser_js or button clicks, then combine with drag, screenshot-guided coordinates, or a human-in-the-loop step when exact selection geometry matters.",
|
|
602
|
+
"severity": "medium"
|
|
603
|
+
},
|
|
604
|
+
{
|
|
605
|
+
"error": "Scenebuilder sequence disappears after leaving the project",
|
|
606
|
+
"context": "Scene assembly workflow",
|
|
607
|
+
"solution": "Official help notes that the Scenebuilder timeline currently resets when you leave the project, even though individual clips remain saved in clip view.",
|
|
608
|
+
"severity": "medium"
|
|
609
|
+
},
|
|
610
|
+
{
|
|
611
|
+
"error": "Insert, Remove, or Camera edits are unavailable on an extended clip",
|
|
612
|
+
"context": "Post-extension clip restrictions",
|
|
613
|
+
"solution": "Official help says extended video clips cannot be edited again with Insert, Remove, or Camera. Apply those edits before extending, or start from the original clip version in History.",
|
|
614
|
+
"severity": "medium"
|
|
615
|
+
},
|
|
616
|
+
{
|
|
617
|
+
"error": "Audio generation fails on Veo 3.1",
|
|
618
|
+
"context": "Experimental audio feature",
|
|
619
|
+
"solution": "Official help describes audio generation as experimental. Retry with a different prompt or without audio guidance; failed audio generations should refund credits.",
|
|
620
|
+
"severity": "low"
|
|
621
|
+
}
|
|
622
|
+
],
|
|
623
|
+
"policyNotes": {
|
|
624
|
+
"access_requirements": [
|
|
625
|
+
"Flow requires an eligible signed-in Google account.",
|
|
626
|
+
"As of March 11, 2026, official help lists age verification, a supported region, and Google AI Pro, Google AI Ultra, or a qualifying Workspace plan as prerequisites."
|
|
627
|
+
],
|
|
628
|
+
"feature_constraints": [
|
|
629
|
+
"Ingredients to Video support is model-dependent. Official help says Veo 3.1 Fast supports it, while some other Veo modes do not.",
|
|
630
|
+
"Image editing uses the Nano Banana model family and exposes prompt, crop, select, and draw-guided workflows.",
|
|
631
|
+
"Extended clips cannot be edited again with Insert, Remove, or Camera modes.",
|
|
632
|
+
"Audio generation is experimental and currently only available on Veo 3.1.",
|
|
633
|
+
"Scenebuilder timelines reset when you leave the project."
|
|
634
|
+
],
|
|
635
|
+
"safety": [
|
|
636
|
+
"Do not automate login, billing, or subscription purchase flows.",
|
|
637
|
+
"Check the active model and current AI credit cost before clicking Create.",
|
|
638
|
+
"Prefer one generation at a time to avoid accidental credit burn.",
|
|
639
|
+
"Review generated media manually before download, export, or sharing."
|
|
640
|
+
],
|
|
641
|
+
"tool_preferences": [
|
|
642
|
+
"browser_navigate or direct anchor clicks for dashboard and help navigation",
|
|
643
|
+
"browser_js for React fiber access, promptBoxStore settings, Slate textEditor insertion, and internal onSubmit submission",
|
|
644
|
+
"browser_click or browser_human_click for visible edit-mode buttons and camera presets",
|
|
645
|
+
"screenshot for verification when the result is primarily visual rather than DOM-rich",
|
|
646
|
+
"avoid browser_type and browser_fill_form on the Flow prompt box"
|
|
647
|
+
]
|
|
648
|
+
}
|
|
649
|
+
}
|