takomi 2.1.39 → 2.1.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +217 -422
- package/assets/.agent/skills/takomi-flow/SKILL.md +10 -1
- package/docs/features/TakomiFlow_Portable_Plugin.md +7 -3
- package/docs/takomi-flow-onboarding.md +10 -0
- package/package.json +1 -1
- package/plugins/takomi-flow/assets/capabilities.json +33 -1
- package/plugins/takomi-flow/assets/request.schema.json +24 -0
- package/plugins/takomi-flow/assets/result.schema.json +1 -0
- package/plugins/takomi-flow/assets/templates/image-request.json +4 -0
- package/plugins/takomi-flow/assets/templates/video-request.json +4 -0
- package/plugins/takomi-flow/references/flow-provider-contract.md +25 -2
- package/plugins/takomi-flow/scripts/lib/agent-plan.mjs +10 -0
- package/plugins/takomi-flow/scripts/lib/args.mjs +12 -3
- package/plugins/takomi-flow/scripts/lib/browser.mjs +42 -3
- package/plugins/takomi-flow/scripts/lib/commands.mjs +28 -11
- package/plugins/takomi-flow/scripts/lib/flow-media.mjs +39 -0
- package/plugins/takomi-flow/scripts/lib/flow-outcome.mjs +37 -12
- package/plugins/takomi-flow/scripts/lib/flow-project-session.mjs +153 -0
- package/plugins/takomi-flow/scripts/lib/flow-ui.mjs +54 -15
- package/plugins/takomi-flow/scripts/lib/generation.mjs +128 -28
- package/plugins/takomi-flow/scripts/lib/mcp-tools.mjs +15 -0
- package/plugins/takomi-flow/scripts/lib/request-validator.mjs +15 -0
- package/plugins/takomi-flow/scripts/lib/request.mjs +12 -1
- package/plugins/takomi-flow/scripts/lib/settings-plan.mjs +6 -1
- package/plugins/takomi-flow/scripts/mcp-smoke.mjs +2 -0
- package/plugins/takomi-flow/skills/takomi-flow/SKILL.md +10 -1
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { FLOW_URL } from './paths.mjs';
|
|
2
|
+
import { hasText, openFlow } from './flow-ui.mjs';
|
|
3
|
+
|
|
4
|
+
export async function prepareProjectEditor(browser, request, result) {
|
|
5
|
+
const diagnostics = { projectUrl: undefined, recovered: false };
|
|
6
|
+
const waitMs = Number(request.editorWaitMs || process.env.TAKOMI_FLOW_EDITOR_WAIT_MS || 90000);
|
|
7
|
+
|
|
8
|
+
let mayUseCurrentProject = false;
|
|
9
|
+
if (request.projectUrl) {
|
|
10
|
+
browser.page = await openProjectUrl(browser, request.projectUrl);
|
|
11
|
+
mayUseCurrentProject = true;
|
|
12
|
+
} else if (request.reuseCurrentProject) {
|
|
13
|
+
browser.page = await findProjectPage(browser) || browser.page;
|
|
14
|
+
mayUseCurrentProject = true;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
if (mayUseCurrentProject && isProjectUrl(browser.page.url())) {
|
|
18
|
+
const ready = await waitForProjectEditor(browser.page, waitMs);
|
|
19
|
+
diagnostics.projectUrl = browser.page.url();
|
|
20
|
+
return { ok: ready.ok, diagnostics, manualActions: ready.manualActions };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
await openFlow(browser.page);
|
|
24
|
+
if (request.allowNewProject) {
|
|
25
|
+
const opened = await clickNewProject(browser.page);
|
|
26
|
+
if (!opened) {
|
|
27
|
+
return { ok: false, diagnostics, manualActions: ['New project button was not found. Open a Flow project manually, then retry with --project-url.'] };
|
|
28
|
+
}
|
|
29
|
+
const ready = await waitForProjectEditor(browser.page, waitMs);
|
|
30
|
+
diagnostics.projectUrl = browser.page.url();
|
|
31
|
+
return { ok: ready.ok, diagnostics, manualActions: ready.manualActions };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
Object.assign(result, await currentFlowState(browser.page));
|
|
35
|
+
return {
|
|
36
|
+
ok: false,
|
|
37
|
+
diagnostics,
|
|
38
|
+
manualActions: ['Open an existing Flow project, pass --project-url, or set --allow-new-project to create a new Flow project.'],
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function recoverFreshChat(page, request) {
|
|
43
|
+
if (!request.freshChatOnFailure) return { recovered: false, reason: 'freshChatOnFailure is disabled.' };
|
|
44
|
+
if (!isProjectUrl(page.url())) return { recovered: false, reason: 'Current page is not a Flow project.' };
|
|
45
|
+
const before = page.url();
|
|
46
|
+
await leaveMediaEditor(page);
|
|
47
|
+
const candidates = [
|
|
48
|
+
page.getByRole('button', { name: /new chat|new session|start new chat|new generation/i }).first(),
|
|
49
|
+
page.locator('button,[role="button"]').filter({ hasText: /new chat|new session|start new chat|new generation/i }).first(),
|
|
50
|
+
page.locator('[aria-label*="New chat" i], [aria-label*="New session" i], [aria-label*="Start new" i]').first(),
|
|
51
|
+
];
|
|
52
|
+
for (const locator of candidates) {
|
|
53
|
+
if (!(await locator.count().catch(() => 0))) continue;
|
|
54
|
+
try {
|
|
55
|
+
await locator.click({ timeout: 10000 });
|
|
56
|
+
await page.waitForTimeout(1500);
|
|
57
|
+
const ready = await waitForProjectEditor(page, request.editorWaitMs || 60000);
|
|
58
|
+
return { recovered: ready.ok, reason: ready.ok ? 'fresh chat opened' : ready.manualActions.join(' ') };
|
|
59
|
+
} catch {}
|
|
60
|
+
}
|
|
61
|
+
if (await hasText(page, /oops|something went wrong|try again|chat failed/i)) {
|
|
62
|
+
await page.goto(projectBaseUrl(before), { waitUntil: 'domcontentloaded', timeout: 90000 }).catch(() => {});
|
|
63
|
+
const ready = await waitForProjectEditor(page, request.editorWaitMs || 60000);
|
|
64
|
+
return { recovered: ready.ok, reason: ready.ok ? 'project reloaded' : ready.manualActions.join(' ') };
|
|
65
|
+
}
|
|
66
|
+
return { recovered: false, reason: 'No same-project fresh chat control was found.' };
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
async function openProjectUrl(browser, projectUrl) {
|
|
70
|
+
const page = (await findProjectPage(browser, projectUrl)) || browser.page;
|
|
71
|
+
await page.goto(projectBaseUrl(projectUrl), { waitUntil: 'domcontentloaded', timeout: 90000 });
|
|
72
|
+
await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(() => {});
|
|
73
|
+
await leaveMediaEditor(page);
|
|
74
|
+
return page;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function findProjectPage(browser, exactUrl = undefined) {
|
|
78
|
+
const pages = browser.context.pages();
|
|
79
|
+
const match = pages.find(page => exactUrl ? page.url() === exactUrl : isProjectUrl(page.url()));
|
|
80
|
+
return match || null;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
async function clickNewProject(page) {
|
|
84
|
+
const candidates = [
|
|
85
|
+
page.getByRole('button', { name: /new project/i }).first(),
|
|
86
|
+
page.locator('button').filter({ hasText: /new project/i }).first(),
|
|
87
|
+
page.getByText('New project').first(),
|
|
88
|
+
];
|
|
89
|
+
for (const locator of candidates) {
|
|
90
|
+
if (!(await locator.count().catch(() => 0))) continue;
|
|
91
|
+
try {
|
|
92
|
+
await locator.click({ timeout: 10000 });
|
|
93
|
+
await page.waitForURL(/\/project\//, { timeout: 30000 }).catch(() => {});
|
|
94
|
+
await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(() => {});
|
|
95
|
+
return isProjectUrl(page.url());
|
|
96
|
+
} catch {}
|
|
97
|
+
}
|
|
98
|
+
return false;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function waitForProjectEditor(page, timeoutMs) {
|
|
102
|
+
await leaveMediaEditor(page);
|
|
103
|
+
await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {});
|
|
104
|
+
await page.waitForLoadState('networkidle', { timeout: 30000 }).catch(() => {});
|
|
105
|
+
const started = Date.now();
|
|
106
|
+
while (Date.now() - started < timeoutMs) {
|
|
107
|
+
const state = await currentFlowState(page);
|
|
108
|
+
if (state.manualActions.length) return { ok: false, manualActions: state.manualActions };
|
|
109
|
+
const hasEditorText = await hasText(page, /what do you want to create|start creating|add media/i);
|
|
110
|
+
const inputCount = await page.locator('div[role="textbox"], textarea:not(.g-recaptcha-response)').count().catch(() => 0);
|
|
111
|
+
if (hasEditorText && inputCount > 0) return { ok: true, manualActions: [] };
|
|
112
|
+
await page.waitForTimeout(1000);
|
|
113
|
+
}
|
|
114
|
+
return { ok: false, manualActions: ['Flow project editor did not become ready before the wait timeout.'] };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function leaveMediaEditor(page) {
|
|
118
|
+
if (!/\/edit\//.test(page.url())) return;
|
|
119
|
+
const candidates = [
|
|
120
|
+
page.getByRole('button', { name: /^Done$/i }).last(),
|
|
121
|
+
page.locator('button').filter({ hasText: /^Done$/i }).last(),
|
|
122
|
+
page.getByRole('button', { name: /back/i }).first(),
|
|
123
|
+
page.locator('button').filter({ hasText: /arrow_back|back/i }).first(),
|
|
124
|
+
];
|
|
125
|
+
for (const locator of candidates) {
|
|
126
|
+
if (!(await locator.count().catch(() => 0))) continue;
|
|
127
|
+
try {
|
|
128
|
+
await locator.click({ timeout: 10000, force: true });
|
|
129
|
+
await page.waitForURL(url => !/\/edit\//.test(url.toString()), { timeout: 15000 }).catch(() => {});
|
|
130
|
+
if (!/\/edit\//.test(page.url())) return;
|
|
131
|
+
} catch {}
|
|
132
|
+
}
|
|
133
|
+
await page.goto(projectBaseUrl(page.url()), { waitUntil: 'domcontentloaded', timeout: 90000 }).catch(() => {});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
async function currentFlowState(page) {
|
|
137
|
+
const text = (await page.locator('body').innerText({ timeout: 5000 }).catch(() => '')).toLowerCase();
|
|
138
|
+
const manualActions = [];
|
|
139
|
+
if (text.includes('sign in') || text.includes('log in')) manualActions.push('Sign into Google/Flow in the opened browser.');
|
|
140
|
+
if (text.includes('captcha') || text.includes('verify')) manualActions.push('Complete the verification challenge in the browser.');
|
|
141
|
+
if (/quota|usage limit|not enough credits|insufficient credits|out of credits/.test(text)) {
|
|
142
|
+
manualActions.push('Review Flow quota or credits message in the browser.');
|
|
143
|
+
}
|
|
144
|
+
return { manualActions };
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function isProjectUrl(url) {
|
|
148
|
+
return /\/project\//.test(url || '');
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function projectBaseUrl(url) {
|
|
152
|
+
return String(url || '').replace(/\/edit\/[^/?#]+.*/, '');
|
|
153
|
+
}
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import path from 'node:path';
|
|
2
2
|
import fs from 'node:fs';
|
|
3
3
|
import { FLOW_URL } from './paths.mjs';
|
|
4
|
+
import { openGeneratedMediaById } from './flow-media.mjs';
|
|
4
5
|
|
|
5
6
|
export async function openFlow(page) {
|
|
6
7
|
await page.goto(FLOW_URL, { waitUntil: 'domcontentloaded', timeout: 90000 });
|
|
7
|
-
await page.waitForLoadState('networkidle', { timeout:
|
|
8
|
+
await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
export async function capture(page, run, name) {
|
|
@@ -43,7 +44,7 @@ export async function ensureProjectEditor(page) {
|
|
|
43
44
|
try {
|
|
44
45
|
await locator.click({ timeout: 10000 });
|
|
45
46
|
await page.waitForURL(/\/project\//, { timeout: 30000 }).catch(() => {});
|
|
46
|
-
await page.waitForLoadState('networkidle', { timeout:
|
|
47
|
+
await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
|
|
47
48
|
return page.url().includes('/project/') && await waitForProjectEditor(page);
|
|
48
49
|
} catch {}
|
|
49
50
|
}
|
|
@@ -51,7 +52,7 @@ export async function ensureProjectEditor(page) {
|
|
|
51
52
|
}
|
|
52
53
|
|
|
53
54
|
async function waitForProjectEditor(page) {
|
|
54
|
-
await page.waitForLoadState('networkidle', { timeout:
|
|
55
|
+
await page.waitForLoadState('networkidle', { timeout: 8000 }).catch(() => {});
|
|
55
56
|
const ready = page.locator('body').filter({ hasText: /what do you want to create|start creating|add media/i });
|
|
56
57
|
await ready.waitFor({ state: 'visible', timeout: 60000 }).catch(() => {});
|
|
57
58
|
return (await page.locator('div[role="textbox"], textarea:not(.g-recaptcha-response)').count().catch(() => 0)) > 0;
|
|
@@ -59,23 +60,43 @@ async function waitForProjectEditor(page) {
|
|
|
59
60
|
|
|
60
61
|
export async function fillPrompt(page, prompt) {
|
|
61
62
|
const candidates = [
|
|
62
|
-
page.locator('
|
|
63
|
-
page.locator('
|
|
64
|
-
page.locator('
|
|
63
|
+
page.locator('textarea:not(.g-recaptcha-response):not([name*="recaptcha"]):visible').last(),
|
|
64
|
+
page.locator('[contenteditable="true"][role="textbox"]:visible').last(),
|
|
65
|
+
page.locator('div[role="textbox"]:visible').last(),
|
|
65
66
|
];
|
|
66
67
|
for (const locator of candidates) {
|
|
67
68
|
if (!(await locator.count().catch(() => 0))) continue;
|
|
68
69
|
try {
|
|
69
70
|
await locator.click({ timeout: 8000 });
|
|
70
|
-
await
|
|
71
|
-
await
|
|
72
|
-
const matched = await page.locator('body').filter({ hasText: prompt.slice(0, 40) }).count().catch(() => 0);
|
|
73
|
-
if (matched) return true;
|
|
71
|
+
if (!(await fillEditable(locator, page, prompt))) continue;
|
|
72
|
+
if (await editableContains(locator, prompt)) return true;
|
|
74
73
|
} catch {}
|
|
75
74
|
}
|
|
76
75
|
return false;
|
|
77
76
|
}
|
|
78
77
|
|
|
78
|
+
async function fillEditable(locator, page, prompt) {
|
|
79
|
+
if (await locator.fill(prompt, { timeout: 8000 }).then(() => true).catch(() => false)) return true;
|
|
80
|
+
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+A' : 'Control+A').catch(() => {});
|
|
81
|
+
await page.keyboard.insertText(prompt).catch(async () => {
|
|
82
|
+
await page.keyboard.type(prompt, { delay: 5 });
|
|
83
|
+
});
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function editableContains(locator, prompt) {
|
|
88
|
+
const expected = compactText(prompt).slice(0, 80);
|
|
89
|
+
const actual = await locator.evaluate(element => {
|
|
90
|
+
const value = element.value || element.innerText || element.textContent || '';
|
|
91
|
+
return value.replace(/\s+/g, ' ').trim();
|
|
92
|
+
}).catch(() => '');
|
|
93
|
+
return compactText(actual).includes(expected);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function compactText(value) {
|
|
97
|
+
return String(value || '').replace(/\s+/g, ' ').trim();
|
|
98
|
+
}
|
|
99
|
+
|
|
79
100
|
export async function submitGeneration(page) {
|
|
80
101
|
const preferred = [
|
|
81
102
|
page.locator('button').filter({ hasText: /arrow_forward\s*Create/i }).last(),
|
|
@@ -85,6 +106,7 @@ export async function submitGeneration(page) {
|
|
|
85
106
|
for (const button of preferred) {
|
|
86
107
|
if (!(await button.count().catch(() => 0))) continue;
|
|
87
108
|
try {
|
|
109
|
+
if (await button.isDisabled().catch(() => false)) continue;
|
|
88
110
|
await button.click({ timeout: 8000 });
|
|
89
111
|
return true;
|
|
90
112
|
} catch {}
|
|
@@ -97,6 +119,7 @@ export async function submitGeneration(page) {
|
|
|
97
119
|
for (const button of buttons) {
|
|
98
120
|
if (!(await button.count().catch(() => 0))) continue;
|
|
99
121
|
try {
|
|
122
|
+
if (await button.isDisabled().catch(() => false)) continue;
|
|
100
123
|
await button.click({ timeout: 8000 });
|
|
101
124
|
return true;
|
|
102
125
|
} catch {}
|
|
@@ -104,17 +127,19 @@ export async function submitGeneration(page) {
|
|
|
104
127
|
return false;
|
|
105
128
|
}
|
|
106
129
|
|
|
107
|
-
export async function tryDownloadAssets(page, run, maxDownloads = 1) {
|
|
130
|
+
export async function tryDownloadAssets(page, run, maxDownloads = 1, options = {}) {
|
|
108
131
|
const assets = [];
|
|
109
|
-
|
|
132
|
+
const allowRenderedImageFallback = options.allowRenderedImageFallback !== false;
|
|
133
|
+
await openGeneratedMedia(page, options.mediaIds || []);
|
|
110
134
|
for (let i = 0; i < maxDownloads; i += 1) {
|
|
111
|
-
const button = page
|
|
135
|
+
const button = downloadButton(page);
|
|
112
136
|
if (!(await button.count().catch(() => 0))) break;
|
|
113
137
|
const [completed] = await Promise.all([
|
|
114
138
|
page.waitForEvent('download', { timeout: 30000 }).catch(() => null),
|
|
115
139
|
button.click({ timeout: 5000 }).catch(() => null),
|
|
116
140
|
]);
|
|
117
141
|
if (!completed) {
|
|
142
|
+
if (!allowRenderedImageFallback) break;
|
|
118
143
|
const imagePath = await downloadLargestRenderedImage(page, run, i).catch(() => null);
|
|
119
144
|
if (!imagePath) break;
|
|
120
145
|
assets.push(imagePath);
|
|
@@ -141,8 +166,11 @@ async function downloadLargestRenderedImage(page, run, index) {
|
|
|
141
166
|
return filePath;
|
|
142
167
|
}
|
|
143
168
|
|
|
144
|
-
export async function openGeneratedMedia(page) {
|
|
145
|
-
if (await page
|
|
169
|
+
export async function openGeneratedMedia(page, mediaIds = []) {
|
|
170
|
+
if (await downloadButton(page).count().catch(() => 0)) return true;
|
|
171
|
+
for (const mediaId of mediaIds) {
|
|
172
|
+
if (await openGeneratedMediaById(page, mediaId)) return true;
|
|
173
|
+
}
|
|
146
174
|
const candidates = [
|
|
147
175
|
page.getByAltText(/video thumbnail|generated image/i).first(),
|
|
148
176
|
page.locator('img[src*="media.getMediaUrlRedirect"]').last(),
|
|
@@ -163,3 +191,14 @@ export async function openGeneratedMedia(page) {
|
|
|
163
191
|
export async function hasText(page, pattern) {
|
|
164
192
|
return (await page.locator('body').filter({ hasText: pattern }).count().catch(() => 0)) > 0;
|
|
165
193
|
}
|
|
194
|
+
|
|
195
|
+
export function downloadButton(page) {
|
|
196
|
+
return page.locator([
|
|
197
|
+
'button[aria-label*="Download" i]',
|
|
198
|
+
'[role="button"][aria-label*="Download" i]',
|
|
199
|
+
'button[title*="Download" i]',
|
|
200
|
+
'[role="button"][title*="Download" i]',
|
|
201
|
+
'button:has-text("download")',
|
|
202
|
+
'[role="button"]:has-text("download")',
|
|
203
|
+
].join(', ')).first();
|
|
204
|
+
}
|
|
@@ -3,14 +3,16 @@ import { defaultProfileDir, FLOW_URL, readJson, resolvePath } from './paths.mjs'
|
|
|
3
3
|
import { normalizeRequest, canSpend } from './request.mjs';
|
|
4
4
|
import { closeFlowBrowser, launchFlowBrowser } from './browser.mjs';
|
|
5
5
|
import { createRun, baseResult, saveResult } from './result.mjs';
|
|
6
|
-
import { capture,
|
|
6
|
+
import { capture, fillPrompt, hasText, submitGeneration, tryDownloadAssets } from './flow-ui.mjs';
|
|
7
7
|
import { handleGenerationFollowups, waitForGenerationOutcome } from './flow-outcome.mjs';
|
|
8
8
|
import { catalogAssets } from './media.mjs';
|
|
9
9
|
import { createSettingsPlan } from './settings-plan.mjs';
|
|
10
|
+
import { prepareProjectEditor, recoverFreshChat } from './flow-project-session.mjs';
|
|
11
|
+
import { listMediaIds } from './flow-media.mjs';
|
|
10
12
|
|
|
11
13
|
export async function generateFromRequest(args = {}) {
|
|
12
14
|
if (!args.request) throw new Error('generate requires a request path.');
|
|
13
|
-
const request = normalizeRequest(readJson(path.resolve(args.request)));
|
|
15
|
+
const request = normalizeRequest({ ...readJson(path.resolve(args.request)), ...requestOverrides(args) });
|
|
14
16
|
const run = createRun(request, 'generate');
|
|
15
17
|
const result = baseResult('blocked', run, {
|
|
16
18
|
command: 'generate',
|
|
@@ -32,41 +34,42 @@ export async function generateFromRequest(args = {}) {
|
|
|
32
34
|
headless: Boolean(args.headless),
|
|
33
35
|
});
|
|
34
36
|
try {
|
|
35
|
-
await
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if (
|
|
39
|
-
requireManual(result,
|
|
37
|
+
result.screenshots.push(await capture(browser.page, run, 'before-project-resolve'));
|
|
38
|
+
const project = await prepareProjectEditor(browser, request, result);
|
|
39
|
+
result.projectSession = project.diagnostics;
|
|
40
|
+
if (!project.ok) {
|
|
41
|
+
requireManual(result, project.manualActions);
|
|
40
42
|
return result;
|
|
41
43
|
}
|
|
42
|
-
|
|
43
|
-
requireManual(result, ['New project button or project editor was not found. Open a Flow project manually, then retry with --cdp-url.']);
|
|
44
|
-
return result;
|
|
45
|
-
}
|
|
46
|
-
result.projectUrl = browser.page.url();
|
|
44
|
+
result.projectUrl = project.diagnostics.projectUrl || browser.page.url();
|
|
47
45
|
result.screenshots.push(await capture(browser.page, run, 'project-editor'));
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
46
|
+
const maxAttempts = request.freshChatOnFailure === false ? 1 : 2;
|
|
47
|
+
result.generationAttempts = [];
|
|
48
|
+
if (request.freshChatOnFailure !== false && await hasText(browser.page, /failed|oops, something went wrong/i)) {
|
|
49
|
+
await recoverFreshChat(browser.page, request);
|
|
51
50
|
}
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
51
|
+
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
52
|
+
const attemptResult = await runGenerationAttempt(browser.page, request, run, attempt);
|
|
53
|
+
result.generationAttempts.push({ attempt, outcome: attemptResult.outcome, recovered: attemptResult.recovered });
|
|
54
|
+
result.projectUrl = browser.page.url();
|
|
55
|
+
result.generationOutcome = attemptResult.outcome;
|
|
56
|
+
result.screenshots.push(await capture(browser.page, run, attempt === 1 ? 'after-wait' : `after-wait-${attempt}`));
|
|
57
|
+
result.assets = attemptResult.assets;
|
|
58
|
+
if (result.assets.length) break;
|
|
59
|
+
if (attempt < maxAttempts && shouldRetryOutcome(attemptResult.outcome)) {
|
|
60
|
+
const recovery = await recoverFreshChat(browser.page, request);
|
|
61
|
+
if (recovery.recovered) {
|
|
62
|
+
result.generationAttempts[result.generationAttempts.length - 1].recovered = true;
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
break;
|
|
56
67
|
}
|
|
57
|
-
result.projectUrl = browser.page.url();
|
|
58
|
-
await handleGenerationFollowups(browser.page, request);
|
|
59
|
-
result.generationOutcome = await waitForGenerationOutcome(browser.page, request);
|
|
60
|
-
result.projectUrl = browser.page.url();
|
|
61
|
-
result.screenshots.push(await capture(browser.page, run, 'after-wait'));
|
|
62
|
-
result.assets = await tryDownloadAssets(browser.page, run, request.variations);
|
|
63
68
|
if (result.assets.length) {
|
|
64
69
|
result.assetCatalogPath = (await catalogAssets(run, result.assets, { frames: request.extractFrames || 0 })).catalogPath;
|
|
65
70
|
}
|
|
66
71
|
result.status = result.assets.length ? 'downloaded' : 'manual_action_required';
|
|
67
|
-
if (!result.assets.length)
|
|
68
|
-
result.manualActions.push(nextActionForOutcome(result.generationOutcome));
|
|
69
|
-
}
|
|
72
|
+
if (!result.assets.length) result.manualActions.push(nextActionForOutcome(result.generationOutcome));
|
|
70
73
|
return result;
|
|
71
74
|
} catch (error) {
|
|
72
75
|
result.status = 'failed';
|
|
@@ -78,6 +81,100 @@ export async function generateFromRequest(args = {}) {
|
|
|
78
81
|
}
|
|
79
82
|
}
|
|
80
83
|
|
|
84
|
+
function requestOverrides(args) {
|
|
85
|
+
return Object.fromEntries([
|
|
86
|
+
['projectUrl', args.projectUrl],
|
|
87
|
+
['reuseCurrentProject', args.reuseCurrentProject],
|
|
88
|
+
['allowNewProject', args.allowNewProject],
|
|
89
|
+
['freshChatOnFailure', args.freshChatOnFailure],
|
|
90
|
+
['editorWaitMs', args.editorWaitMs],
|
|
91
|
+
].filter(([, value]) => value !== undefined));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function fillPromptWithRecovery(page, request) {
|
|
95
|
+
if (await fillPrompt(page, request.prompt)) return true;
|
|
96
|
+
const recovery = await recoverFreshChat(page, request);
|
|
97
|
+
return recovery.recovered && await fillPrompt(page, request.prompt);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
async function submitWithRecovery(page, request) {
|
|
101
|
+
if (await submitGeneration(page)) return true;
|
|
102
|
+
const recovery = await recoverFreshChat(page, request);
|
|
103
|
+
return recovery.recovered
|
|
104
|
+
&& await fillPrompt(page, request.prompt)
|
|
105
|
+
&& await submitGeneration(page);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function runGenerationAttempt(page, request, run, attempt) {
|
|
109
|
+
const baselineMediaIds = await listMediaIds(page);
|
|
110
|
+
const pending = await pendingGenerationState(page, request);
|
|
111
|
+
if (pending.matchesRequest) {
|
|
112
|
+
await handleGenerationFollowups(page, request);
|
|
113
|
+
const outcome = await waitForGenerationOutcome(page, request, { baselineMediaIds });
|
|
114
|
+
const assets = await downloadAttemptAssets(page, run, request, outcome);
|
|
115
|
+
return { outcome, assets, recovered: false };
|
|
116
|
+
}
|
|
117
|
+
if (pending.exists) {
|
|
118
|
+
return {
|
|
119
|
+
outcome: { status: 'active_generation_pending' },
|
|
120
|
+
assets: [],
|
|
121
|
+
recovered: false,
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
if (!(await fillPromptWithRecovery(page, request))) {
|
|
125
|
+
return {
|
|
126
|
+
outcome: { status: 'prompt_not_found' },
|
|
127
|
+
assets: [],
|
|
128
|
+
recovered: false,
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
await capture(page, run, attempt === 1 ? 'prompt-filled' : `prompt-filled-${attempt}`);
|
|
132
|
+
if (!(await submitWithRecovery(page, request))) {
|
|
133
|
+
return {
|
|
134
|
+
outcome: { status: 'submit_not_found' },
|
|
135
|
+
assets: [],
|
|
136
|
+
recovered: false,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
await handleGenerationFollowups(page, request);
|
|
140
|
+
const outcome = await waitForGenerationOutcome(page, request, { baselineMediaIds });
|
|
141
|
+
const assets = await downloadAttemptAssets(page, run, request, outcome);
|
|
142
|
+
return { outcome, assets, recovered: false };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async function downloadAttemptAssets(page, run, request, outcome) {
|
|
146
|
+
const assets = await tryDownloadAssets(page, run, request.variations, {
|
|
147
|
+
mediaIds: outcome.mediaIds || [],
|
|
148
|
+
allowRenderedImageFallback: request.kind !== 'video',
|
|
149
|
+
});
|
|
150
|
+
if (assets.length && !['download_ready', 'media_ready'].includes(outcome.status)) {
|
|
151
|
+
outcome.status = 'download_ready';
|
|
152
|
+
outcome.recoveredFromStaleText = true;
|
|
153
|
+
}
|
|
154
|
+
return assets;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
function shouldRetryOutcome(outcome = {}) {
|
|
158
|
+
return ['failed', 'prompt_not_found', 'submit_not_found'].includes(outcome.status);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
async function pendingGenerationState(page, request) {
|
|
162
|
+
const pendingPattern = /would you like me to kick off|costing\s+\d+\s+credits|currently in the queue|waiting in the queue|been scheduled|i['’]ve scheduled|ready for you shortly/i;
|
|
163
|
+
const text = await page.locator('body').innerText({ timeout: 5000 }).catch(() => '');
|
|
164
|
+
const exists = pendingPattern.test(text);
|
|
165
|
+
if (!exists) return { exists: false, matchesRequest: false };
|
|
166
|
+
const pageText = compactText(text).toLowerCase();
|
|
167
|
+
const promptText = compactText(request.prompt).toLowerCase();
|
|
168
|
+
const corePrompt = compactText(request.prompt.replace(/^create exactly one.*?not still images\.\s*/i, '')).toLowerCase();
|
|
169
|
+
const matchesRequest = pageText.includes(promptText.slice(0, 80))
|
|
170
|
+
|| (corePrompt.length > 20 && pageText.includes(corePrompt.slice(0, 80)));
|
|
171
|
+
return { exists, matchesRequest };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function compactText(value) {
|
|
175
|
+
return String(value || '').replace(/\s+/g, ' ').trim();
|
|
176
|
+
}
|
|
177
|
+
|
|
81
178
|
function requireManual(result, actions) {
|
|
82
179
|
result.status = 'manual_action_required';
|
|
83
180
|
result.manualActions.push(...actions);
|
|
@@ -85,6 +182,9 @@ function requireManual(result, actions) {
|
|
|
85
182
|
|
|
86
183
|
function nextActionForOutcome(outcome = {}) {
|
|
87
184
|
if (outcome.status === 'failed') return 'Flow reported generation failure. Inspect the browser/run screenshots.';
|
|
185
|
+
if (outcome.status === 'active_generation_pending') return 'A different Flow generation is already pending. Wait for it to finish before submitting another prompt.';
|
|
186
|
+
if (outcome.status === 'prompt_not_found') return 'Prompt box was not found. Update Flow selectors or paste the prompt manually.';
|
|
187
|
+
if (outcome.status === 'submit_not_found') return 'Generate/Create button was not found. Submit manually in the opened browser.';
|
|
88
188
|
if (outcome.status === 'timeout') return 'Generation did not finish before the wait timeout. Leave trusted Chrome open and run inspect/observe again.';
|
|
89
189
|
return 'Generation may still be running or download controls changed. Inspect the browser/run screenshots.';
|
|
90
190
|
}
|
|
@@ -28,6 +28,11 @@ export function registerTools(server) {
|
|
|
28
28
|
allowSpend: z.boolean().optional(),
|
|
29
29
|
extractFrames: z.number().int().nonnegative().optional(),
|
|
30
30
|
sourceAssets: z.array(z.string()).optional(),
|
|
31
|
+
projectUrl: z.string().optional(),
|
|
32
|
+
reuseCurrentProject: z.boolean().optional(),
|
|
33
|
+
allowNewProject: z.boolean().optional(),
|
|
34
|
+
freshChatOnFailure: z.boolean().optional(),
|
|
35
|
+
editorWaitMs: z.number().int().positive().optional(),
|
|
31
36
|
submit: z.boolean().optional(),
|
|
32
37
|
targetDir: z.string().optional(),
|
|
33
38
|
}, args => api.plan(args));
|
|
@@ -53,6 +58,11 @@ function registerRequestTools(register) {
|
|
|
53
58
|
allowSpend: z.boolean().optional(),
|
|
54
59
|
extractFrames: z.number().int().nonnegative().optional(),
|
|
55
60
|
sourceAssets: z.array(z.string()).optional(),
|
|
61
|
+
projectUrl: z.string().optional(),
|
|
62
|
+
reuseCurrentProject: z.boolean().optional(),
|
|
63
|
+
allowNewProject: z.boolean().optional(),
|
|
64
|
+
freshChatOnFailure: z.boolean().optional(),
|
|
65
|
+
editorWaitMs: z.number().int().positive().optional(),
|
|
56
66
|
notes: z.string().optional(),
|
|
57
67
|
};
|
|
58
68
|
register('takomi_flow_prepare', 'Create a prepared request JSON file for a Flow image or video generation.', requestShape, args => api.prepare(args));
|
|
@@ -86,6 +96,11 @@ function registerBrowserTools(register) {
|
|
|
86
96
|
browserChannel: z.string().optional(),
|
|
87
97
|
cdpUrl: z.string().optional(),
|
|
88
98
|
headless: z.boolean().optional(),
|
|
99
|
+
projectUrl: z.string().optional(),
|
|
100
|
+
reuseCurrentProject: z.boolean().optional(),
|
|
101
|
+
allowNewProject: z.boolean().optional(),
|
|
102
|
+
freshChatOnFailure: z.boolean().optional(),
|
|
103
|
+
editorWaitMs: z.number().int().positive().optional(),
|
|
89
104
|
}, args => requireBrowser(args, () => api.generate(args)));
|
|
90
105
|
}
|
|
91
106
|
|
|
@@ -39,6 +39,12 @@ export function validateRequest(raw, requestPath = undefined) {
|
|
|
39
39
|
if (normalized.aspectRatio && !capabilities.aspectRatios.includes(normalized.aspectRatio)) {
|
|
40
40
|
warnings.push(`Aspect ratio "${normalized.aspectRatio}" is not in listed options.`);
|
|
41
41
|
}
|
|
42
|
+
if (normalized.projectUrl && !isFlowProjectUrl(normalized.projectUrl)) {
|
|
43
|
+
errors.push(`Project URL must be a Google Flow project URL: ${normalized.projectUrl}`);
|
|
44
|
+
}
|
|
45
|
+
if (normalized.editorWaitMs !== undefined && normalized.editorWaitMs < 1000) {
|
|
46
|
+
errors.push('editorWaitMs must be at least 1000 milliseconds.');
|
|
47
|
+
}
|
|
42
48
|
for (const asset of normalized.sourceAssets) {
|
|
43
49
|
if (!fs.existsSync(path.resolve(asset))) {
|
|
44
50
|
errors.push(`Source asset does not exist: ${asset}`);
|
|
@@ -56,3 +62,12 @@ export function validateRequest(raw, requestPath = undefined) {
|
|
|
56
62
|
warnings,
|
|
57
63
|
};
|
|
58
64
|
}
|
|
65
|
+
|
|
66
|
+
function isFlowProjectUrl(value) {
|
|
67
|
+
try {
|
|
68
|
+
const url = new URL(value);
|
|
69
|
+
return url.hostname === 'labs.google' && url.pathname.includes('/fx/tools/flow/project/');
|
|
70
|
+
} catch {
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
@@ -23,9 +23,14 @@ export function normalizeRequest(input) {
|
|
|
23
23
|
mode: input.mode || (kind === 'video' ? 'text-to-video' : 'text-to-image'),
|
|
24
24
|
modelHint: input.modelHint || input.model || 'best-available',
|
|
25
25
|
outputDir: path.resolve(input.outputDir || input['output-dir'] || defaultRunsDir()),
|
|
26
|
-
allowSpend:
|
|
26
|
+
allowSpend: toOptionalBoolean(input.allowSpend ?? input['allow-spend'], false),
|
|
27
27
|
extractFrames: Math.max(0, Number.parseInt(input.extractFrames || input['extract-frames'] || '0', 10) || 0),
|
|
28
28
|
sourceAssets: normalizeList(input.sourceAssets || input.assets),
|
|
29
|
+
projectUrl: input.projectUrl || input['project-url'] || undefined,
|
|
30
|
+
reuseCurrentProject: toOptionalBoolean(input.reuseCurrentProject ?? input['reuse-current-project'], true),
|
|
31
|
+
allowNewProject: toOptionalBoolean(input.allowNewProject ?? input['allow-new-project'], false),
|
|
32
|
+
freshChatOnFailure: toOptionalBoolean(input.freshChatOnFailure ?? input['fresh-chat-on-failure'], true),
|
|
33
|
+
editorWaitMs: toOptionalNumber(input.editorWaitMs || input['editor-wait-ms']),
|
|
29
34
|
notes: input.notes || undefined,
|
|
30
35
|
};
|
|
31
36
|
}
|
|
@@ -48,6 +53,12 @@ function toOptionalNumber(value) {
|
|
|
48
53
|
return Number.isFinite(parsed) ? parsed : undefined;
|
|
49
54
|
}
|
|
50
55
|
|
|
56
|
+
function toOptionalBoolean(value, defaultValue) {
|
|
57
|
+
if (value === undefined || value === null || value === '') return defaultValue;
|
|
58
|
+
if (typeof value === 'boolean') return value;
|
|
59
|
+
return ['1', 'true', 'yes', 'on'].includes(String(value).toLowerCase());
|
|
60
|
+
}
|
|
61
|
+
|
|
51
62
|
function normalizeList(value) {
|
|
52
63
|
if (!value) return [];
|
|
53
64
|
if (Array.isArray(value)) return value;
|
|
@@ -8,8 +8,13 @@ export function createSettingsPlan(request) {
|
|
|
8
8
|
modelHint: request.modelHint,
|
|
9
9
|
extractFrames: request.extractFrames,
|
|
10
10
|
sourceAssets: request.sourceAssets || [],
|
|
11
|
+
projectUrl: request.projectUrl,
|
|
12
|
+
reuseCurrentProject: request.reuseCurrentProject,
|
|
13
|
+
allowNewProject: request.allowNewProject,
|
|
14
|
+
freshChatOnFailure: request.freshChatOnFailure,
|
|
15
|
+
editorWaitMs: request.editorWaitMs,
|
|
11
16
|
};
|
|
12
|
-
const automatic = ['prompt', 'download folder', 'metadata', 'asset catalog'];
|
|
17
|
+
const automatic = ['prompt', 'project reuse', 'same-project chat recovery', 'download folder', 'metadata', 'asset catalog'];
|
|
13
18
|
if (request.extractFrames > 0) automatic.push('video review frame extraction after download');
|
|
14
19
|
return {
|
|
15
20
|
schemaVersion: 1,
|
|
@@ -6,10 +6,12 @@ import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
|
|
6
6
|
import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
|
|
7
7
|
|
|
8
8
|
const client = new Client({ name: 'takomi-flow-smoke', version: '0.1.0' });
|
|
9
|
+
const pluginRoot = path.resolve(import.meta.dirname, '..');
|
|
9
10
|
const reviewRunDir = path.join(os.tmpdir(), 'takomi-flow-mcp-smoke-review');
|
|
10
11
|
const transport = new StdioClientTransport({
|
|
11
12
|
command: 'node',
|
|
12
13
|
args: ['scripts/mcp-server.mjs'],
|
|
14
|
+
cwd: pluginRoot,
|
|
13
15
|
});
|
|
14
16
|
|
|
15
17
|
try {
|
|
@@ -69,7 +69,6 @@ If Git is unavailable, ask the user to download the VibeCode Protocol Suite ZIP
|
|
|
69
69
|
## Agent Tool Surface
|
|
70
70
|
|
|
71
71
|
Prefer MCP tools when they are available in the active Codex session. Use the CLI commands as the stable fallback.
|
|
72
|
-
|
|
73
72
|
MCP tools:
|
|
74
73
|
|
|
75
74
|
- `takomi_flow_capabilities`
|
|
@@ -116,7 +115,13 @@ Browser-opening MCP tools require `allowBrowser=true`. Generation still requires
|
|
|
116
115
|
- Use public Flow UI automation only.
|
|
117
116
|
- Do not bypass captchas, login challenges, quotas, safety checks, rate limits, or hidden endpoints.
|
|
118
117
|
- Prefer headed mode first so the user can handle Google login, consent, quota, and safety prompts.
|
|
118
|
+
- Browser commands should start from trusted Chrome/CDP by default; use Playwright-launched browsers only as an explicit fallback.
|
|
119
119
|
- Never submit a paid generation unless the user explicitly requested it and `allowSpend` or `TAKOMI_FLOW_ALLOW_SPEND=true` is set.
|
|
120
|
+
- Reuse the current Flow project or pass `projectUrl`; create new projects only when the user asks or `allowNewProject` is set.
|
|
121
|
+
- Treat Flow as one active generation at a time: wait for the current approved generation to finish and download it before submitting another paid generation.
|
|
122
|
+
- A gray preview placeholder with scheduled, queue, or ready-shortly copy means the generation is in progress, not failed.
|
|
123
|
+
- If stale scheduled or failure text remains after media is ready, probe/open the generated media and use the top toolbar Download control instead of waiting for the full timeout.
|
|
124
|
+
- Download generated media by opening the media and clicking the top toolbar `Download` icon/button.
|
|
120
125
|
- Keep credentials out of prompts, logs, metadata, and project files.
|
|
121
126
|
- Store run artifacts in a predictable folder and report the exact result paths.
|
|
122
127
|
|
|
@@ -138,6 +143,7 @@ node scripts/takomi-flow.mjs smoke
|
|
|
138
143
|
node scripts/takomi-flow.mjs template --kind video
|
|
139
144
|
node scripts/takomi-flow.mjs prepare --kind video --prompt "cinematic AI lab scene" --variations 2
|
|
140
145
|
node scripts/takomi-flow.mjs workflow --kind video --prompt "cinematic AI lab scene" --variations 2
|
|
146
|
+
node scripts/takomi-flow.mjs workflow --kind video --prompt "cinematic AI lab scene" --project-url "<Flow project URL>" --reuse-current-project
|
|
141
147
|
node scripts/takomi-flow.mjs validate --request output/takomi-flow/requests/<file>.json
|
|
142
148
|
node scripts/takomi-flow.mjs generate --request output/takomi-flow/requests/<file>.json
|
|
143
149
|
node scripts/takomi-flow.mjs inspect --run output/takomi-flow/<runId>/run.json
|
|
@@ -173,6 +179,9 @@ Important defaults:
|
|
|
173
179
|
- Keep that Chrome window open and use `--cdp-url http://127.0.0.1:9222` for observe/generate.
|
|
174
180
|
- A signed-in dashboard should show project cards and a `New project` button during `observe`.
|
|
175
181
|
- The project editor prompt textbox currently contains `What do you want to create?`.
|
|
182
|
+
- Leave Chrome on the desired project editor or pass `--project-url`; TakomiFlow does not click `New project` unless `--allow-new-project` is set.
|
|
183
|
+
- Rename a project by editing the top-left header title/date-time input.
|
|
184
|
+
- If the chat breaks, keep the same project and let `freshChatOnFailure` try one fresh chat before manual recovery.
|
|
176
185
|
- Prefer video durations `4`, `6`, `8`, or `10` seconds to avoid a Flow follow-up question.
|
|
177
186
|
- Use `bootstrap` only when Google accepts the launched browser.
|
|
178
187
|
- Ask the user to log into Google Flow manually in the opened browser.
|