utilitas 1999.1.79 → 1999.1.81

package/lib/gen.mjs

@@ -1,42 +1,39 @@
 import {
-    ensureArray, ensureString, ignoreErrFunc, log as _log, need, throwError,
-    tryUntil,
+    ensureArray, ensureString, log as _log, need, throwError,
+    tryUntil, timeout,
 } from './utilitas.mjs';
 
-import { assertExist, exec } from './shell.mjs';
-import { convert, MIME_PNG } from './storage.mjs';
+import { convert, MIME_PNG, MIME_MP4, getTempPath } from './storage.mjs';
 import { createReadStream } from 'fs';
 
-const _NEED = ['OpenAI'];
+const _NEED = ['OpenAI', '@google/genai'];
 const log = (cnt, opt) => _log(cnt, import.meta.url, { time: 1, ...opt || {} });
 const [
-    clients, OPENAI, GEMINI, BASE64, BUFFER, ERROR_GENERATING, IMAGEN_MODEL,
-    OPENAI_MODEL, VEO_MODEL,
+    clients, OPENAI, GEMINI, BASE64, FILE, BUFFER, ERROR_GENERATING,
+    IMAGEN_MODEL, OPENAI_MODEL, VEO_MODEL,
 ] = [
-        {}, 'OPENAI', 'GEMINI', 'BASE64', 'BUFFER', 'Error generating image.',
-        'imagen-3.0-generate-002', 'gpt-image-1', 'veo-2.0-generate-001',
+        {}, 'OPENAI', 'GEMINI', 'BASE64', 'FILE', 'BUFFER',
+        'Error generating media.', 'imagen-3.0-generate-002', 'gpt-image-1',
+        'veo-2.0-generate-001',
     ];
 
 const init = async (options) => {
-    assert(
-        options?.apiKey || (options?.credentials && options?.projectId),
-        'API key or credentials are required.'
-    );
+    assert(options?.apiKey, 'API key is required.');
     const provider = ensureString(options?.provider, { case: 'UP' });
     switch (provider) {
         case OPENAI:
             const OpenAI = await need('openai');
-            const openai = new OpenAI(options);
+            var client = new OpenAI(options);
             clients[provider] = {
-                image: openai.images,
+                image: client.images,
                 toFile: OpenAI.toFile,
             };
             break;
         case GEMINI:
+            const { GoogleGenAI } = await need('@google/genai');
+            var client = new GoogleGenAI({ vertexai: false, ...options });
             clients[provider] = {
-                apiKey: options.apiKey,
-                projectId: options.projectId,
-                credentials: options.credentials,
+                gen: client,
             };
             break;
         default:
@@ -50,7 +47,7 @@ const extractImage = async (data, options) => await convert(
 );
 
 const extractVideo = async (data, options) => await convert(
-    data, { input: BASE64, suffix: 'mp4', ...options || {} }
+    data, { input: FILE, suffix: 'mp4', ...options || {} }
 );
 
 const prepareImage = async (files, repack, options) => {
@@ -68,8 +65,8 @@ const prepareImage = async (files, repack, options) => {
 
 const image = async (prompt, options) => {
     let provider = ensureString(options?.provider, { case: 'UP' });
-    if (!provider && clients?.[GEMINI]?.apiKey) { provider = GEMINI; }
-    if (!provider && clients?.[OPENAI]) { provider = OPENAI; }
+    if (!provider && clients?.[GEMINI]) { provider = GEMINI; }
+    else if (!provider && clients?.[OPENAI]) { provider = OPENAI; }
     const client = clients?.[provider];
     const n = options?.n || 4;
     assert(client, 'No available image generation provider.');
@@ -110,66 +107,25 @@ const image = async (prompt, options) => {
             }
             return resp?.data;
         case GEMINI:
-            // Image editing failed with the following error: imagen-3.0-capability-001 is unavailable.
-            // @todo: https://cloud.google.com/vertex-ai/generative-ai/docs/image/overview#feature-launch-stage
-            // cat << EOF > request.json
-            // {
-            //     "endpoint": "projects/backend-alpha-97077/locations/us-central1/publishers/google/models/imagen-3.0-capability-001",
-            //     "instances": [
-            //         {
-            //             "prompt": "ENTER PROMPT HERE",
-            //             "referenceImages": [
-            //                 {
-            //                     "referenceId": 1,
-            //                     "referenceType": "REFERENCE_TYPE_SUBJECT",
-            //                     "referenceImage": {
-            //                         "bytesBase64Encoded":
-            //                         },
-            //                     "subjectImageConfig" {
-            //                         "subjectDescription": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
-            //                         "subjectType": "SUBJECT_TYPE_DEFAULT"
-            //                     }
-            //                 }
-            //             ],
-            //         }
-            //     ],
-            //     "parameters": {
-            //         "aspectRatio": "1:1",
-            //         "sampleCount": 4,
-            //         "negativePrompt": "",
-            //         "enhancePrompt": false,
-            //         "personGeneration": "",
-            //         "safetySetting": "",
-            //         "addWatermark": true,
-            //         "includeRaiReason": true,
-            //         "language": "auto",
-            //     }
-            // }
-            // curl \
-            // -X POST \
-            // -H "Content-Type: application/json" \
-            // -H "Authorization: Bearer $(gcloud auth print-access-token)" \
-            // "https://${API_ENDPOINT}/v1/projects/${PROJECT_ID}/locations/${LOCATION_ID}/publishers/google/models/${MODEL_ID}:predict" -d '@request.json'
-            // ARGs: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api?authuser=4#rest_1
-            var resp = await (await fetch(
-                'https://generativelanguage.googleapis.com/v1beta/models/'
-                + `${IMAGEN_MODEL}:predict?key=${client.apiKey}`, {
-                method: 'POST', headers: { 'Content-Type': 'application/json' },
-                body: JSON.stringify({
-                    instances: [{ prompt }], parameters: {
-                        // "1:1" (default), "3:4", "4:3", "9:16", and "16:9"
-                        aspectRatio: '16:9', includeRaiReason: true,
-                        personGeneration: 'allow_adult', sampleCount: n,
-                        ...options?.params || {},
-                    },
-                })
-            })).json();
-            assert(!resp?.error, resp?.error?.message || ERROR_GENERATING);
+            var resp = await client.gen.models.generateImages({
+                model: IMAGEN_MODEL, prompt, config: {
+                    numberOfImages: n, includeRaiReason: true,
+                    // "1:1" (default), "3:4", "4:3", "9:16", and "16:9"
+                    aspectRatio: '16:9', personGeneration: 'allow_adult',
+                    ...options?.config || {},
+                },
+            });
+            const generated = resp?.generatedImages;
+            assert(!resp?.error && generated?.filter(
+                x => !x.raiFilteredReason
+            ).length, resp?.error?.message || generated?.find(
+                x => x.raiFilteredReason
+            )?.raiFilteredReason || ERROR_GENERATING);
             if (!options?.raw) {
-                resp = await Promise.all((resp?.predictions || []).map(
+                resp = await Promise.all((resp?.generatedImages || []).map(
                     async x => ({
                         caption: `🎨 by ${IMAGEN_MODEL}`,
-                        data: await extractImage(x.bytesBase64Encoded, options),
+                        data: await extractImage(x.image.imageBytes, options),
                         mimeType: x.mimeType,
                     })
                 ));
@@ -180,46 +136,11 @@ const image = async (prompt, options) => {
     }
 };
 
-const getGeminiAccessToken = async (credentials) => {
-    const bin = 'gcloud';
-    await assertExist(bin);
-    const actResp = await ignoreErrFunc(async () => await exec(
-        `${bin} auth activate-service-account --key-file=${credentials}`,
-        { acceptError: true }
-    ), { log: true });
-    assert(actResp?.includes?.('Activated service account credentials'),
-        'Failed to activate service account credentials.', 500);
-    const tokResp = (await exec(`gcloud auth print-access-token`)).trim();
-    assert(tokResp, 'Failed to get access token.', 500);
-    return tokResp;
-};
-
-const getGeminiVideo = async (jobId, accessToken) => {
-    const client = clients?.[GEMINI];
-    assert(client, 'No available video generation provider.');
-    const resp = await (await fetch(
-        'https://us-central1-aiplatform.googleapis.com/v1/projects/'
-        + `${client.projectId}/locations/us-central1/publishers/google/models/`
-        + `${VEO_MODEL}:fetchPredictOperation`, {
-        method: 'POST', headers: {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${accessToken}`,
-        }, body: JSON.stringify({ operationName: jobId })
-    })).json();
-    assert(resp?.response?.videos?.length,
-        'Waiting for Gemini video generation: '
-        + jobId.replace(/^.*\/([^/]+)$/, '$1'));
-    return resp?.response?.videos;
-};
-
 const video = async (prompt, options) => {
     let provider = ensureString(options?.provider, { case: 'UP' });
-    if (!provider
-        && clients?.[GEMINI]?.credentials
-        && clients?.[GEMINI]?.projectId) { provider = GEMINI; }
+    if (!provider && clients?.[GEMINI]) { provider = GEMINI; }
     const client = clients?.[provider];
     assert(client, 'No available video generation provider.');
-    const accessToken = await getGeminiAccessToken(client.credentials);
     prompt = ensureString(prompt);
     assert(prompt.length <= 4000,
         'Prompt must be less than 4000 characters.', 400);
@@ -229,39 +150,50 @@ const video = async (prompt, options) => {
     };
     switch (provider) {
         case GEMINI:
-            var resp = await (await fetch(
-                'https://us-central1-aiplatform.googleapis.com/v1/projects/'
-                + `${client.projectId}/locations/us-central1/publishers/google/`
-                + `models/${VEO_MODEL}:predictLongRunning`, {
-                method: 'POST', headers: {
-                    'Content-Type': 'application/json',
-                    'Authorization': `Bearer ${accessToken}`,
+            var resp = await client.gen.models.generateVideos({
+                model: VEO_MODEL, prompt, config: {
+                    aspectRatio: '16:9', numberOfVideos: 1,
+                    personGeneration: 'allow_adult',
+                    enablePromptRewriting: true, addWatermark: false,
+                    includeRaiReason: true, ...options?.config || {},
                 },
-                body: JSON.stringify({
-                    instances: [{ prompt }], parameters: {
-                        aspectRatio: '16:9', sampleCount: 4,
-                        durationSeconds: '8', fps: '24',
-                        personGeneration: 'allow_adult',
-                        enablePromptRewriting: true, addWatermark: false,
-                        includeRaiReason: true, ...options?.params || {},
-                    },
-                })
-            })).json();
-            assert(
-                !resp?.error && resp?.name,
-                resp?.error?.message || ERROR_GENERATING
-            );
+            });
+            assert(!resp?.error, resp?.error?.message || ERROR_GENERATING);
             if (options?.generateRaw) { return resp; }
-            var videos = await tryUntil(async () => await getGeminiVideo(
-                resp.name, accessToken
-            ), { maxTry: 60 * 10, log });
-            assert(videos?.length, 'Failed to generate Gemini video.');
-            if (options?.videoRaw) { return videos; }
-            return await Promise.all(videos.map(async x => ({
-                caption: `🎥 by ${VEO_MODEL}`,
-                data: await extractVideo(x.bytesBase64Encoded, options),
-                mimeType: x.mimeType, jobId: resp.name,
-            })));
+            await tryUntil(async () => {
+                resp = await client.gen.operations.getVideosOperation({
+                    operation: resp,
+                });
+                assert(
+                    resp?.done,
+                    `Waiting for Gemini video generation: ${resp.name}`,
+                );
+            }, { maxTry: 60 * 10, log });
+            let generated = resp?.response?.generatedVideos;
+            assert(!resp?.error && generated?.filter(
+                x => !x.raiFilteredReason
+            ).length, resp?.error?.message || generated?.find(
+                x => x.raiFilteredReason
+            )?.raiFilteredReason || ERROR_GENERATING);
+            if (!options?.videoRaw) {
+                generated = await Promise.all(generated?.filter(
+                    x => x?.video?.uri
+                ).map(async (x, i) => {
+                    const downloadPath = `${getTempPath({
+                        seed: x?.video?.uri
+                    })}.mp4`;
+                    // @todo: fix this
+                    // https://github.com/googleapis/js-genai/compare/main...Leask:js-genai:main
+                    await client.gen.files.download({ file: x, downloadPath });
+                    await timeout(1000 * 10); // hack to wait for file to be downloaded
+                    return {
+                        caption: `🎥 by ${VEO_MODEL}`,
+                        data: await extractVideo(downloadPath, options),
+                        mimeType: MIME_MP4, jobId: resp.name,
+                    };
+                }));
+            }
+            return generated;
         default:
             throw new Error('Invalid provider.');
     }

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.79",
+    "version": "1999.1.81",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.79",
+    "version": "1999.1.81",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",