utilitas 1999.1.58 → 1999.1.60

package/lib/gen.mjs

@@ -1,9 +1,10 @@
 import {
-    ensureString, log as _log, need, throwError, tryUntil,
+    ensureArray, ensureString, log as _log, need, throwError, tryUntil,
 } from './utilitas.mjs';
 
-import { convert, MIME_PNG } from './storage.mjs';
 import { assertCommand, exec } from './shell.mjs';
+import { convert, MIME_PNG } from './storage.mjs';
+import { createReadStream } from 'fs';
 
 const _NEED = ['OpenAI'];
 const log = (cnt, opt) => _log(cnt, import.meta.url, { time: 1, ...opt || {} });
@@ -25,7 +26,10 @@ const init = async (options) => {
         case OPENAI:
             const OpenAI = await need('openai');
             const openai = new OpenAI(options);
-            clients[provider] = openai.images;
+            clients[provider] = {
+                image: openai.images,
+                toFile: OpenAI.toFile,
+            };
             break;
         case GEMINI:
             clients[provider] = {
@@ -48,7 +52,20 @@ const extractVideo = async (data, options) => await convert(
     data, { input: BASE64, suffix: 'mp4', ...options || {} }
 );
 
-const generateImage = async (prompt, options) => {
+const prepareImage = async (files, repack, options) => {
+    if (!files) { return }
+    const multiple = Array.isArray(files);
+    files = ensureArray(files);
+    const resp = await Promise.all(files.map(async x => await repack(
+        createReadStream(await convert(
+            x, { expected: 'FILE', ...options || {} }
+        )), null, { type: MIME_PNG } // don't need to be right MIME type
+    )));
+    return multiple ? resp : resp[0];
+};
+
+
+const image = async (prompt, options) => {
     let provider = ensureString(options?.provider, { case: 'UP' });
     if (!provider && clients?.[GEMINI]?.apiKey) { provider = GEMINI; }
     if (!provider && clients?.[OPENAI]) { provider = OPENAI; }
@@ -64,33 +81,86 @@ const generateImage = async (prompt, options) => {
     };
     switch (provider) {
         case OPENAI:
+            let [func, extraOptions] = ['generate', {}];
+            if (options?.reference || options?.mask) {
+                func = 'edit';
+                extraOptions = {
+                    image: await prepareImage(options?.reference, client.toFile, options),
+                    mask: await prepareImage(options?.mask, client.toFile, options),
+                };
+            }
             try { // https://platform.openai.com/docs/guides/image-generation?image-generation-model=gpt-image-1
-                var resp = await client.generate({
+                var resp = await client.image[func]({
                     prompt, model: OPENAI_MODEL, n, quality: 'high',
                     size: '1536x1024', moderation: 'low',
                     // 1024x1024 (square), 1536x1024 (landscape), 1024x1536 (portrait), auto (default)
                     // background: 'transparent',
-                    ...options?.params || {},
+                    ...extraOptions, ...options?.params || {},
                 });
             } catch (err) { throwError(err?.message || ERROR_GENERATING); }
             if (!options?.raw) {
                 resp.data = await Promise.all(resp.data.map(async x => ({
                     caption: `🎨 by ${OPENAI_MODEL}`,
-                    data: await extractImage(x.b64_json, options),
+                    data: await extractImage(x.b64_json, {
+                        ...options || {}, input: BASE64,
+                    }),
                     mimeType: MIME_PNG,
                 })));
             }
             return resp?.data;
         case GEMINI:
+            // Image editing failed with the following error: imagen-3.0-capability-001 is unavailable.
+            // @todo: https://cloud.google.com/vertex-ai/generative-ai/docs/image/overview#feature-launch-stage
+            // cat << EOF > request.json
+            // {
+            //     "endpoint": "projects/backend-alpha-97077/locations/us-central1/publishers/google/models/imagen-3.0-capability-001",
+            //     "instances": [
+            //         {
+            //             "prompt": "ENTER PROMPT HERE",
+            //             "referenceImages": [
+            //                 {
+            //                     "referenceId": 1,
+            //                     "referenceType": "REFERENCE_TYPE_SUBJECT",
+            //                     "referenceImage": {
+            //                         "bytesBase64Encoded":
+            //                         },
+            //                     "subjectImageConfig" {
+            //                         "subjectDescription": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+            //                         "subjectType": "SUBJECT_TYPE_DEFAULT"
+            //                     }
+            //                 }
+            //             ],
+            //         }
+            //     ],
+            //     "parameters": {
+            //         "aspectRatio": "1:1",
+            //         "sampleCount": 4,
+            //         "negativePrompt": "",
+            //         "enhancePrompt": false,
+            //         "personGeneration": "",
+            //         "safetySetting": "",
+            //         "addWatermark": true,
+            //         "includeRaiReason": true,
+            //         "language": "auto",
+            //     }
+            // }
+            // curl \
+            // -X POST \
+            // -H "Content-Type: application/json" \
+            // -H "Authorization: Bearer $(gcloud auth print-access-token)" \
+            // "https://${API_ENDPOINT}/v1/projects/${PROJECT_ID}/locations/${LOCATION_ID}/publishers/google/models/${MODEL_ID}:predict" -d '@request.json'
+            // ARGs: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/imagen-api?authuser=4#rest_1
             var resp = await (await fetch(
                 'https://generativelanguage.googleapis.com/v1beta/models/'
                 + `${IMAGEN_MODEL}:predict?key=${client.apiKey}`, {
                 method: 'POST', headers: { 'Content-Type': 'application/json' },
                 body: JSON.stringify({
                     instances: [{ prompt }], parameters: {
-                        sampleCount: n, aspectRatio: '16:9',
+                        // "1:1" (default), "3:4", "4:3", "9:16", and "16:9"
+                        aspectRatio: '16:9', includeRaiReason: true,
+                        personGeneration: 'allow_adult', sampleCount: n,
                         ...options?.params || {},
-                    }, // "1:1" (default), "3:4", "4:3", "9:16", and "16:9"
+                    },
                 })
             })).json();
             assert(!resp?.error, resp?.error?.message || ERROR_GENERATING);
@@ -140,7 +210,7 @@ const getGeminiVideo = async (jobId, accessToken) => {
     return resp?.response?.videos;
 };
 
-const generateVideo = async (prompt, options) => {
+const video = async (prompt, options) => {
     let provider = ensureString(options?.provider, { case: 'UP' });
     if (!provider
         && clients?.[GEMINI]?.credentials
@@ -198,8 +268,7 @@ const generateVideo = async (prompt, options) => {
 export default init;
 export {
     _NEED,
-    generateImage,
-    generateVideo,
-    getGeminiVideo,
+    image,
     init,
+    video,
 };

package/lib/manifest.mjs

@@ -1,7 +1,7 @@
 const manifest = {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.58",
+    "version": "1999.1.60",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "utilitas",
     "description": "Just another common utility for JavaScript.",
-    "version": "1999.1.58",
+    "version": "1999.1.60",
     "private": false,
     "homepage": "https://github.com/Leask/utilitas",
     "main": "index.mjs",