wuchale 0.22.7 → 0.22.8

package/dist/ai/index.d.ts

@@ -5,6 +5,7 @@ type Batch = {
     targetLocales: string[];
     messages: Item[];
 };
+type GroupKey = string | string[];
 export type AIPassThruOpts = {
     batchSize: number;
     parallel: number;
@@ -16,7 +17,7 @@ export type AI = AIPassThruOpts & {
 };
 export default class AIQueue {
     #private;
-    batches: Batch[];
+    batches: Map<GroupKey, Batch[]>;
     nextBatchId: number;
     running: Promise<void> | null;
     sourceLocale: string;
@@ -27,6 +28,8 @@ export default class AIQueue {
     constructor(sourceLocale: string, ai: AI, onComplete: () => Promise<void>, log: Logger);
     translate: (batch: Batch, attempt?: number) => Promise<void>;
     run: () => Promise<void>;
-    add: (messages: Item[]) => void;
+    groupItemsByLocales: (items: Item[]) => Map<GroupKey, Item[]>;
+    prepItemsInBatches: (itemsByGroup: Map<GroupKey, Item[]>) => [string, Batch, number][];
+    add: (items: Item[]) => void;
 }
 export {};

package/dist/ai/index.js

@@ -48,7 +48,7 @@ Respond ONLY with raw compact JSON. Do not wrap it in markdown code fences or ad
 // implements a queue for a sequential translation useful for vite's transform during dev
 // as vite can do async transform
 export default class AIQueue {
-    batches = [];
+    batches = new Map();
     nextBatchId = 0;
     running = null;
     sourceLocale;
@@ -138,62 +138,80 @@ export default class AIQueue {
         await this.translate(batch, attempt);
     };
     run = async () => {
-        while (this.batches.length > 0) {
+        while (this.batches.size > 0) {
             const allBatches = [];
-            while (this.batches.length > 0 && allBatches.length < this.ai.parallel) {
-                allBatches.push(this.batches.pop());
+            for (const [group, batches] of this.batches) {
+                while (batches.length > 0 && allBatches.length < this.ai.parallel) {
+                    allBatches.push(batches.pop());
+                }
+                if (batches.length === 0) {
+                    this.batches.delete(group);
+                }
             }
             await Promise.all(allBatches.map(this.translate));
         }
         await this.onComplete();
         this.running = null;
     };
-    add = (messages) => {
-        if (!this.ai) {
-            return;
-        }
+    groupItemsByLocales = (items) => {
         const itemsByLocales = new Map();
-        for (const item of messages) {
+        for (const item of items) {
             for (const [loc, transl] of item.translations.entries()) {
                 if (loc === this.sourceLocale || transl[0]) {
                     continue;
                 }
                 const group = this.ai.group[this.sourceLocale]?.find(g => g.includes(loc));
                 const groupKey = group ?? loc;
-                if (!itemsByLocales.has(groupKey)) {
-                    itemsByLocales.set(groupKey, []);
+                const groupItems = itemsByLocales.get(groupKey);
+                if (groupItems == null) {
+                    itemsByLocales.set(groupKey, [item]);
                 }
-                itemsByLocales.get(groupKey)?.push(item);
+                else {
+                    groupItems.push(item);
+                }
+            }
+        }
+        return itemsByLocales;
+    };
+    prepItemsInBatches = (itemsByGroup) => {
+        const opInfo = [];
+        for (let [groupKey, items] of itemsByGroup) {
+            const groupBatches = this.batches.get(groupKey) ?? [];
+            const lastBatch = groupBatches.at(-1);
+            if (lastBatch && lastBatch.messages.length < this.ai.batchSize) {
+                const lastBatchFree = this.ai.batchSize - lastBatch.messages.length;
+                const itemsToAdd = items.slice(0, lastBatchFree);
+                opInfo.push(['(add)', lastBatch, itemsToAdd.length]);
+                lastBatch.messages.push(...itemsToAdd);
+                items = items.slice(lastBatchFree);
             }
+            for (let i = 0; i < items.length; i += this.ai.batchSize) {
+                const chunk = items.slice(i, i + this.ai.batchSize);
+                const batch = {
+                    id: this.nextBatchId,
+                    targetLocales: Array.isArray(groupKey) ? groupKey : [groupKey],
+                    messages: chunk,
+                };
+                groupBatches.push(batch);
+                opInfo.push([color.yellow('(new)'), batch, chunk.length]);
+                this.nextBatchId++;
+            }
+            if (!this.batches.has(groupKey)) {
+                this.batches.set(groupKey, groupBatches);
+            }
+        }
+        return opInfo;
+    };
+    add = (items) => {
+        if (!this.ai) {
+            return;
         }
+        const itemsByLocales = this.groupItemsByLocales(items);
         if (itemsByLocales.size === 0) {
             // all translated
             return;
         }
-        const opInfo = [];
-        const lastBatch = this.batches.at(-1);
-        if (lastBatch && lastBatch.messages.length < this.ai.batchSize && itemsByLocales.has(lastBatch.targetLocales)) {
-            const lastBatchFree = this.ai.batchSize - lastBatch.messages.length;
-            const localeItems = itemsByLocales.get(lastBatch.targetLocales);
-            const msgs = localeItems.slice(0, lastBatchFree);
-            opInfo.push(['(add)', lastBatch, msgs.length]);
-            lastBatch.messages.push(...msgs);
-            itemsByLocales.set(lastBatch.targetLocales, localeItems.slice(lastBatchFree));
-        }
-        for (const [groupKey, items] of itemsByLocales) {
-            if (items.length === 0) {
-                continue;
-            }
-            const batch = {
-                id: this.nextBatchId,
-                targetLocales: Array.isArray(groupKey) ? groupKey : [groupKey],
-                messages: items,
-            };
-            this.batches.push(batch);
-            opInfo.push([color.yellow('(new)'), batch, items.length]);
-            this.nextBatchId++;
-        }
-        for (const [opType, batch, msgsLen] of opInfo) {
+        for (const [opType, batch, msgsLen] of this.prepItemsInBatches(itemsByLocales)) {
             this.log.info(`${this.#requestName(batch.id, batch.targetLocales)}: ${opType} translate ${color.cyan(msgsLen)} messages`);
         }
         if (!this.running) {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "wuchale",
-    "version": "0.22.7",
+    "version": "0.22.8",
     "description": "Protobuf-like i18n from plain code",
     "scripts": {
         "dev": "tsc --watch",