react-native-executorch 0.4.4 → 0.4.6

package/android/build.gradle

@@ -103,7 +103,7 @@ dependencies {
   implementation "com.facebook.react:react-android:+"
   implementation 'org.opencv:opencv:4.10.0'
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
-	implementation("com.github.software-mansion:react-native-executorch:main-SNAPSHOT")
+	implementation("com.github.software-mansion:react-native-executorch:v0.4.5")
   implementation 'org.opencv:opencv:4.10.0'
   implementation("com.squareup.okhttp3:okhttp:4.9.2")
 }

package/lib/module/controllers/LLMController.js

@@ -19,12 +19,19 @@ export class LLMController {
   // User callbacks
 
   constructor({
+    tokenCallback,
     responseCallback,
     messageHistoryCallback,
     isReadyCallback,
     isGeneratingCallback,
     onDownloadProgressCallback
   }) {
+    if (responseCallback !== undefined) {
+      console.warn('Passing response callback is deprecated and will be removed in 0.6.0');
+    }
+    this.tokenCallback = token => {
+      tokenCallback?.(token);
+    };
     this.responseCallback = response => {
       this._response = response;
       responseCallback?.(response);
@@ -74,9 +81,10 @@ export class LLMController {
       await this.nativeModule.loadLLM(modelFileUri, tokenizerFileUri);
       this.isReadyCallback(true);
       this.onToken = this.nativeModule.onToken(data => {
-        if (!data) {
+        if (!data || SPECIAL_TOKENS.EOS_TOKEN in this.tokenizerConfig && data === this.tokenizerConfig.eos_token || SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig && data === this.tokenizerConfig.pad_token) {
           return;
         }
+        this.tokenCallback(data);
         this.responseCallback(this._response + data);
       });
     } catch (e) {
@@ -84,6 +92,9 @@ export class LLMController {
       throw new Error(getError(e));
     }
   }
+  setTokenCallback(tokenCallback) {
+    this.tokenCallback = tokenCallback;
+  }
   configure({
     chatConfig,
     toolsConfig
@@ -146,11 +157,6 @@ export class LLMController {
       add_generation_prompt: true
     });
     await this.forward(renderedChat);
-    if (!this._response) {
-      return;
-    }
-    const cleanedResponse = this._response.replaceAll(this.tokenizerConfig.eos_token, '').replaceAll(this.tokenizerConfig.pad_token, '');
-    this.responseCallback(cleanedResponse);
   }
   async sendMessage(message) {
     this.messageHistoryCallback([...this._messageHistory, {
@@ -163,7 +169,6 @@ export class LLMController {
     }, ...this._messageHistory.slice(-this.chatConfig.contextWindowLength)];
     await this.generate(messageHistoryWithPrompt, this.toolsConfig?.tools);
     if (!this.toolsConfig || this.toolsConfig.displayToolCalls) {
-      this.responseCallback(this._response.replace(this.tokenizerConfig.eos_token, ''));
       this.messageHistoryCallback([...this._messageHistory, {
         content: this._response,
         role: 'assistant'
@@ -195,7 +200,7 @@ export class LLMController {
       throw Error("Tokenizer config doesn't include chat_template");
     }
     const template = new Template(tokenizerConfig.chat_template);
-    const specialTokens = Object.fromEntries(SPECIAL_TOKENS.filter(key => key in tokenizerConfig).map(key => [key, tokenizerConfig[key]]));
+    const specialTokens = Object.fromEntries(Object.keys(SPECIAL_TOKENS).filter(key => key in tokenizerConfig).map(key => [key, tokenizerConfig[key]]));
     const result = template.render({
       messages,
       tools,

package/lib/module/controllers/LLMController.js.map

@@ -1 +1 @@
-{"version":3,"names":["ResourceFetcher","ETError","getError","Template","DEFAULT_CHAT_CONFIG","readAsStringAsync","SPECIAL_TOKENS","LLMNativeModule","parseToolCall","LLMController","chatConfig","onToken","_response","_isReady","_isGenerating","_messageHistory","constructor","responseCallback","messageHistoryCallback","isReadyCallback","isGeneratingCallback","onDownloadProgressCallback","response","messageHistory","isReady","isGenerating","nativeModule","load","modelSource","tokenizerSource","tokenizerConfigSource","initialMessageHistory","tokenizerFileUri","fetch","tokenizerConfigFileUri","tokenizerConfig","JSON","parse","modelFileUri","loadLLM","data","e","Error","configure","toolsConfig","delete","ModelGenerating","remove","releaseResources","forward","input","ModuleNotLoaded","interrupt","generate","messages","tools","length","role","console","warn","renderedChat","applyChatTemplate","tools_in_user_message","add_generation_prompt","cleanedResponse","replaceAll","eos_token","pad_token","sendMessage","message","content","messageHistoryWithPrompt","systemPrompt","slice","contextWindowLength","displayToolCalls","replace","toolCalls","toolCall","executeToolCallback","then","toolResponse","deleteMessage","index","newMessageHistory","templateFlags","chat_template","template","specialTokens","Object","fromEntries","filter","key","map","result","render"],"sourceRoot":"../../../src","sources":["controllers/LLMController.ts"],"mappings":";;AAEA,SAASA,eAAe,QAAQ,0BAA0B;AAC1D,SAASC,OAAO,EAAEC,QAAQ,QAAQ,UAAU;AAC5C,SAASC,QAAQ,QAAQ,oBAAoB;AAC7C,SAASC,mBAAmB,QAAQ,0BAA0B;AAC9D,SAASC,iBAAiB,QAAQ,kBAAkB;AACpD,SAIEC,cAAc,QAET,cAAc;AACrB,SAASC,eAAe,QAAQ,+BAA+B;AAC/D,SAASC,aAAa,QAAQ,cAAc;AAE5C,OAAO,MAAMC,aAAa,CAAC;EAEjBC,UAAU,GAAeN,mBAAmB;EAG5CO,OAAO,GAA6B,IAAI;EACxCC,SAAS,GAAG,EAAE;EACdC,QAAQ,GAAG,KAAK;EAChBC,aAAa,GAAG,KAAK;EACrBC,eAAe,GAAc,EAAE;;EAEvC;;EASAC,WAAWA,CAAC;IACVC,gBAAgB;IAChBC,sBAAsB;IACtBC,eAAe;IACfC,oBAAoB;IACpBC;EAOF,CAAC,EAAE;IACD,IAAI,CAACJ,gBAAgB,GAAIK,QAAQ,IAAK;MACpC,IAAI,CAACV,SAAS,GAAGU,QAAQ;MACzBL,gBAAgB,GAAGK,QAAQ,CAAC;IAC9B,CAAC;IACD,IAAI,CAACJ,sBAAsB,GAAIK,cAAc,IAAK;MAChD,IAAI,CAACR,eAAe,GAAGQ,cAAc;MACrCL,sBAAsB,GAAGK,cAAc,CAAC;IAC1C,CAAC;IACD,IAAI,CAACJ,eAAe,GAAIK,OAAO,IAAK;MAClC,IAAI,CAACX,QAAQ,GAAGW,OAAO;MACvBL,eAAe,GAAGK,OAAO,CAAC;IAC5B,CAAC;IACD,IAAI,CAACJ,oBAAoB,GAAIK,YAAY,IAAK;MAC5C,IAAI,CAACX,aAAa,GAAGW,YAAY;MACjCL,oBAAoB,GAAGK,YAAY,CAAC;IACtC,CAAC;IAED,IAAI,CAACJ,0BAA0B,GAAGA,0BAA0B;IAE5D,IAAI,CAACK,YAAY,GAAGnB,eAAe;EACrC;EAEA,IAAWe,QAAQA,CAAA,EAAG;IACpB,OAAO,IAAI,CAACV,SAAS;EACvB;EACA,IAAWY,OAAOA,CAAA,EAAG;IACnB,OAAO,IAAI,CAACX,QAAQ;EACtB;EACA,IAAWY,YAAYA,CAAA,EAAG;IACxB,OAAO,IAAI,CAACX,aAAa;EAC3B;EACA,IAAWS,cAAcA,CAAA,EAAG;IAC1B,OAAO,IAAI,CAACR,eAAe;EAC7B;EAEA,MAAaY,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC;EAKF,CAAC,EAAE;IACD;IACA,IAAI,CAACb,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACqB,qBAAqB,CAAC;IAClE,IAAI,CAACX,oBAAoB,CAAC,KAAK,CAAC;IAChC,IAAI,CAACD,eAAe,CAAC,KAAK,CAAC;IAE3B,IAAI;MACF,MAAMa,gBAAgB,GAAG,MAAMhC,eAAe,CAACiC,KAAK,CAACJ,eAAe,CAAC;MACrE,MAAMK,sBAAsB,GAAG,MAAMlC,eAAe,CAACiC,KAAK,CACxDH,qBACF,CAAC;MACD,IAAI,CAACK,eAAe,GAAGC,IAAI,CAACC,KAAK,CAC/B,MAAMhC,iBAAiB,CAAC,SAAS,GAAG6B,sBAAsB,CAC5D,CAAC;MAED,MAAMI,YAAY,GAAG,MAAMtC,eAAe,CAACiC,KAAK,CAC9CL,WAAW,EACX,IAAI,CAACP,0BACP,CAAC;MAED,MAAM,IAAI,CAACK,YAAY,CAACa,OAAO,CAACD,YAAY,EAAEN,gBAAgB,CAAC;MAC/D,IAAI,CAACb,eAAe,CAAC,IAAI,CAAC;MAC1B,IAAI,CAACR,OAAO,GAAG,IAAI,CAACe,YAAY,CAACf,OAAO,CAAE6B,IAAwB,IAAK;QACrE,IAAI,CAACA,IAAI,EAAE;UACT;QACF;QACA,IAAI,CAACvB,gBAAgB,CAAC,IAAI,CAACL,SAAS,GAAG4B,IAAI,CAAC;MAC9C,CAAC,CAAC;IACJ,CAAC,CAAC,OAAOC,CAAC,EAAE;MACV,IAAI,CAACtB,eAAe,CAAC,KAAK,CAAC;MAC3B,MAAM,IAAIuB,KAAK,CAACxC,QAAQ,CAACuC,CAAC,CAAC,CAAC;IAC9B;EACF;EAEOE,SAASA,CAAC;IACfjC,UAAU;IACVkC;EAIF,CAAC,EAAE;IACD,IAAI,CAAClC,UAAU,GAAG;MAAE,GAAGN,mBAAmB;MAAE,GAAGM;IAAW,CAAC;IAC3D,IAAI,CAACkC,WAAW,GAAGA,WAAW;;IAE9B;IACA,IAAI,CAAC3B,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACqB,qBAAqB,CAAC;IAClE,IAAI,CAACX,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEOyB,MAAMA,CAAA,EAAG;IACd,IAAI,IAAI,CAAC/B,aAAa,EAAE;MACtB,MAAM,IAAI4B,KAAK,CACbxC,QAAQ,CAACD,OAAO,CAAC6C,eAAe,CAAC,GAC/B,+DACJ,CAAC;IACH;IACA,IAAI,CAACnC,OAAO,EAAEoC,MAAM,CAAC,CAAC;IACtB,IAAI,CAACpC,OAAO,GAAG,IAAI;IACnB,IAAI,CAACe,YAAY,CAACsB,gBAAgB,CAAC,CAAC;IACpC,IAAI,CAAC7B,eAAe,CAAC,KAAK,CAAC;IAC3B,IAAI,CAACC,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEA,MAAa6B,OAAOA,CAACC,KAAa,EAAE;IAClC,IAAI,CAAC,IAAI,CAACrC,QAAQ,EAAE;MAClB,MAAM,IAAI6B,KAAK,CAACxC,QAAQ,CAACD,OAAO,CAACkD,eAAe,CAAC,CAAC;IACpD;IACA,IAAI,IAAI,CAACrC,aAAa,EAAE;MACtB,MAAM,IAAI4B,KAAK,CAACxC,QAAQ,CAACD,OAAO,CAAC6C,eAAe,CAAC,CAAC;IACpD;IACA,IAAI;MACF,IAAI,CAAC7B,gBAAgB,CAAC,EAAE,CAAC;MACzB,IAAI,CAACG,oBAAoB,CAAC,IAAI,CAAC;MAC/B,MAAM,IAAI,CAACM,YAAY,CAACuB,OAAO,CAACC,KAAK,CAAC;IACxC,CAAC,CAAC,OAAOT,CAAC,EAAE;MACV,MAAM,IAAIC,KAAK,CAACxC,QAAQ,CAACuC,CAAC,CAAC,CAAC;IAC9B,CAAC,SAAS;MACR,IAAI,CAACrB,oBAAoB,CAAC,KAAK,CAAC;IAClC;EACF;EAEOgC,SAASA,CAAA,EAAG;IACjB,IAAI,CAAC1B,YAAY,CAAC0B,SAAS,CAAC,CAAC;EAC/B;EAEA,MAAaC,QAAQA,CAACC,QAAmB,EAAEC,KAAiB,EAAE;IAC5D,IAAI,CAAC,IAAI,CAAC1C,QAAQ,EAAE;MAClB,MAAM,IAAI6B,KAAK,CAACxC,QAAQ,CAACD,OAAO,CAACkD,eAAe,CAAC,CAAC;IACpD;IACA,IAAIG,QAAQ,CAACE,MAAM,KAAK,CAAC,EAAE;MACzB,MAAM,IAAId,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IACA,IAAIY,QAAQ,CAAC,CAAC,CAAC,IAAIA,QAAQ,CAAC,CAAC,CAAC,CAACG,IAAI,KAAK,QAAQ,EAAE;MAChDC,OAAO,CAACC,IAAI,CACV,0LACF,CAAC;IACH;IAEA,MAAMC,YAAoB,GAAG,IAAI,CAACC,iBAAiB,CACjDP,QAAQ,EACR,IAAI,CAACnB,eAAe,EACpBoB,KAAK;IACL;IACA;MAAEO,qBAAqB,EAAE,KAAK;MAAEC,qBAAqB,EAAE;IAAK,CAC9D,CAAC;IAED,MAAM,IAAI,CAACd,OAAO,CAACW,YAAY,CAAC;IAEhC,IAAI,CAAC,IAAI,CAAChD,SAAS,EAAE;MACnB;IACF;IAEA,MAAMoD,eAAe,GAAG,IAAI,CAACpD,SAAS,CACnCqD,UAAU,CAAC,IAAI,CAAC9B,eAAe,CAAC+B,SAAS,EAAE,EAAE,CAAC,CAC9CD,UAAU,CAAC,IAAI,CAAC9B,eAAe,CAACgC,SAAS,EAAE,EAAE,CAAC;IACjD,IAAI,CAAClD,gBAAgB,CAAC+C,eAAe,CAAC;EACxC;EAEA,MAAaI,WAAWA,CAACC,OAAe,EAAE;IACxC,IAAI,CAACnD,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACH,eAAe,EACvB;MAAEuD,OAAO,EAAED,OAAO;MAAEZ,IAAI,EAAE;IAAO,CAAC,CACnC,CAAC;IAEF,MAAMc,wBAAmC,GAAG,CAC1C;MAAED,OAAO,EAAE,IAAI,CAAC5D,UAAU,CAAC8D,YAAY;MAAEf,IAAI,EAAE;IAAS,CAAC,EACzD,GAAG,IAAI,CAAC1C,eAAe,CAAC0D,KAAK,CAAC,CAAC,IAAI,CAAC/D,UAAU,CAACgE,mBAAmB,CAAC,CACpE;IAED,MAAM,IAAI,CAACrB,QAAQ,CAACkB,wBAAwB,EAAE,IAAI,CAAC3B,WAAW,EAAEW,KAAK,CAAC;IAEtE,IAAI,CAAC,IAAI,CAACX,WAAW,IAAI,IAAI,CAACA,WAAW,CAAC+B,gBAAgB,EAAE;MAC1D,IAAI,CAAC1D,gBAAgB,CACnB,IAAI,CAACL,SAAS,CAACgE,OAAO,CAAC,IAAI,CAACzC,eAAe,CAAC+B,SAAS,EAAE,EAAE,CAC3D,CAAC;MACD,IAAI,CAAChD,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACH,eAAe,EACvB;QAAEuD,OAAO,EAAE,IAAI,CAAC1D,SAAS;QAAE6C,IAAI,EAAE;MAAY,CAAC,CAC/C,CAAC;IACJ;IACA,IAAI,CAAC,IAAI,CAACb,WAAW,EAAE;MACrB;IACF;IAEA,MAAMiC,SAAS,GAAGrE,aAAa,CAAC,IAAI,CAACI,SAAS,CAAC;IAE/C,KAAK,MAAMkE,QAAQ,IAAID,SAAS,EAAE;MAChC,IAAI,CAACjC,WAAW,CACbmC,mBAAmB,CAACD,QAAQ,CAAC,CAC7BE,IAAI,CAAEC,YAA2B,IAAK;QACrC,IAAIA,YAAY,EAAE;UAChB,IAAI,CAAC/D,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACH,eAAe,EACvB;YAAEuD,OAAO,EAAEW,YAAY;YAAExB,IAAI,EAAE;UAAY,CAAC,CAC7C,CAAC;QACJ;MACF,CAAC,CAAC;IACN;EACF;EAEOyB,aAAaA,CAACC,KAAa,EAAE;IAClC;IACA;IACA,MAAMC,iBAAiB,GAAG,IAAI,CAACrE,eAAe,CAAC0D,KAAK,CAAC,CAAC,EAAEU,KAAK,CAAC;IAE9D,IAAI,CAACjE,sBAAsB,CAACkE,iBAAiB,CAAC;EAChD;EAEQvB,iBAAiBA,CACvBP,QAAmB,EACnBnB,eAAoB,EACpBoB,KAAiB,EACjB8B,aAAsB,EACd;IACR,IAAI,CAAClD,eAAe,CAACmD,aAAa,EAAE;MAClC,MAAM5C,KAAK,CAAC,gDAAgD,CAAC;IAC/D;IACA,MAAM6C,QAAQ,GAAG,IAAIpF,QAAQ,CAACgC,eAAe,CAACmD,aAAa,CAAC;IAE5D,MAAME,aAAa,GAAGC,MAAM,CAACC,WAAW,CACtCpF,cAAc,CAACqF,MAAM,CAAEC,GAAG,IAAKA,GAAG,IAAIzD,eAAe,CAAC,CAAC0D,GAAG,CAAED,GAAG,IAAK,CAClEA,GAAG,EACHzD,eAAe,CAACyD,GAAG,CAAC,CACrB,CACH,CAAC;IAED,MAAME,MAAM,GAAGP,QAAQ,CAACQ,MAAM,CAAC;MAC7BzC,QAAQ;MACRC,KAAK;MACL,GAAG8B,aAAa;MAChB,GAAGG;IACL,CAAC,CAAC;IACF,OAAOM,MAAM;EACf;AACF","ignoreList":[]}
+{"version":3,"names":["ResourceFetcher","ETError","getError","Template","DEFAULT_CHAT_CONFIG","readAsStringAsync","SPECIAL_TOKENS","LLMNativeModule","parseToolCall","LLMController","chatConfig","onToken","_response","_isReady","_isGenerating","_messageHistory","constructor","tokenCallback","responseCallback","messageHistoryCallback","isReadyCallback","isGeneratingCallback","onDownloadProgressCallback","undefined","console","warn","token","response","messageHistory","isReady","isGenerating","nativeModule","load","modelSource","tokenizerSource","tokenizerConfigSource","initialMessageHistory","tokenizerFileUri","fetch","tokenizerConfigFileUri","tokenizerConfig","JSON","parse","modelFileUri","loadLLM","data","EOS_TOKEN","eos_token","PAD_TOKEN","pad_token","e","Error","setTokenCallback","configure","toolsConfig","delete","ModelGenerating","remove","releaseResources","forward","input","ModuleNotLoaded","interrupt","generate","messages","tools","length","role","renderedChat","applyChatTemplate","tools_in_user_message","add_generation_prompt","sendMessage","message","content","messageHistoryWithPrompt","systemPrompt","slice","contextWindowLength","displayToolCalls","toolCalls","toolCall","executeToolCallback","then","toolResponse","deleteMessage","index","newMessageHistory","templateFlags","chat_template","template","specialTokens","Object","fromEntries","keys","filter","key","map","result","render"],"sourceRoot":"../../../src","sources":["controllers/LLMController.ts"],"mappings":";;AAEA,SAASA,eAAe,QAAQ,0BAA0B;AAC1D,SAASC,OAAO,EAAEC,QAAQ,QAAQ,UAAU;AAC5C,SAASC,QAAQ,QAAQ,oBAAoB;AAC7C,SAASC,mBAAmB,QAAQ,0BAA0B;AAC9D,SAASC,iBAAiB,QAAQ,kBAAkB;AACpD,SAIEC,cAAc,QAET,cAAc;AACrB,SAASC,eAAe,QAAQ,+BAA+B;AAC/D,SAASC,aAAa,QAAQ,cAAc;AAE5C,OAAO,MAAMC,aAAa,CAAC;EAEjBC,UAAU,GAAeN,mBAAmB;EAG5CO,OAAO,GAA6B,IAAI;EACxCC,SAAS,GAAG,EAAE;EACdC,QAAQ,GAAG,KAAK;EAChBC,aAAa,GAAG,KAAK;EACrBC,eAAe,GAAc,EAAE;;EAEvC;;EAUAC,WAAWA,CAAC;IACVC,aAAa;IACbC,gBAAgB;IAChBC,sBAAsB;IACtBC,eAAe;IACfC,oBAAoB;IACpBC;EAQF,CAAC,EAAE;IACD,IAAIJ,gBAAgB,KAAKK,SAAS,EAAE;MAClCC,OAAO,CAACC,IAAI,CACV,sEACF,CAAC;IACH;IACA,IAAI,CAACR,aAAa,GAAIS,KAAK,IAAK;MAC9BT,aAAa,GAAGS,KAAK,CAAC;IACxB,CAAC;IACD,IAAI,CAACR,gBAAgB,GAAIS,QAAQ,IAAK;MACpC,IAAI,CAACf,SAAS,GAAGe,QAAQ;MACzBT,gBAAgB,GAAGS,QAAQ,CAAC;IAC9B,CAAC;IACD,IAAI,CAACR,sBAAsB,GAAIS,cAAc,IAAK;MAChD,IAAI,CAACb,eAAe,GAAGa,cAAc;MACrCT,sBAAsB,GAAGS,cAAc,CAAC;IAC1C,CAAC;IACD,IAAI,CAACR,eAAe,GAAIS,OAAO,IAAK;MAClC,IAAI,CAAChB,QAAQ,GAAGgB,OAAO;MACvBT,eAAe,GAAGS,OAAO,CAAC;IAC5B,CAAC;IACD,IAAI,CAACR,oBAAoB,GAAIS,YAAY,IAAK;MAC5C,IAAI,CAAChB,aAAa,GAAGgB,YAAY;MACjCT,oBAAoB,GAAGS,YAAY,CAAC;IACtC,CAAC;IAED,IAAI,CAACR,0BAA0B,GAAGA,0BAA0B;IAE5D,IAAI,CAACS,YAAY,GAAGxB,eAAe;EACrC;EAEA,IAAWoB,QAAQA,CAAA,EAAG;IACpB,OAAO,IAAI,CAACf,SAAS;EACvB;EACA,IAAWiB,OAAOA,CAAA,EAAG;IACnB,OAAO,IAAI,CAAChB,QAAQ;EACtB;EACA,IAAWiB,YAAYA,CAAA,EAAG;IACxB,OAAO,IAAI,CAAChB,aAAa;EAC3B;EACA,IAAWc,cAAcA,CAAA,EAAG;IAC1B,OAAO,IAAI,CAACb,eAAe;EAC7B;EAEA,MAAaiB,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC;EAKF,CAAC,EAAE;IACD;IACA,IAAI,CAACjB,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACT,UAAU,CAAC0B,qBAAqB,CAAC;IAClE,IAAI,CAACf,oBAAoB,CAAC,KAAK,CAAC;IAChC,IAAI,CAACD,eAAe,CAAC,KAAK,CAAC;IAE3B,IAAI;MACF,MAAMiB,gBAAgB,GAAG,MAAMrC,eAAe,CAACsC,KAAK,CAACJ,eAAe,CAAC;MACrE,MAAMK,sBAAsB,GAAG,MAAMvC,eAAe,CAACsC,KAAK,CACxDH,qBACF,CAAC;MACD,IAAI,CAACK,eAAe,GAAGC,IAAI,CAACC,KAAK,CAC/B,MAAMrC,iBAAiB,CAAC,SAAS,GAAGkC,sBAAsB,CAC5D,CAAC;MAED,MAAMI,YAAY,GAAG,MAAM3C,eAAe,CAACsC,KAAK,CAC9CL,WAAW,EACX,IAAI,CAACX,0BACP,CAAC;MAED,MAAM,IAAI,CAACS,YAAY,CAACa,OAAO,CAACD,YAAY,EAAEN,gBAAgB,CAAC;MAC/D,IAAI,CAACjB,eAAe,CAAC,IAAI,CAAC;MAC1B,IAAI,CAACT,OAAO,GAAG,IAAI,CAACoB,YAAY,CAACpB,OAAO,CAAEkC,IAAY,IAAK;QACzD,IACE,CAACA,IAAI,IACJvC,cAAc,CAACwC,SAAS,IAAI,IAAI,CAACN,eAAe,IAC/CK,IAAI,KAAK,IAAI,CAACL,eAAe,CAACO,SAAU,IACzCzC,cAAc,CAAC0C,SAAS,IAAI,IAAI,CAACR,eAAe,IAC/CK,IAAI,KAAK,IAAI,CAACL,eAAe,CAACS,SAAU,EAC1C;UACA;QACF;QAEA,IAAI,CAAChC,aAAa,CAAC4B,IAAI,CAAC;QACxB,IAAI,CAAC3B,gBAAgB,CAAC,IAAI,CAACN,SAAS,GAAGiC,IAAI,CAAC;MAC9C,CAAC,CAAC;IACJ,CAAC,CAAC,OAAOK,CAAC,EAAE;MACV,IAAI,CAAC9B,eAAe,CAAC,KAAK,CAAC;MAC3B,MAAM,IAAI+B,KAAK,CAACjD,QAAQ,CAACgD,CAAC,CAAC,CAAC;IAC9B;EACF;EAEOE,gBAAgBA,CAACnC,aAAsC,EAAE;IAC9D,IAAI,CAACA,aAAa,GAAGA,aAAa;EACpC;EAEOoC,SAASA,CAAC;IACf3C,UAAU;IACV4C;EAIF,CAAC,EAAE;IACD,IAAI,CAAC5C,UAAU,GAAG;MAAE,GAAGN,mBAAmB;MAAE,GAAGM;IAAW,CAAC;IAC3D,IAAI,CAAC4C,WAAW,GAAGA,WAAW;;IAE9B;IACA,IAAI,CAACpC,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACT,UAAU,CAAC0B,qBAAqB,CAAC;IAClE,IAAI,CAACf,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEOkC,MAAMA,CAAA,EAAG;IACd,IAAI,IAAI,CAACzC,aAAa,EAAE;MACtB,MAAM,IAAIqC,KAAK,CACbjD,QAAQ,CAACD,OAAO,CAACuD,eAAe,CAAC,GAC/B,+DACJ,CAAC;IACH;IACA,IAAI,CAAC7C,OAAO,EAAE8C,MAAM,CAAC,CAAC;IACtB,IAAI,CAAC9C,OAAO,GAAG,IAAI;IACnB,IAAI,CAACoB,YAAY,CAAC2B,gBAAgB,CAAC,CAAC;IACpC,IAAI,CAACtC,eAAe,CAAC,KAAK,CAAC;IAC3B,IAAI,CAACC,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEA,MAAasC,OAAOA,CAACC,KAAa,EAAE;IAClC,IAAI,CAAC,IAAI,CAAC/C,QAAQ,EAAE;MAClB,MAAM,IAAIsC,KAAK,CAACjD,QAAQ,CAACD,OAAO,CAAC4D,eAAe,CAAC,CAAC;IACpD;IACA,IAAI,IAAI,CAAC/C,aAAa,EAAE;MACtB,MAAM,IAAIqC,KAAK,CAACjD,QAAQ,CAACD,OAAO,CAACuD,eAAe,CAAC,CAAC;IACpD;IACA,IAAI;MACF,IAAI,CAACtC,gBAAgB,CAAC,EAAE,CAAC;MACzB,IAAI,CAACG,oBAAoB,CAAC,IAAI,CAAC;MAC/B,MAAM,IAAI,CAACU,YAAY,CAAC4B,OAAO,CAACC,KAAK,CAAC;IACxC,CAAC,CAAC,OAAOV,CAAC,EAAE;MACV,MAAM,IAAIC,KAAK,CAACjD,QAAQ,CAACgD,CAAC,CAAC,CAAC;IAC9B,CAAC,SAAS;MACR,IAAI,CAAC7B,oBAAoB,CAAC,KAAK,CAAC;IAClC;EACF;EAEOyC,SAASA,CAAA,EAAG;IACjB,IAAI,CAAC/B,YAAY,CAAC+B,SAAS,CAAC,CAAC;EAC/B;EAEA,MAAaC,QAAQA,CAACC,QAAmB,EAAEC,KAAiB,EAAE;IAC5D,IAAI,CAAC,IAAI,CAACpD,QAAQ,EAAE;MAClB,MAAM,IAAIsC,KAAK,CAACjD,QAAQ,CAACD,OAAO,CAAC4D,eAAe,CAAC,CAAC;IACpD;IACA,IAAIG,QAAQ,CAACE,MAAM,KAAK,CAAC,EAAE;MACzB,MAAM,IAAIf,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IACA,IAAIa,QAAQ,CAAC,CAAC,CAAC,IAAIA,QAAQ,CAAC,CAAC,CAAC,CAACG,IAAI,KAAK,QAAQ,EAAE;MAChD3C,OAAO,CAACC,IAAI,CACV,0LACF,CAAC;IACH;IAEA,MAAM2C,YAAoB,GAAG,IAAI,CAACC,iBAAiB,CACjDL,QAAQ,EACR,IAAI,CAACxB,eAAe,EACpByB,KAAK;IACL;IACA;MAAEK,qBAAqB,EAAE,KAAK;MAAEC,qBAAqB,EAAE;IAAK,CAC9D,CAAC;IAED,MAAM,IAAI,CAACZ,OAAO,CAACS,YAAY,CAAC;EAClC;EAEA,MAAaI,WAAWA,CAACC,OAAe,EAAE;IACxC,IAAI,CAACtD,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;MAAE2D,OAAO,EAAED,OAAO;MAAEN,IAAI,EAAE;IAAO,CAAC,CACnC,CAAC;IAEF,MAAMQ,wBAAmC,GAAG,CAC1C;MAAED,OAAO,EAAE,IAAI,CAAChE,UAAU,CAACkE,YAAY;MAAET,IAAI,EAAE;IAAS,CAAC,EACzD,GAAG,IAAI,CAACpD,eAAe,CAAC8D,KAAK,CAAC,CAAC,IAAI,CAACnE,UAAU,CAACoE,mBAAmB,CAAC,CACpE;IAED,MAAM,IAAI,CAACf,QAAQ,CAACY,wBAAwB,EAAE,IAAI,CAACrB,WAAW,EAAEW,KAAK,CAAC;IAEtE,IAAI,CAAC,IAAI,CAACX,WAAW,IAAI,IAAI,CAACA,WAAW,CAACyB,gBAAgB,EAAE;MAC1D,IAAI,CAAC5D,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;QAAE2D,OAAO,EAAE,IAAI,CAAC9D,SAAS;QAAEuD,IAAI,EAAE;MAAY,CAAC,CAC/C,CAAC;IACJ;IACA,IAAI,CAAC,IAAI,CAACb,WAAW,EAAE;MACrB;IACF;IAEA,MAAM0B,SAAS,GAAGxE,aAAa,CAAC,IAAI,CAACI,SAAS,CAAC;IAE/C,KAAK,MAAMqE,QAAQ,IAAID,SAAS,EAAE;MAChC,IAAI,CAAC1B,WAAW,CACb4B,mBAAmB,CAACD,QAAQ,CAAC,CAC7BE,IAAI,CAAEC,YAA2B,IAAK;QACrC,IAAIA,YAAY,EAAE;UAChB,IAAI,CAACjE,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;YAAE2D,OAAO,EAAEU,YAAY;YAAEjB,IAAI,EAAE;UAAY,CAAC,CAC7C,CAAC;QACJ;MACF,CAAC,CAAC;IACN;EACF;EAEOkB,aAAaA,CAACC,KAAa,EAAE;IAClC;IACA;IACA,MAAMC,iBAAiB,GAAG,IAAI,CAACxE,eAAe,CAAC8D,KAAK,CAAC,CAAC,EAAES,KAAK,CAAC;IAE9D,IAAI,CAACnE,sBAAsB,CAACoE,iBAAiB,CAAC;EAChD;EAEQlB,iBAAiBA,CACvBL,QAAmB,EACnBxB,eAAoB,EACpByB,KAAiB,EACjBuB,aAAsB,EACd;IACR,IAAI,CAAChD,eAAe,CAACiD,aAAa,EAAE;MAClC,MAAMtC,KAAK,CAAC,gDAAgD,CAAC;IAC/D;IACA,MAAMuC,QAAQ,GAAG,IAAIvF,QAAQ,CAACqC,eAAe,CAACiD,aAAa,CAAC;IAE5D,MAAME,aAAa,GAAGC,MAAM,CAACC,WAAW,CACtCD,MAAM,CAACE,IAAI,CAACxF,cAAc,CAAC,CACxByF,MAAM,CAAEC,GAAG,IAAKA,GAAG,IAAIxD,eAAe,CAAC,CACvCyD,GAAG,CAAED,GAAG,IAAK,CAACA,GAAG,EAAExD,eAAe,CAACwD,GAAG,CAAC,CAAC,CAC7C,CAAC;IAED,MAAME,MAAM,GAAGR,QAAQ,CAACS,MAAM,CAAC;MAC7BnC,QAAQ;MACRC,KAAK;MACL,GAAGuB,aAAa;MAChB,GAAGG;IACL,CAAC,CAAC;IACF,OAAOO,MAAM;EACf;AACF","ignoreList":[]}

package/lib/module/hooks/natural_language_processing/useLLM.js

@@ -4,7 +4,7 @@ import { useCallback, useEffect, useMemo, useState } from 'react';
 import { LLMController } from '../../controllers/LLMController';
 
 /*
-Hook version of LLMController
+Hook version of LLMModule
 */
 export const useLLM = ({
   modelSource,
@@ -12,19 +12,24 @@ export const useLLM = ({
   tokenizerConfigSource,
   preventLoad = false
 }) => {
+  const [token, setToken] = useState('');
   const [response, setResponse] = useState('');
   const [messageHistory, setMessageHistory] = useState([]);
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
   const [error, setError] = useState(null);
+  const tokenCallback = useCallback(newToken => {
+    setToken(newToken);
+    setResponse(prevResponse => prevResponse + newToken);
+  }, []);
   const model = useMemo(() => new LLMController({
-    responseCallback: setResponse,
+    tokenCallback: tokenCallback,
     messageHistoryCallback: setMessageHistory,
     isReadyCallback: setIsReady,
     isGeneratingCallback: setIsGenerating,
     onDownloadProgressCallback: setDownloadProgress
-  }), []);
+  }), [tokenCallback]);
   useEffect(() => {
     setDownloadProgress(0);
     setError(null);
@@ -54,13 +59,20 @@ export const useLLM = ({
     chatConfig,
     toolsConfig
   }), [model]);
-  const generate = useCallback((messages, tools) => model.generate(messages, tools), [model]);
-  const sendMessage = useCallback(message => model.sendMessage(message), [model]);
+  const generate = useCallback((messages, tools) => {
+    setResponse('');
+    return model.generate(messages, tools);
+  }, [model]);
+  const sendMessage = useCallback(message => {
+    setResponse('');
+    return model.sendMessage(message);
+  }, [model]);
   const deleteMessage = useCallback(index => model.deleteMessage(index), [model]);
   const interrupt = useCallback(() => model.interrupt(), [model]);
   return {
     messageHistory,
     response,
+    token,
     isReady,
     isGenerating,
     downloadProgress,

package/lib/module/hooks/natural_language_processing/useLLM.js.map

@@ -1 +1 @@
-{"version":3,"names":["useCallback","useEffect","useMemo","useState","LLMController","useLLM","modelSource","tokenizerSource","tokenizerConfigSource","preventLoad","response","setResponse","messageHistory","setMessageHistory","isReady","setIsReady","isGenerating","setIsGenerating","downloadProgress","setDownloadProgress","error","setError","model","responseCallback","messageHistoryCallback","isReadyCallback","isGeneratingCallback","onDownloadProgressCallback","load","e","delete","configure","chatConfig","toolsConfig","generate","messages","tools","sendMessage","message","deleteMessage","index","interrupt"],"sourceRoot":"../../../../src","sources":["hooks/natural_language_processing/useLLM.ts"],"mappings":";;AAAA,SAASA,WAAW,EAAEC,SAAS,EAAEC,OAAO,EAAEC,QAAQ,QAAQ,OAAO;AASjE,SAASC,aAAa,QAAQ,iCAAiC;;AAE/D;AACA;AACA;AACA,OAAO,MAAMC,MAAM,GAAGA,CAAC;EACrBC,WAAW;EACXC,eAAe;EACfC,qBAAqB;EACrBC,WAAW,GAAG;AAMhB,CAAC,KAAc;EACb,MAAM,CAACC,QAAQ,EAAEC,WAAW,CAAC,GAAGR,QAAQ,CAAC,EAAE,CAAC;EAC5C,MAAM,CAACS,cAAc,EAAEC,iBAAiB,CAAC,GAAGV,QAAQ,CAAY,EAAE,CAAC;EACnE,MAAM,CAACW,OAAO,EAAEC,UAAU,CAAC,GAAGZ,QAAQ,CAAC,KAAK,CAAC;EAC7C,MAAM,CAACa,YAAY,EAAEC,eAAe,CAAC,GAAGd,QAAQ,CAAC,KAAK,CAAC;EACvD,MAAM,CAACe,gBAAgB,EAAEC,mBAAmB,CAAC,GAAGhB,QAAQ,CAAC,CAAC,CAAC;EAC3D,MAAM,CAACiB,KAAK,EAAEC,QAAQ,CAAC,GAAGlB,QAAQ,CAAM,IAAI,CAAC;EAE7C,MAAMmB,KAAK,GAAGpB,OAAO,CACnB,MACE,IAAIE,aAAa,CAAC;IAChBmB,gBAAgB,EAAEZ,WAAW;IAC7Ba,sBAAsB,EAAEX,iBAAiB;IACzCY,eAAe,EAAEV,UAAU;IAC3BW,oBAAoB,EAAET,eAAe;IACrCU,0BAA0B,EAAER;EAC9B,CAAC,CAAC,EACJ,EACF,CAAC;EAEDlB,SAAS,CAAC,MAAM;IACdkB,mBAAmB,CAAC,CAAC,CAAC;IACtBE,QAAQ,CAAC,IAAI,CAAC;IAEd,IAAI,CAACZ,WAAW,EAAE;MAChB,IAAI;QACF,CAAC,YAAY;UACX,MAAMa,KAAK,CAACM,IAAI,CAAC;YACftB,WAAW;YACXC,eAAe;YACfC;UACF,CAAC,CAAC;QACJ,CAAC,EAAE,CAAC;MACN,CAAC,CAAC,OAAOqB,CAAC,EAAE;QACVR,QAAQ,CAACQ,CAAC,CAAC;MACb;IACF;IAEA,OAAO,MAAM;MACXP,KAAK,CAACQ,MAAM,CAAC,CAAC;IAChB,CAAC;EACH,CAAC,EAAE,CAACxB,WAAW,EAAEC,eAAe,EAAEC,qBAAqB,EAAEC,WAAW,EAAEa,KAAK,CAAC,CAAC;;EAE7E;EACA,MAAMS,SAAS,GAAG/B,WAAW,CAC3B,CAAC;IACCgC,UAAU;IACVC;EAIF,CAAC,KAAKX,KAAK,CAACS,SAAS,CAAC;IAAEC,UAAU;IAAEC;EAAY,CAAC,CAAC,EAClD,CAACX,KAAK,CACR,CAAC;EAED,MAAMY,QAAQ,GAAGlC,WAAW,CAC1B,CAACmC,QAAmB,EAAEC,KAAiB,KAAKd,KAAK,CAACY,QAAQ,CAACC,QAAQ,EAAEC,KAAK,CAAC,EAC3E,CAACd,KAAK,CACR,CAAC;EAED,MAAMe,WAAW,GAAGrC,WAAW,CAC5BsC,OAAe,IAAKhB,KAAK,CAACe,WAAW,CAACC,OAAO,CAAC,EAC/C,CAAChB,KAAK,CACR,CAAC;EAED,MAAMiB,aAAa,GAAGvC,WAAW,CAC9BwC,KAAa,IAAKlB,KAAK,CAACiB,aAAa,CAACC,KAAK,CAAC,EAC7C,CAAClB,KAAK,CACR,CAAC;EACD,MAAMmB,SAAS,GAAGzC,WAAW,CAAC,MAAMsB,KAAK,CAACmB,SAAS,CAAC,CAAC,EAAE,CAACnB,KAAK,CAAC,CAAC;EAE/D,OAAO;IACLV,cAAc;IACdF,QAAQ;IACRI,OAAO;IACPE,YAAY;IACZE,gBAAgB;IAChBE,KAAK;IACLW,SAAS,EAAEA,SAAS;IACpBG,QAAQ,EAAEA,QAAQ;IAClBG,WAAW,EAAEA,WAAW;IACxBE,aAAa,EAAEA,aAAa;IAC5BE,SAAS,EAAEA;EACb,CAAC;AACH,CAAC","ignoreList":[]}
+{"version":3,"names":["useCallback","useEffect","useMemo","useState","LLMController","useLLM","modelSource","tokenizerSource","tokenizerConfigSource","preventLoad","token","setToken","response","setResponse","messageHistory","setMessageHistory","isReady","setIsReady","isGenerating","setIsGenerating","downloadProgress","setDownloadProgress","error","setError","tokenCallback","newToken","prevResponse","model","messageHistoryCallback","isReadyCallback","isGeneratingCallback","onDownloadProgressCallback","load","e","delete","configure","chatConfig","toolsConfig","generate","messages","tools","sendMessage","message","deleteMessage","index","interrupt"],"sourceRoot":"../../../../src","sources":["hooks/natural_language_processing/useLLM.ts"],"mappings":";;AAAA,SAASA,WAAW,EAAEC,SAAS,EAAEC,OAAO,EAAEC,QAAQ,QAAQ,OAAO;AASjE,SAASC,aAAa,QAAQ,iCAAiC;;AAE/D;AACA;AACA;AACA,OAAO,MAAMC,MAAM,GAAGA,CAAC;EACrBC,WAAW;EACXC,eAAe;EACfC,qBAAqB;EACrBC,WAAW,GAAG;AAMhB,CAAC,KAAc;EACb,MAAM,CAACC,KAAK,EAAEC,QAAQ,CAAC,GAAGR,QAAQ,CAAS,EAAE,CAAC;EAC9C,MAAM,CAACS,QAAQ,EAAEC,WAAW,CAAC,GAAGV,QAAQ,CAAS,EAAE,CAAC;EACpD,MAAM,CAACW,cAAc,EAAEC,iBAAiB,CAAC,GAAGZ,QAAQ,CAAY,EAAE,CAAC;EACnE,MAAM,CAACa,OAAO,EAAEC,UAAU,CAAC,GAAGd,QAAQ,CAAC,KAAK,CAAC;EAC7C,MAAM,CAACe,YAAY,EAAEC,eAAe,CAAC,GAAGhB,QAAQ,CAAC,KAAK,CAAC;EACvD,MAAM,CAACiB,gBAAgB,EAAEC,mBAAmB,CAAC,GAAGlB,QAAQ,CAAC,CAAC,CAAC;EAC3D,MAAM,CAACmB,KAAK,EAAEC,QAAQ,CAAC,GAAGpB,QAAQ,CAAM,IAAI,CAAC;EAE7C,MAAMqB,aAAa,GAAGxB,WAAW,CAAEyB,QAAgB,IAAK;IACtDd,QAAQ,CAACc,QAAQ,CAAC;IAClBZ,WAAW,CAAEa,YAAY,IAAKA,YAAY,GAAGD,QAAQ,CAAC;EACxD,CAAC,EAAE,EAAE,CAAC;EAEN,MAAME,KAAK,GAAGzB,OAAO,CACnB,MACE,IAAIE,aAAa,CAAC;IAChBoB,aAAa,EAAEA,aAAa;IAC5BI,sBAAsB,EAAEb,iBAAiB;IACzCc,eAAe,EAAEZ,UAAU;IAC3Ba,oBAAoB,EAAEX,eAAe;IACrCY,0BAA0B,EAAEV;EAC9B,CAAC,CAAC,EACJ,CAACG,aAAa,CAChB,CAAC;EAEDvB,SAAS,CAAC,MAAM;IACdoB,mBAAmB,CAAC,CAAC,CAAC;IACtBE,QAAQ,CAAC,IAAI,CAAC;IAEd,IAAI,CAACd,WAAW,EAAE;MAChB,IAAI;QACF,CAAC,YAAY;UACX,MAAMkB,KAAK,CAACK,IAAI,CAAC;YACf1B,WAAW;YACXC,eAAe;YACfC;UACF,CAAC,CAAC;QACJ,CAAC,EAAE,CAAC;MACN,CAAC,CAAC,OAAOyB,CAAC,EAAE;QACVV,QAAQ,CAACU,CAAC,CAAC;MACb;IACF;IAEA,OAAO,MAAM;MACXN,KAAK,CAACO,MAAM,CAAC,CAAC;IAChB,CAAC;EACH,CAAC,EAAE,CAAC5B,WAAW,EAAEC,eAAe,EAAEC,qBAAqB,EAAEC,WAAW,EAAEkB,KAAK,CAAC,CAAC;;EAE7E;EACA,MAAMQ,SAAS,GAAGnC,WAAW,CAC3B,CAAC;IACCoC,UAAU;IACVC;EAIF,CAAC,KAAKV,KAAK,CAACQ,SAAS,CAAC;IAAEC,UAAU;IAAEC;EAAY,CAAC,CAAC,EAClD,CAACV,KAAK,CACR,CAAC;EAED,MAAMW,QAAQ,GAAGtC,WAAW,CAC1B,CAACuC,QAAmB,EAAEC,KAAiB,KAAK;IAC1C3B,WAAW,CAAC,EAAE,CAAC;IACf,OAAOc,KAAK,CAACW,QAAQ,CAACC,QAAQ,EAAEC,KAAK,CAAC;EACxC,CAAC,EACD,CAACb,KAAK,CACR,CAAC;EAED,MAAMc,WAAW,GAAGzC,WAAW,CAC5B0C,OAAe,IAAK;IACnB7B,WAAW,CAAC,EAAE,CAAC;IACf,OAAOc,KAAK,CAACc,WAAW,CAACC,OAAO,CAAC;EACnC,CAAC,EACD,CAACf,KAAK,CACR,CAAC;EAED,MAAMgB,aAAa,GAAG3C,WAAW,CAC9B4C,KAAa,IAAKjB,KAAK,CAACgB,aAAa,CAACC,KAAK,CAAC,EAC7C,CAACjB,KAAK,CACR,CAAC;EACD,MAAMkB,SAAS,GAAG7C,WAAW,CAAC,MAAM2B,KAAK,CAACkB,SAAS,CAAC,CAAC,EAAE,CAAClB,KAAK,CAAC,CAAC;EAE/D,OAAO;IACLb,cAAc;IACdF,QAAQ;IACRF,KAAK;IACLM,OAAO;IACPE,YAAY;IACZE,gBAAgB;IAChBE,KAAK;IACLa,SAAS,EAAEA,SAAS;IACpBG,QAAQ,EAAEA,QAAQ;IAClBG,WAAW,EAAEA,WAAW;IACxBE,aAAa,EAAEA,aAAa;IAC5BE,SAAS,EAAEA;EACb,CAAC;AACH,CAAC","ignoreList":[]}

package/lib/module/modules/natural_language_processing/LLMModule.js

@@ -7,10 +7,12 @@ export class LLMModule {
     tokenizerSource,
     tokenizerConfigSource,
     onDownloadProgressCallback,
+    tokenCallback,
     responseCallback,
     messageHistoryCallback
   }) {
     this.controller = new LLMController({
+      tokenCallback: tokenCallback,
       responseCallback: responseCallback,
       messageHistoryCallback: messageHistoryCallback,
       onDownloadProgressCallback: onDownloadProgressCallback
@@ -21,6 +23,11 @@ export class LLMModule {
       tokenizerConfigSource
     });
   }
+  static setTokenCallback({
+    tokenCallback
+  }) {
+    this.controller.setTokenCallback(tokenCallback);
+  }
   static configure({
     chatConfig,
     toolsConfig

package/lib/module/modules/natural_language_processing/LLMModule.js.map

@@ -1 +1 @@
-{"version":3,"names":["LLMController","LLMModule","load","modelSource","tokenizerSource","tokenizerConfigSource","onDownloadProgressCallback","responseCallback","messageHistoryCallback","controller","configure","chatConfig","toolsConfig","forward","input","response","generate","messages","tools","sendMessage","message","messageHistory","deleteMessage","index","interrupt","delete"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/LLMModule.ts"],"mappings":";;AAAA,SAASA,aAAa,QAAQ,iCAAiC;AAI/D,OAAO,MAAMC,SAAS,CAAC;EAGrB,aAAaC,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC,qBAAqB;IACrBC,0BAA0B;IAC1BC,gBAAgB;IAChBC;EAQF,CAAC,EAAE;IACD,IAAI,CAACC,UAAU,GAAG,IAAIT,aAAa,CAAC;MAClCO,gBAAgB,EAAEA,gBAAgB;MAClCC,sBAAsB,EAAEA,sBAAsB;MAC9CF,0BAA0B,EAAEA;IAC9B,CAAC,CAAC;IACF,MAAM,IAAI,CAACG,UAAU,CAACP,IAAI,CAAC;MACzBC,WAAW;MACXC,eAAe;MACfC;IACF,CAAC,CAAC;EACJ;EAEA,OAAOK,SAASA,CAAC;IACfC,UAAU;IACVC;EAIF,CAAC,EAAE;IACD,IAAI,CAACH,UAAU,CAACC,SAAS,CAAC;MAAEC,UAAU;MAAEC;IAAY,CAAC,CAAC;EACxD;EAEA,aAAaC,OAAOA,CAACC,KAAa,EAAmB;IACnD,MAAM,IAAI,CAACL,UAAU,CAACI,OAAO,CAACC,KAAK,CAAC;IACpC,OAAO,IAAI,CAACL,UAAU,CAACM,QAAQ;EACjC;EAEA,aAAaC,QAAQA,CACnBC,QAAmB,EACnBC,KAAiB,EACA;IACjB,MAAM,IAAI,CAACT,UAAU,CAACO,QAAQ,CAACC,QAAQ,EAAEC,KAAK,CAAC;IAC/C,OAAO,IAAI,CAACT,UAAU,CAACM,QAAQ;EACjC;EAEA,aAAaI,WAAWA,CAACC,OAAe,EAAsB;IAC5D,MAAM,IAAI,CAACX,UAAU,CAACU,WAAW,CAACC,OAAO,CAAC;IAC1C,OAAO,IAAI,CAACX,UAAU,CAACY,cAAc;EACvC;EAEA,aAAaC,aAAaA,CAACC,KAAa,EAAsB;IAC5D,MAAM,IAAI,CAACd,UAAU,CAACa,aAAa,CAACC,KAAK,CAAC;IAC1C,OAAO,IAAI,CAACd,UAAU,CAACY,cAAc;EACvC;EAEA,OAAOG,SAASA,CAAA,EAAG;IACjB,IAAI,CAACf,UAAU,CAACe,SAAS,CAAC,CAAC;EAC7B;EAEA,OAAOC,MAAMA,CAAA,EAAG;IACd,IAAI,CAAChB,UAAU,CAACgB,MAAM,CAAC,CAAC;EAC1B;AACF","ignoreList":[]}
+{"version":3,"names":["LLMController","LLMModule","load","modelSource","tokenizerSource","tokenizerConfigSource","onDownloadProgressCallback","tokenCallback","responseCallback","messageHistoryCallback","controller","setTokenCallback","configure","chatConfig","toolsConfig","forward","input","response","generate","messages","tools","sendMessage","message","messageHistory","deleteMessage","index","interrupt","delete"],"sourceRoot":"../../../../src","sources":["modules/natural_language_processing/LLMModule.ts"],"mappings":";;AAAA,SAASA,aAAa,QAAQ,iCAAiC;AAI/D,OAAO,MAAMC,SAAS,CAAC;EAGrB,aAAaC,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC,qBAAqB;IACrBC,0BAA0B;IAC1BC,aAAa;IACbC,gBAAgB;IAChBC;EASF,CAAC,EAAE;IACD,IAAI,CAACC,UAAU,GAAG,IAAIV,aAAa,CAAC;MAClCO,aAAa,EAAEA,aAAa;MAC5BC,gBAAgB,EAAEA,gBAAgB;MAClCC,sBAAsB,EAAEA,sBAAsB;MAC9CH,0BAA0B,EAAEA;IAC9B,CAAC,CAAC;IACF,MAAM,IAAI,CAACI,UAAU,CAACR,IAAI,CAAC;MACzBC,WAAW;MACXC,eAAe;MACfC;IACF,CAAC,CAAC;EACJ;EAEA,OAAOM,gBAAgBA,CAAC;IACtBJ;EAGF,CAAC,EAAE;IACD,IAAI,CAACG,UAAU,CAACC,gBAAgB,CAACJ,aAAa,CAAC;EACjD;EAEA,OAAOK,SAASA,CAAC;IACfC,UAAU;IACVC;EAIF,CAAC,EAAE;IACD,IAAI,CAACJ,UAAU,CAACE,SAAS,CAAC;MAAEC,UAAU;MAAEC;IAAY,CAAC,CAAC;EACxD;EAEA,aAAaC,OAAOA,CAACC,KAAa,EAAmB;IACnD,MAAM,IAAI,CAACN,UAAU,CAACK,OAAO,CAACC,KAAK,CAAC;IACpC,OAAO,IAAI,CAACN,UAAU,CAACO,QAAQ;EACjC;EAEA,aAAaC,QAAQA,CACnBC,QAAmB,EACnBC,KAAiB,EACA;IACjB,MAAM,IAAI,CAACV,UAAU,CAACQ,QAAQ,CAACC,QAAQ,EAAEC,KAAK,CAAC;IAC/C,OAAO,IAAI,CAACV,UAAU,CAACO,QAAQ;EACjC;EAEA,aAAaI,WAAWA,CAACC,OAAe,EAAsB;IAC5D,MAAM,IAAI,CAACZ,UAAU,CAACW,WAAW,CAACC,OAAO,CAAC;IAC1C,OAAO,IAAI,CAACZ,UAAU,CAACa,cAAc;EACvC;EAEA,aAAaC,aAAaA,CAACC,KAAa,EAAsB;IAC5D,MAAM,IAAI,CAACf,UAAU,CAACc,aAAa,CAACC,KAAK,CAAC;IAC1C,OAAO,IAAI,CAACf,UAAU,CAACa,cAAc;EACvC;EAEA,OAAOG,SAASA,CAAA,EAAG;IACjB,IAAI,CAAChB,UAAU,CAACgB,SAAS,CAAC,CAAC;EAC7B;EAEA,OAAOC,MAAMA,CAAA,EAAG;IACd,IAAI,CAACjB,UAAU,CAACiB,MAAM,CAAC,CAAC;EAC1B;AACF","ignoreList":[]}

package/lib/module/types/llm.js

@@ -3,5 +3,13 @@
 // usually tool is represented with dictionary (Object), but fields depend on the model
 // unfortunately there's no one standard so it's hard to type it better
 
-export const SPECIAL_TOKENS = ['bos_token', 'eos_token', 'unk_token', 'sep_token', 'pad_token', 'cls_token', 'mask_token'];
+export const SPECIAL_TOKENS = {
+  BOS_TOKEN: 'bos_token',
+  EOS_TOKEN: 'eos_token',
+  UNK_TOKEN: 'unk_token',
+  SEP_TOKEN: 'sep_token',
+  PAD_TOKEN: 'pad_token',
+  CLS_TOKEN: 'cls_token',
+  MASK_TOKEN: 'mask_token'
+};
 //# sourceMappingURL=llm.js.map

package/lib/module/types/llm.js.map

@@ -1 +1 @@
-{"version":3,"names":["SPECIAL_TOKENS"],"sourceRoot":"../../../src","sources":["types/llm.ts"],"mappings":";;AAgCA;AACA;;AAeA,OAAO,MAAMA,cAAc,GAAG,CAC5B,WAAW,EACX,WAAW,EACX,WAAW,EACX,WAAW,EACX,WAAW,EACX,WAAW,EACX,YAAY,CACb","ignoreList":[]}
+{"version":3,"names":["SPECIAL_TOKENS","BOS_TOKEN","EOS_TOKEN","UNK_TOKEN","SEP_TOKEN","PAD_TOKEN","CLS_TOKEN","MASK_TOKEN"],"sourceRoot":"../../../src","sources":["types/llm.ts"],"mappings":";;AAiCA;AACA;;AAeA,OAAO,MAAMA,cAAc,GAAG;EAC5BC,SAAS,EAAE,WAAW;EACtBC,SAAS,EAAE,WAAW;EACtBC,SAAS,EAAE,WAAW;EACtBC,SAAS,EAAE,WAAW;EACtBC,SAAS,EAAE,WAAW;EACtBC,SAAS,EAAE,WAAW;EACtBC,UAAU,EAAE;AACd,CAAC","ignoreList":[]}

package/lib/typescript/controllers/LLMController.d.ts

@@ -10,12 +10,14 @@ export declare class LLMController {
     private _isReady;
     private _isGenerating;
     private _messageHistory;
+    private tokenCallback;
     private responseCallback;
     private messageHistoryCallback;
     private isReadyCallback;
     private isGeneratingCallback;
     private onDownloadProgressCallback;
-    constructor({ responseCallback, messageHistoryCallback, isReadyCallback, isGeneratingCallback, onDownloadProgressCallback, }: {
+    constructor({ tokenCallback, responseCallback, messageHistoryCallback, isReadyCallback, isGeneratingCallback, onDownloadProgressCallback, }: {
+        tokenCallback?: (token: string) => void;
         responseCallback?: (response: string) => void;
         messageHistoryCallback?: (messageHistory: Message[]) => void;
         isReadyCallback?: (isReady: boolean) => void;
@@ -31,6 +33,7 @@ export declare class LLMController {
         tokenizerSource: ResourceSource;
         tokenizerConfigSource: ResourceSource;
     }): Promise<void>;
+    setTokenCallback(tokenCallback: (token: string) => void): void;
     configure({ chatConfig, toolsConfig, }: {
         chatConfig?: Partial<ChatConfig>;
         toolsConfig?: ToolsConfig;

package/lib/typescript/controllers/LLMController.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LLMController.d.ts","sourceRoot":"","sources":["../../../src/controllers/LLMController.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAMjD,OAAO,EACL,UAAU,EACV,OAAO,EACP,OAAO,EAEP,WAAW,EACZ,MAAM,cAAc,CAAC;AAItB,qBAAa,aAAa;IACxB,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,UAAU,CAAmC;IACrD,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAAM;IACvB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAiB;IAGxC,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,sBAAsB,CAAsC;IACpE,OAAO,CAAC,eAAe,CAA6B;IACpD,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,0BAA0B,CAEpB;gBAEF,EACV,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,oBAAoB,EACpB,0BAA0B,GAC3B,EAAE;QACD,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAC9C,sBAAsB,CAAC,EAAE,CAAC,cAAc,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;QAC7D,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;QAC7C,oBAAoB,CAAC,EAAE,CAAC,YAAY,EAAE,OAAO,KAAK,IAAI,CAAC;QACvD,0BAA0B,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;KACjE;IAuBD,IAAW,QAAQ,WAElB;IACD,IAAW,OAAO,YAEjB;IACD,IAAW,YAAY,YAEtB;IACD,IAAW,cAAc,cAExB;IAEY,IAAI,CAAC,EAChB,WAAW,EACX,eAAe,EACf,qBAAqB,GACtB,EAAE;QACD,WAAW,EAAE,cAAc,CAAC;QAC5B,eAAe,EAAE,cAAc,CAAC;QAChC,qBAAqB,EAAE,cAAc,CAAC;KACvC;IAmCM,SAAS,CAAC,EACf,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;IAUM,MAAM;IAcA,OAAO,CAAC,KAAK,EAAE,MAAM;IAkB3B,SAAS;IAIH,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE;IAiC/C,WAAW,CAAC,OAAO,EAAE,MAAM;IA0CjC,aAAa,CAAC,KAAK,EAAE,MAAM;IAQlC,OAAO,CAAC,iBAAiB;CA0B1B"}
+{"version":3,"file":"LLMController.d.ts","sourceRoot":"","sources":["../../../src/controllers/LLMController.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAMjD,OAAO,EACL,UAAU,EACV,OAAO,EACP,OAAO,EAEP,WAAW,EACZ,MAAM,cAAc,CAAC;AAItB,qBAAa,aAAa;IACxB,OAAO,CAAC,YAAY,CAAyB;IAC7C,OAAO,CAAC,UAAU,CAAmC;IACrD,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,eAAe,CAAM;IAC7B,OAAO,CAAC,OAAO,CAAkC;IACjD,OAAO,CAAC,SAAS,CAAM;IACvB,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,eAAe,CAAiB;IAGxC,OAAO,CAAC,aAAa,CAA0B;IAC/C,OAAO,CAAC,gBAAgB,CAA6B;IACrD,OAAO,CAAC,sBAAsB,CAAsC;IACpE,OAAO,CAAC,eAAe,CAA6B;IACpD,OAAO,CAAC,oBAAoB,CAAkC;IAC9D,OAAO,CAAC,0BAA0B,CAEpB;gBAEF,EACV,aAAa,EACb,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EACf,oBAAoB,EACpB,0BAA0B,GAC3B,EAAE;QACD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAC9C,sBAAsB,CAAC,EAAE,CAAC,cAAc,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;QAC7D,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,KAAK,IAAI,CAAC;QAC7C,oBAAoB,CAAC,EAAE,CAAC,YAAY,EAAE,OAAO,KAAK,IAAI,CAAC;QACvD,0BAA0B,CAAC,EAAE,CAAC,gBAAgB,EAAE,MAAM,KAAK,IAAI,CAAC;KACjE;IA+BD,IAAW,QAAQ,WAElB;IACD,IAAW,OAAO,YAEjB;IACD,IAAW,YAAY,YAEtB;IACD,IAAW,cAAc,cAExB;IAEY,IAAI,CAAC,EAChB,WAAW,EACX,eAAe,EACf,qBAAqB,GACtB,EAAE;QACD,WAAW,EAAE,cAAc,CAAC;QAC5B,eAAe,EAAE,cAAc,CAAC;QAChC,qBAAqB,EAAE,cAAc,CAAC;KACvC;IA2CM,gBAAgB,CAAC,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI;IAIvD,SAAS,CAAC,EACf,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;IAUM,MAAM;IAcA,OAAO,CAAC,KAAK,EAAE,MAAM;IAkB3B,SAAS;IAIH,QAAQ,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE;IAwB/C,WAAW,CAAC,OAAO,EAAE,MAAM;IAuCjC,aAAa,CAAC,KAAK,EAAE,MAAM;IAQlC,OAAO,CAAC,iBAAiB;CAyB1B"}

package/lib/typescript/hooks/natural_language_processing/useLLM.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"useLLM.d.ts","sourceRoot":"","sources":["../../../../src/hooks/natural_language_processing/useLLM.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAGL,OAAO,EAGR,MAAM,iBAAiB,CAAC;AAMzB,eAAO,MAAM,MAAM,GAAI,uEAKpB;IACD,WAAW,EAAE,cAAc,CAAC;IAC5B,eAAe,EAAE,cAAc,CAAC;IAChC,qBAAqB,EAAE,cAAc,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,KAAG,OAoFH,CAAC"}
+{"version":3,"file":"useLLM.d.ts","sourceRoot":"","sources":["../../../../src/hooks/natural_language_processing/useLLM.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAGL,OAAO,EAGR,MAAM,iBAAiB,CAAC;AAMzB,eAAO,MAAM,MAAM,GAAI,uEAKpB;IACD,WAAW,EAAE,cAAc,CAAC;IAC5B,eAAe,EAAE,cAAc,CAAC;IAChC,qBAAqB,EAAE,cAAc,CAAC;IACtC,WAAW,CAAC,EAAE,OAAO,CAAC;CACvB,KAAG,OAiGH,CAAC"}

package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts

@@ -3,14 +3,18 @@ import { ResourceSource } from '../../types/common';
 import { ChatConfig, LLMTool, Message, ToolsConfig } from '../../types/llm';
 export declare class LLMModule {
     static controller: LLMController;
-    static load({ modelSource, tokenizerSource, tokenizerConfigSource, onDownloadProgressCallback, responseCallback, messageHistoryCallback, }: {
+    static load({ modelSource, tokenizerSource, tokenizerConfigSource, onDownloadProgressCallback, tokenCallback, responseCallback, messageHistoryCallback, }: {
         modelSource: ResourceSource;
         tokenizerSource: ResourceSource;
         tokenizerConfigSource: ResourceSource;
         onDownloadProgressCallback?: (_downloadProgress: number) => void;
+        tokenCallback?: (token: string) => void;
         responseCallback?: (response: string) => void;
         messageHistoryCallback?: (messageHistory: Message[]) => void;
     }): Promise<void>;
+    static setTokenCallback({ tokenCallback, }: {
+        tokenCallback: (token: string) => void;
+    }): void;
     static configure({ chatConfig, toolsConfig, }: {
         chatConfig?: Partial<ChatConfig>;
         toolsConfig?: ToolsConfig;

package/lib/typescript/modules/natural_language_processing/LLMModule.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"LLMModule.d.ts","sourceRoot":"","sources":["../../../../src/modules/natural_language_processing/LLMModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE5E,qBAAa,SAAS;IACpB,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC;WAEpB,IAAI,CAAC,EAChB,WAAW,EACX,eAAe,EACf,qBAAqB,EACrB,0BAA0B,EAC1B,gBAAgB,EAChB,sBAAsB,GACvB,EAAE;QACD,WAAW,EAAE,cAAc,CAAC;QAC5B,eAAe,EAAE,cAAc,CAAC;QAChC,qBAAqB,EAAE,cAAc,CAAC;QACtC,0BAA0B,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,KAAK,IAAI,CAAC;QACjE,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAC9C,sBAAsB,CAAC,EAAE,CAAC,cAAc,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;KAC9D;IAaD,MAAM,CAAC,SAAS,CAAC,EACf,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;WAIY,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;WAKvC,QAAQ,CACnB,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,GAChB,OAAO,CAAC,MAAM,CAAC;WAKL,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;WAKhD,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAK7D,MAAM,CAAC,SAAS;IAIhB,MAAM,CAAC,MAAM;CAGd"}
+{"version":3,"file":"LLMModule.d.ts","sourceRoot":"","sources":["../../../../src/modules/natural_language_processing/LLMModule.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE5E,qBAAa,SAAS;IACpB,MAAM,CAAC,UAAU,EAAE,aAAa,CAAC;WAEpB,IAAI,CAAC,EAChB,WAAW,EACX,eAAe,EACf,qBAAqB,EACrB,0BAA0B,EAC1B,aAAa,EACb,gBAAgB,EAChB,sBAAsB,GACvB,EAAE;QACD,WAAW,EAAE,cAAc,CAAC;QAC5B,eAAe,EAAE,cAAc,CAAC;QAChC,qBAAqB,EAAE,cAAc,CAAC;QACtC,0BAA0B,CAAC,EAAE,CAAC,iBAAiB,EAAE,MAAM,KAAK,IAAI,CAAC;QACjE,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxC,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAC9C,sBAAsB,CAAC,EAAE,CAAC,cAAc,EAAE,OAAO,EAAE,KAAK,IAAI,CAAC;KAC9D;IAcD,MAAM,CAAC,gBAAgB,CAAC,EACtB,aAAa,GACd,EAAE;QACD,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACxC;IAID,MAAM,CAAC,SAAS,CAAC,EACf,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B;WAIY,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;WAKvC,QAAQ,CACnB,QAAQ,EAAE,OAAO,EAAE,EACnB,KAAK,CAAC,EAAE,OAAO,EAAE,GAChB,OAAO,CAAC,MAAM,CAAC;WAKL,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;WAKhD,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;IAK7D,MAAM,CAAC,SAAS;IAIhB,MAAM,CAAC,MAAM;CAGd"}

package/lib/typescript/types/llm.d.ts

@@ -1,6 +1,7 @@
 export interface LLMType {
     messageHistory: Message[];
     response: string;
+    token: string;
     isReady: boolean;
     isGenerating: boolean;
     downloadProgress: number;
@@ -34,5 +35,13 @@ export interface ToolsConfig {
     executeToolCallback: (call: ToolCall) => Promise<string | null>;
     displayToolCalls?: boolean;
 }
-export declare const SPECIAL_TOKENS: string[];
+export declare const SPECIAL_TOKENS: {
+    BOS_TOKEN: string;
+    EOS_TOKEN: string;
+    UNK_TOKEN: string;
+    SEP_TOKEN: string;
+    PAD_TOKEN: string;
+    CLS_TOKEN: string;
+    MASK_TOKEN: string;
+};
 //# sourceMappingURL=llm.d.ts.map

package/lib/typescript/types/llm.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../../src/types/llm.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,OAAO,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,CAAC,EACV,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE1D,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B,MAAM,WAAW,UAAU;IACzB,qBAAqB,EAAE,OAAO,EAAE,CAAC;IACjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,mBAAmB,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,cAAc,UAQ1B,CAAC"}
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../../src/types/llm.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,OAAO,EAAE,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,CAAC,EACV,UAAU,EACV,WAAW,GACZ,EAAE;QACD,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QACjC,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,QAAQ,EAAE,CAAC,QAAQ,EAAE,OAAO,EAAE,EAAE,KAAK,CAAC,EAAE,OAAO,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACpE,WAAW,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,aAAa,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACvC,SAAS,EAAE,MAAM,IAAI,CAAC;CACvB;AAED,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,WAAW,GAAG,QAAQ,CAAC;AAE1D,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAID,MAAM,MAAM,OAAO,GAAG,MAAM,CAAC;AAE7B,MAAM,WAAW,UAAU;IACzB,qBAAqB,EAAE,OAAO,EAAE,CAAC;IACjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,mBAAmB,EAAE,CAAC,IAAI,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAChE,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,cAAc;;;;;;;;CAQ1B,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "react-native-executorch",
-  "version": "0.4.4",
+  "version": "0.4.6",
   "description": "An easy way to run AI models in react native with ExecuTorch",
   "source": "./src/index.ts",
   "main": "./lib/commonjs/index.js",

package/src/controllers/LLMController.ts

@@ -27,6 +27,7 @@ export class LLMController {
   private _messageHistory: Message[] = [];
 
   // User callbacks
+  private tokenCallback: (token: string) => void;
   private responseCallback: (response: string) => void;
   private messageHistoryCallback: (messageHistory: Message[]) => void;
   private isReadyCallback: (isReady: boolean) => void;
@@ -36,18 +37,28 @@ export class LLMController {
     | undefined;
 
   constructor({
+    tokenCallback,
     responseCallback,
     messageHistoryCallback,
     isReadyCallback,
     isGeneratingCallback,
     onDownloadProgressCallback,
   }: {
+    tokenCallback?: (token: string) => void;
     responseCallback?: (response: string) => void;
     messageHistoryCallback?: (messageHistory: Message[]) => void;
     isReadyCallback?: (isReady: boolean) => void;
     isGeneratingCallback?: (isGenerating: boolean) => void;
     onDownloadProgressCallback?: (downloadProgress: number) => void;
   }) {
+    if (responseCallback !== undefined) {
+      console.warn(
+        'Passing response callback is deprecated and will be removed in 0.6.0'
+      );
+    }
+    this.tokenCallback = (token) => {
+      tokenCallback?.(token);
+    };
     this.responseCallback = (response) => {
       this._response = response;
       responseCallback?.(response);
@@ -114,10 +125,18 @@ export class LLMController {
 
       await this.nativeModule.loadLLM(modelFileUri, tokenizerFileUri);
       this.isReadyCallback(true);
-      this.onToken = this.nativeModule.onToken((data: string | undefined) => {
-        if (!data) {
+      this.onToken = this.nativeModule.onToken((data: string) => {
+        if (
+          !data ||
+          (SPECIAL_TOKENS.EOS_TOKEN in this.tokenizerConfig &&
+            data === this.tokenizerConfig.eos_token) ||
+          (SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig &&
+            data === this.tokenizerConfig.pad_token)
+        ) {
           return;
         }
+
+        this.tokenCallback(data);
         this.responseCallback(this._response + data);
       });
     } catch (e) {
@@ -126,6 +145,10 @@ export class LLMController {
     }
   }
 
+  public setTokenCallback(tokenCallback: (token: string) => void) {
+    this.tokenCallback = tokenCallback;
+  }
+
   public configure({
     chatConfig,
     toolsConfig,
@@ -200,15 +223,6 @@ export class LLMController {
     );
 
     await this.forward(renderedChat);
-
-    if (!this._response) {
-      return;
-    }
-
-    const cleanedResponse = this._response
-      .replaceAll(this.tokenizerConfig.eos_token, '')
-      .replaceAll(this.tokenizerConfig.pad_token, '');
-    this.responseCallback(cleanedResponse);
   }
 
   public async sendMessage(message: string) {
@@ -225,9 +239,6 @@ export class LLMController {
     await this.generate(messageHistoryWithPrompt, this.toolsConfig?.tools);
 
     if (!this.toolsConfig || this.toolsConfig.displayToolCalls) {
-      this.responseCallback(
-        this._response.replace(this.tokenizerConfig.eos_token, '')
-      );
       this.messageHistoryCallback([
         ...this._messageHistory,
         { content: this._response, role: 'assistant' },
@@ -273,10 +284,9 @@ export class LLMController {
     const template = new Template(tokenizerConfig.chat_template);
 
     const specialTokens = Object.fromEntries(
-      SPECIAL_TOKENS.filter((key) => key in tokenizerConfig).map((key) => [
-        key,
-        tokenizerConfig[key],
-      ])
+      Object.keys(SPECIAL_TOKENS)
+        .filter((key) => key in tokenizerConfig)
+        .map((key) => [key, tokenizerConfig[key]])
     );
 
     const result = template.render({

package/src/hooks/natural_language_processing/useLLM.ts

@@ -10,7 +10,7 @@ import {
 import { LLMController } from '../../controllers/LLMController';
 
 /*
-Hook version of LLMController
+Hook version of LLMModule
 */
 export const useLLM = ({
   modelSource,
@@ -23,23 +23,29 @@ export const useLLM = ({
   tokenizerConfigSource: ResourceSource;
   preventLoad?: boolean;
 }): LLMType => {
-  const [response, setResponse] = useState('');
+  const [token, setToken] = useState<string>('');
+  const [response, setResponse] = useState<string>('');
   const [messageHistory, setMessageHistory] = useState<Message[]>([]);
   const [isReady, setIsReady] = useState(false);
   const [isGenerating, setIsGenerating] = useState(false);
   const [downloadProgress, setDownloadProgress] = useState(0);
   const [error, setError] = useState<any>(null);
 
+  const tokenCallback = useCallback((newToken: string) => {
+    setToken(newToken);
+    setResponse((prevResponse) => prevResponse + newToken);
+  }, []);
+
   const model = useMemo(
     () =>
       new LLMController({
-        responseCallback: setResponse,
+        tokenCallback: tokenCallback,
         messageHistoryCallback: setMessageHistory,
         isReadyCallback: setIsReady,
         isGeneratingCallback: setIsGenerating,
         onDownloadProgressCallback: setDownloadProgress,
       }),
-    []
+    [tokenCallback]
   );
 
   useEffect(() => {
@@ -78,12 +84,18 @@ export const useLLM = ({
   );
 
   const generate = useCallback(
-    (messages: Message[], tools?: LLMTool[]) => model.generate(messages, tools),
+    (messages: Message[], tools?: LLMTool[]) => {
+      setResponse('');
+      return model.generate(messages, tools);
+    },
     [model]
   );
 
   const sendMessage = useCallback(
-    (message: string) => model.sendMessage(message),
+    (message: string) => {
+      setResponse('');
+      return model.sendMessage(message);
+    },
     [model]
   );
 
@@ -96,6 +108,7 @@ export const useLLM = ({
   return {
     messageHistory,
     response,
+    token,
     isReady,
     isGenerating,
     downloadProgress,

package/src/modules/natural_language_processing/LLMModule.ts

@@ -10,6 +10,7 @@ export class LLMModule {
     tokenizerSource,
     tokenizerConfigSource,
     onDownloadProgressCallback,
+    tokenCallback,
     responseCallback,
     messageHistoryCallback,
   }: {
@@ -17,10 +18,12 @@ export class LLMModule {
     tokenizerSource: ResourceSource;
     tokenizerConfigSource: ResourceSource;
     onDownloadProgressCallback?: (_downloadProgress: number) => void;
+    tokenCallback?: (token: string) => void;
     responseCallback?: (response: string) => void;
     messageHistoryCallback?: (messageHistory: Message[]) => void;
   }) {
     this.controller = new LLMController({
+      tokenCallback: tokenCallback,
       responseCallback: responseCallback,
       messageHistoryCallback: messageHistoryCallback,
       onDownloadProgressCallback: onDownloadProgressCallback,
@@ -32,6 +35,14 @@ export class LLMModule {
     });
   }
 
+  static setTokenCallback({
+    tokenCallback,
+  }: {
+    tokenCallback: (token: string) => void;
+  }) {
+    this.controller.setTokenCallback(tokenCallback);
+  }
+
   static configure({
     chatConfig,
     toolsConfig,

package/src/types/llm.ts

@@ -1,6 +1,7 @@
 export interface LLMType {
   messageHistory: Message[];
   response: string;
+  token: string;
   isReady: boolean;
   isGenerating: boolean;
   downloadProgress: number;
@@ -46,12 +47,12 @@ export interface ToolsConfig {
   displayToolCalls?: boolean;
 }
 
-export const SPECIAL_TOKENS = [
-  'bos_token',
-  'eos_token',
-  'unk_token',
-  'sep_token',
-  'pad_token',
-  'cls_token',
-  'mask_token',
-];
+export const SPECIAL_TOKENS = {
+  BOS_TOKEN: 'bos_token',
+  EOS_TOKEN: 'eos_token',
+  UNK_TOKEN: 'unk_token',
+  SEP_TOKEN: 'sep_token',
+  PAD_TOKEN: 'pad_token',
+  CLS_TOKEN: 'cls_token',
+  MASK_TOKEN: 'mask_token',
+};