react-native-executorch 0.5.5 → 0.5.7

package/common/rnexecutorch/threads/HighPerformanceThreadPool.h

@@ -0,0 +1,364 @@
+// HighPerformanceThreadPool.h
+#pragma once
+
+#include <algorithm>
+#include <atomic>
+#include <chrono>
+#include <condition_variable>
+#include <fstream>
+#include <functional>
+#include <future>
+#include <memory>
+#include <mutex>
+#include <pthread.h>
+#include <queue>
+#include <ranges>
+#include <sched.h>
+#include <sys/resource.h>
+#include <thread>
+#include <unistd.h>
+#include <vector>
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <rnexecutorch/Log.h>
+
+#ifdef __APPLE__
+#include <sys/syscall.h>
+#endif
+
+#ifdef __ANDROID__
+#include <sys/types.h>
+#endif
+
+namespace rnexecutorch::threads {
+
+enum class Priority { LOW, NORMAL, HIGH, REALTIME };
+
+struct ThreadConfig {
+  bool pinToPerformanceCores{true};
+  std::string namePrefix{"RN_ET_Worker"};
+};
+
+class HighPerformanceThreadPool {
+public:
+  explicit HighPerformanceThreadPool(size_t numThreads = 1,
+                                     ThreadConfig cfg = ThreadConfig())
+      : config(std::move(cfg)) {
+
+#ifdef __ANDROID__
+    detectCPUTopology();
+    numThreads = std::min(numThreads, performanceCores.size());
+#endif
+
+    for (size_t i = 0; i < numThreads; i++) {
+      workers.emplace_back(&HighPerformanceThreadPool::workerThread, this, i);
+    }
+
+    log(LOG_LEVEL::Debug, "Thread pool initialized with", numThreads,
+        "workers.");
+  }
+
+  ~HighPerformanceThreadPool() { shutdown(); }
+
+  // Submit a task and get a future for the result
+  template <typename Func, typename... Args>
+  auto submit(Func &&func, Args &&...args)
+      -> std::future<decltype(func(args...))> {
+    return submitWithPriority(Priority::NORMAL, std::forward<Func>(func),
+                              std::forward<Args>(args)...);
+  }
+
+  // Submit a task with specific priority
+  template <typename Func, typename... Args>
+  auto submitWithPriority(Priority priority, Func &&func, Args &&...args)
+      -> std::future<decltype(func(args...))> {
+
+    using ReturnType = decltype(func(args...));
+
+    // Create a packaged task
+    auto boundFunc =
+        std::bind(std::forward<Func>(func), std::forward<Args>(args)...);
+    auto task = std::make_unique<Task<decltype(boundFunc), ReturnType>>(
+        std::move(boundFunc));
+    auto future = task->getFuture();
+
+    // Add to queue
+    {
+      std::scoped_lock lock(queueMutex);
+
+      if (!running) {
+        throw std::runtime_error("Thread pool is shutting down");
+      }
+
+      WorkItem item(std::move(task), priority,
+                    std::chrono::steady_clock::now());
+
+      taskQueue.push(std::move(item));
+    }
+
+    condition.notify_one();
+    return future;
+  }
+
+  // Execute a task and wait for result
+  template <typename Func, typename... Args>
+  auto execute(Func &&func, Args &&...args) -> decltype(func(args...)) {
+    auto future = submit(std::forward<Func>(func), std::forward<Args>(args)...);
+    return future.get();
+  }
+
+  // Fire and forget task
+  template <typename Func, typename... Args>
+  void submitDetached(Func &&func, Args &&...args) {
+    submit(std::forward<Func>(func), std::forward<Args>(args)...);
+    // Future is destroyed, task still runs
+  }
+
+  void shutdown() {
+    if (!running.exchange(false)) {
+      return;
+    }
+
+    condition.notify_all();
+
+    for (auto &worker : workers) {
+      if (worker.joinable()) {
+        worker.join();
+      }
+    }
+  }
+
+private:
+  // Task wrapper that can hold any callable
+  class ITask {
+  public:
+    virtual ~ITask() = default;
+    virtual void execute() = 0;
+  };
+
+  template <typename Func, typename Result> class Task : public ITask {
+  public:
+    Task(Func &&f) : func(std::forward<Func>(f)) {}
+
+    void execute() override {
+      try {
+        if constexpr (std::is_void_v<Result>) {
+          func();
+          promise.set_value();
+        } else {
+          promise.set_value(func());
+        }
+      } catch (...) {
+        promise.set_exception(std::current_exception());
+      }
+    }
+
+    std::future<Result> getFuture() { return promise.get_future(); }
+
+  private:
+    Func func;
+    std::promise<Result> promise;
+  };
+
+  class WorkItem {
+  public:
+    WorkItem() = default;
+    WorkItem(std::unique_ptr<ITask> task, Priority priority,
+             std::chrono::steady_clock::time_point enqueueTime)
+        : task(std::move(task)), priority(priority), enqueueTime(enqueueTime) {}
+
+    std::unique_ptr<ITask> task;
+
+    bool operator<(const WorkItem &other) const {
+      return priority != other.priority ? priority < other.priority
+                                        : enqueueTime > other.enqueueTime;
+    }
+
+  private:
+    Priority priority;
+    std::chrono::steady_clock::time_point enqueueTime;
+  };
+
+  // Thread pool state
+  std::vector<std::thread> workers;
+  std::priority_queue<WorkItem> taskQueue;
+  std::mutex queueMutex;
+  std::condition_variable condition;
+  std::atomic<bool> running{true};
+  std::atomic<size_t> activeWorkers{0};
+  std::atomic<size_t> totalTasksProcessed{0};
+
+#ifdef __ANDROID__
+  // Performance cores
+  std::vector<int32_t> performanceCores;
+  std::vector<int32_t> efficiencyCores;
+#endif
+
+  // Configuration
+  ThreadConfig config;
+
+  void detectCPUTopology() {
+#ifdef __ANDROID__
+    struct CoreInfo {
+      int32_t id;
+      int64_t maxFreq;
+    };
+
+    std::vector<CoreInfo> cores;
+    const auto numOfCores = std::thread::hardware_concurrency();
+
+    for (int32_t i = 0; std::cmp_less(i, numOfCores); ++i) {
+      std::string path = "/sys/devices/system/cpu/cpu" + std::to_string(i) +
+                         "/cpufreq/cpuinfo_max_freq";
+      std::ifstream file(path);
+      if (!file.good()) {
+        break;
+      }
+
+      CoreInfo info;
+      info.id = i;
+      file >> info.maxFreq;
+      cores.push_back(info);
+    }
+
+    if (cores.empty()) {
+      log(LOG_LEVEL::Debug, "Could not detect CPU topology");
+      return;
+    }
+
+    // Sort by frequency
+    std::ranges::sort(cores, [](const CoreInfo &a, const CoreInfo &b) {
+      return a.maxFreq > b.maxFreq;
+    });
+
+    // Classify cores
+    const auto numOfPerfCores =
+        ::executorch::extension::cpuinfo::get_num_performant_cores();
+
+    constexpr float kKiloToGigaRatio = 1e6;
+    for (int32_t i = 0; i < cores.size(); ++i) {
+      if (i < numOfPerfCores) {
+        performanceCores.push_back(cores[i].id);
+        log(LOG_LEVEL::Debug, "Performance core:", cores[i].id, "(",
+            cores[i].maxFreq / kKiloToGigaRatio, "GHz)");
+      } else {
+        efficiencyCores.push_back(cores[i].id);
+        log(LOG_LEVEL::Debug, "Efficiency core:", cores[i].id, "(",
+            cores[i].maxFreq / kKiloToGigaRatio, "GHz)");
+      }
+    }
+#endif
+  }
+
+#ifdef __ANDROID__
+  inline uint64_t getCurrentThreadId() { return gettid(); }
+#endif
+
+  inline void setCurrentThreadName(const std::string &name) {
+#ifdef __ANDROID__
+    pthread_setname_np(pthread_self(), name.c_str());
+#elif defined(__APPLE__)
+    pthread_setname_np(name.c_str());
+#endif
+  }
+
+  void configureThread(uint32_t workerIndex) {
+    std::string threadName = config.namePrefix + std::to_string(workerIndex);
+    setCurrentThreadName(threadName.c_str());
+
+#ifdef __ANDROID__
+    if (config.pinToPerformanceCores && !performanceCores.empty()) {
+      setCPUAffinity();
+    }
+#endif
+
+    setThreadPriority();
+
+    log(LOG_LEVEL::Debug, "Worker", workerIndex,
+        "configured:", threadName.c_str());
+  }
+
+  void setCPUAffinity() {
+    // AFAIK it is not possible on iOS
+#ifdef __ANDROID__
+    if (performanceCores.empty()) {
+      log(LOG_LEVEL::Error, "No cores specified for affinity setting");
+      return;
+    }
+
+    cpu_set_t cpuset;
+    CPU_ZERO(&cpuset);
+
+    for (int32_t core : performanceCores) {
+      CPU_SET(core, &cpuset);
+    }
+
+    pid_t tid = getCurrentThreadId();
+    log(LOG_LEVEL::Debug, "Thread id", tid);
+    if (sched_setaffinity(tid, sizeof(cpuset), &cpuset) == 0) {
+      log(LOG_LEVEL::Debug, "Thread pinned to cores:", performanceCores);
+    } else {
+      log(LOG_LEVEL::Debug, "Failed to set CPU affinity (error:", errno,
+          "). Continuing without affinity.");
+    }
+#endif
+  }
+
+  void setThreadPriority() {
+    // pthread_setschedparam doesn't work on android because permissions reasons
+    // and in general does not provide visible improvements on iOS
+
+    // Set nice value as fallback or additional priority boost
+    constexpr int nice_value = 0;
+    if (setpriority(PRIO_PROCESS, 0, nice_value) != 0) {
+      log(LOG_LEVEL::Debug, "Failed to set nice value");
+    } else {
+      log(LOG_LEVEL::Debug, "Set nice value", nice_value);
+    }
+  }
+
+  void processTask(const WorkItem &item) {
+    activeWorkers++;
+
+    try {
+      item.task->execute();
+    } catch (const std::exception &e) {
+      log(LOG_LEVEL::Error, "Task failed:", e.what());
+      activeWorkers--;
+      throw;
+    }
+
+    activeWorkers--;
+    totalTasksProcessed++;
+  }
+
+  void workerThread(int workerIndex) {
+    configureThread(workerIndex);
+
+    while (running) {
+      WorkItem item;
+
+      {
+        std::unique_lock<std::mutex> lock(queueMutex);
+        condition.wait(lock, [this] { return !taskQueue.empty() || !running; });
+
+        if (!running && taskQueue.empty()) {
+          break;
+        }
+
+        if (!taskQueue.empty()) {
+          item = std::move(const_cast<WorkItem &>(taskQueue.top()));
+          taskQueue.pop();
+        } else {
+          continue;
+        }
+      }
+
+      processTask(item);
+    }
+
+    log(LOG_LEVEL::Debug, "Worker", workerIndex, "shutting down");
+  }
+};
+
+} // namespace rnexecutorch::threads

package/common/rnexecutorch/threads/utils/ThreadUtils.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include <executorch/extension/threadpool/cpuinfo_utils.h>
+#include <executorch/extension/threadpool/threadpool.h>
+#include <rnexecutorch/Log.h>
+
+namespace rnexecutorch::threads::utils {
+
+void unsafeSetupThreadPool(uint32_t num_of_cores = 0) {
+  auto num_of_perf_cores =
+      ::executorch::extension::cpuinfo::get_num_performant_cores();
+  log(LOG_LEVEL::Info, "Detected ", num_of_perf_cores, " performant cores");
+  // setting num_of_cores to floor(num_of_perf_cores / 2) + 1) because
+  // depending on cpu arch as when possible we want to leave at least 2
+  // performant cores for other tasks (setting more actually results in drop
+  // of performance). For older devices (i.e. samsung s22) resolves to 3
+  // cores, and for newer ones (like OnePlus 12) resolves to 4, which when
+  // benchmarked gives highest throughput. For iPhones they usually have 2
+  // performance cores
+  auto _num_of_cores = num_of_cores
+                           ? num_of_cores
+                           : static_cast<uint32_t>(num_of_perf_cores / 2) + 1;
+  const auto threadpool = ::executorch::extension::threadpool::get_threadpool();
+  threadpool->_unsafe_reset_threadpool(_num_of_cores);
+  log(LOG_LEVEL::Info, "Configuring xnnpack for",
+      threadpool->get_thread_count(), "threads");
+}
+
+} // namespace rnexecutorch::threads::utils

package/common/runner/runner.cpp

@@ -222,9 +222,6 @@ Error Runner::generate(const std::string &prompt,
   RUNNER_ET_LOG(warmup, "RSS after prompt prefill: %f MiB (0 if unsupported)",
                 llm::get_rss_bytes() / 1024.0 / 1024.0);
 
-  if (cur_decoded != "�") {
-    wrapped_callback(cur_decoded);
-  }
   // start the main loop
   prompt_tokens_uint64.push_back(cur_token);
   int64_t num_generated_tokens = ET_UNWRAP(text_token_generator_->generate(
@@ -275,4 +272,13 @@ void Runner::stop() {
     ET_LOG(Error, "Token generator is not loaded, cannot stop");
   }
 }
+
+void Runner::set_count_interval(size_t count_interval) {
+  text_token_generator_->set_count_interval(count_interval);
+}
+
+void Runner::set_time_interval(size_t time_interval) {
+  text_token_generator_->set_time_interval(time_interval);
+}
+
 } // namespace example

package/common/runner/runner.h

@@ -43,8 +43,12 @@ public:
                stats_callback = {},
            bool echo = true, bool warming = false);
   ::executorch::runtime::Error warmup(const std::string &prompt);
+  void set_count_interval(size_t count_interval);
+  void set_time_interval(size_t time_interval);
   void stop();
 
+  ::executorch::extension::llm::Stats stats_;
+
 private:
   float temperature_;
   bool shouldStop_{false};
@@ -59,9 +63,6 @@ private:
   std::unique_ptr<::executorch::extension::llm::TextPrefiller> text_prefiller_;
   std::unique_ptr<::executorch::extension::llm::TextTokenGenerator>
       text_token_generator_;
-
-  // stats
-  ::executorch::extension::llm::Stats stats_;
 };
 
 } // namespace example

package/common/runner/text_token_generator.h

@@ -11,6 +11,7 @@
 
 #include "stats.h"
 #include "text_decoder_runner.h"
+#include <chrono>
 #include <executorch/extension/tensor/tensor.h>
 #include <iostream>
 #include <tokenizers-cpp/tokenizers_cpp.h>
@@ -27,7 +28,7 @@ public:
                      Stats *stats)
       : tokenizer_(tokenizer), text_decoder_runner_(text_decoder_runner),
         eos_ids_(std::move(eos_ids)), use_kv_cache_(use_kv_cache),
-        stats_(stats) {}
+        stats_(stats), timestamp_(std::chrono::high_resolution_clock::now()) {}
 
   /**
    * Token generation loop.
@@ -55,12 +56,8 @@ public:
     uint64_t prev_token;
     // cache to keep tokens if they were decoded into illegal character
     std::vector<int32_t> token_cache;
-    // if first token after prefill was part of multi-token character we need to
-    // add this to cache here
-    if (tokenizer_->Decode(
-            std::vector<int32_t>{static_cast<int32_t>(cur_token)}) == "�") {
-      token_cache.push_back(static_cast<int32_t>(cur_token));
-    }
+    // add first token after prefill to cache here
+    token_cache.push_back(static_cast<int32_t>(cur_token));
 
     if (use_kv_cache_) {
       // hard code these to size 1 as kv cache is locked to static size right
@@ -79,7 +76,7 @@ public:
         from_blob(&pos, {1}, executorch::aten::ScalarType::Long);
 
     should_stop_ = false;
-
+    timestamp_ = std::chrono::high_resolution_clock::now();
     // Generate our tokens
     while (pos < seq_len - 1) {
       // Run the model
@@ -112,9 +109,19 @@ public:
       token_cache.push_back(static_cast<int32_t>(cur_token));
       const std::string cache_decoded = tokenizer_->Decode(token_cache);
 
-      if (cache_decoded != "�" && cache_decoded != " �") {
+      const auto timeIntervalElapsed =
+          std::chrono::duration_cast<std::chrono::milliseconds>(
+              std::chrono::high_resolution_clock::now() - timestamp_) >
+          time_interval_;
+      const auto countIntervalElapsed = token_cache.size() > count_interval_;
+      const auto eos_reached = eos_ids_->contains(cur_token);
+
+      if (!cache_decoded.ends_with("�") &&
+          (countIntervalElapsed || timeIntervalElapsed || should_stop_ ||
+           eos_reached)) {
         token_callback(cache_decoded);
         token_cache.clear();
+        timestamp_ = std::chrono::high_resolution_clock::now();
       }
 
       if (should_stop_) {
@@ -122,7 +129,7 @@ public:
       }
 
       // data-dependent terminating condition: we have n_eos_ number of EOS
-      if (eos_ids_->find(cur_token) != eos_ids_->end()) {
+      if (eos_reached) {
         printf("\n");
         ET_LOG(Info, "\nReached to the end of generation");
         break;
@@ -136,11 +143,22 @@ public:
    */
   inline void stop() { should_stop_ = true; }
 
+  void set_count_interval(size_t count_interval) {
+    count_interval_ = count_interval;
+  }
+
+  void set_time_interval(size_t time_interval) {
+    time_interval_ = std::chrono::milliseconds(time_interval);
+  }
+
 private:
   tokenizers::Tokenizer *tokenizer_;
   TextDecoderRunner *text_decoder_runner_;
   std::unique_ptr<std::unordered_set<uint64_t>> eos_ids_;
   bool use_kv_cache_;
+  size_t count_interval_{10};
+  std::chrono::milliseconds time_interval_{120};
+  std::chrono::high_resolution_clock::time_point timestamp_;
 
   // state machine
   bool should_stop_ = false;

package/lib/module/controllers/LLMController.js

@@ -84,7 +84,16 @@ export class LLMController {
       this.nativeModule = global.loadLLM(modelPath, tokenizerPath);
       this.isReadyCallback(true);
       this.onToken = data => {
-        if (!data || SPECIAL_TOKENS.EOS_TOKEN in this.tokenizerConfig && data === this.tokenizerConfig.eos_token || SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig && data === this.tokenizerConfig.pad_token) {
+        if (!data) {
+          return;
+        }
+        if (SPECIAL_TOKENS.EOS_TOKEN in this.tokenizerConfig && data.indexOf(this.tokenizerConfig.eos_token) >= 0) {
+          data = data.replaceAll(this.tokenizerConfig.eos_token, '');
+        }
+        if (SPECIAL_TOKENS.PAD_TOKEN in this.tokenizerConfig && data.indexOf(this.tokenizerConfig.pad_token) >= 0) {
+          data = data.replaceAll(this.tokenizerConfig.pad_token, '');
+        }
+        if (data.length === 0) {
           return;
         }
         this.tokenCallback(data);
@@ -100,13 +109,20 @@ export class LLMController {
   }
   configure({
     chatConfig,
-    toolsConfig
+    toolsConfig,
+    generationConfig
   }) {
     this.chatConfig = {
       ...DEFAULT_CHAT_CONFIG,
       ...chatConfig
     };
     this.toolsConfig = toolsConfig;
+    if (generationConfig?.outputTokenBatchSize) {
+      this.nativeModule.setCountInterval(generationConfig.outputTokenBatchSize);
+    }
+    if (generationConfig?.batchTimeInterval) {
+      this.nativeModule.setTimeInterval(generationConfig.batchTimeInterval);
+    }
 
     // reset inner state when loading new configuration
     this.responseCallback('');
@@ -142,6 +158,9 @@ export class LLMController {
   interrupt() {
     this.nativeModule.interrupt();
   }
+  getGeneratedTokenCount() {
+    return this.nativeModule.getGeneratedTokenCount();
+  }
   async generate(messages, tools) {
     if (!this._isReady) {
       throw new Error(getError(ETError.ModuleNotLoaded));

package/lib/module/controllers/LLMController.js.map

@@ -1 +1 @@
-{"version":3,"names":["ResourceFetcher","ETError","getError","Template","DEFAULT_CHAT_CONFIG","readAsStringAsync","SPECIAL_TOKENS","parseToolCall","Logger","LLMController","chatConfig","_response","_isReady","_isGenerating","_messageHistory","constructor","tokenCallback","responseCallback","messageHistoryCallback","isReadyCallback","isGeneratingCallback","undefined","warn","token","response","messageHistory","isReady","isGenerating","load","modelSource","tokenizerSource","tokenizerConfigSource","onDownloadProgressCallback","initialMessageHistory","tokenizersPromise","fetch","modelPromise","tokenizersResults","modelResult","Promise","all","tokenizerPath","tokenizerConfigPath","modelPath","Error","tokenizerConfig","JSON","parse","nativeModule","global","loadLLM","onToken","data","EOS_TOKEN","eos_token","PAD_TOKEN","pad_token","e","setTokenCallback","configure","toolsConfig","delete","ModelGenerating","unload","forward","input","ModuleNotLoaded","generate","interrupt","messages","tools","length","role","renderedChat","applyChatTemplate","tools_in_user_message","add_generation_prompt","sendMessage","message","content","messageHistoryWithPrompt","systemPrompt","slice","contextWindowLength","displayToolCalls","toolCalls","toolCall","executeToolCallback","then","toolResponse","deleteMessage","index","newMessageHistory","templateFlags","chat_template","template","specialTokens","Object","fromEntries","keys","filter","key","map","result","render"],"sourceRoot":"../../../src","sources":["controllers/LLMController.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,0BAA0B;AAC1D,SAASC,OAAO,EAAEC,QAAQ,QAAQ,UAAU;AAC5C,SAASC,QAAQ,QAAQ,oBAAoB;AAC7C,SAASC,mBAAmB,QAAQ,0BAA0B;AAC9D,SAASC,iBAAiB,QAAQ,kBAAkB;AACpD,SAIEC,cAAc,QAET,cAAc;AACrB,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,MAAM,QAAQ,kBAAkB;AAEzC,OAAO,MAAMC,aAAa,CAAC;EAEjBC,UAAU,GAAeN,mBAAmB;EAI5CO,SAAS,GAAG,EAAE;EACdC,QAAQ,GAAG,KAAK;EAChBC,aAAa,GAAG,KAAK;EACrBC,eAAe,GAAc,EAAE;;EAEvC;;EAOAC,WAAWA,CAAC;IACVC,aAAa;IACbC,gBAAgB;IAChBC,sBAAsB;IACtBC,eAAe;IACfC;EAOF,CAAC,EAAE;IACD,IAAIH,gBAAgB,KAAKI,SAAS,EAAE;MAClCb,MAAM,CAACc,IAAI,CACT,sEACF,CAAC;IACH;IACA,IAAI,CAACN,aAAa,GAAIO,KAAK,IAAK;MAC9BP,aAAa,GAAGO,KAAK,CAAC;IACxB,CAAC;IACD,IAAI,CAACN,gBAAgB,GAAIO,QAAQ,IAAK;MACpC,IAAI,CAACb,SAAS,GAAGa,QAAQ;MACzBP,gBAAgB,GAAGO,QAAQ,CAAC;IAC9B,CAAC;IACD,IAAI,CAACN,sBAAsB,GAAIO,cAAc,IAAK;MAChD,IAAI,CAACX,eAAe,GAAGW,cAAc;MACrCP,sBAAsB,GAAGO,cAAc,CAAC;IAC1C,CAAC;IACD,IAAI,CAACN,eAAe,GAAIO,OAAO,IAAK;MAClC,IAAI,CAACd,QAAQ,GAAGc,OAAO;MACvBP,eAAe,GAAGO,OAAO,CAAC;IAC5B,CAAC;IACD,IAAI,CAACN,oBAAoB,GAAIO,YAAY,IAAK;MAC5C,IAAI,CAACd,aAAa,GAAGc,YAAY;MACjCP,oBAAoB,GAAGO,YAAY,CAAC;IACtC,CAAC;EACH;EAEA,IAAWH,QAAQA,CAAA,EAAG;IACpB,OAAO,IAAI,CAACb,SAAS;EACvB;EACA,IAAWe,OAAOA,CAAA,EAAG;IACnB,OAAO,IAAI,CAACd,QAAQ;EACtB;EACA,IAAWe,YAAYA,CAAA,EAAG;IACxB,OAAO,IAAI,CAACd,aAAa;EAC3B;EACA,IAAWY,cAAcA,CAAA,EAAG;IAC1B,OAAO,IAAI,CAACX,eAAe;EAC7B;EAEA,MAAac,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC,qBAAqB;IACrBC;EAMF,CAAC,EAAE;IACD;IACA,IAAI,CAACf,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACuB,qBAAqB,CAAC;IAClE,IAAI,CAACb,oBAAoB,CAAC,KAAK,CAAC;IAChC,IAAI,CAACD,eAAe,CAAC,KAAK,CAAC;IAE3B,IAAI;MACF,MAAMe,iBAAiB,GAAGlC,eAAe,CAACmC,KAAK,CAC7Cd,SAAS,EACTS,eAAe,EACfC,qBACF,CAAC;MAED,MAAMK,YAAY,GAAGpC,eAAe,CAACmC,KAAK,CACxCH,0BAA0B,EAC1BH,WACF,CAAC;MAED,MAAM,CAACQ,iBAAiB,EAAEC,WAAW,CAAC,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CACzDN,iBAAiB,EACjBE,YAAY,CACb,CAAC;MAEF,MAAMK,aAAa,GAAGJ,iBAAiB,GAAG,CAAC,CAAC;MAC5C,MAAMK,mBAAmB,GAAGL,iBAAiB,GAAG,CAAC,CAAC;MAClD,MAAMM,SAAS,GAAGL,WAAW,GAAG,CAAC,CAAC;MAElC,IAAI,CAACG,aAAa,IAAI,CAACC,mBAAmB,IAAI,CAACC,SAAS,EAAE;QACxD,MAAM,IAAIC,KAAK,CAAC,uBAAuB,CAAC;MAC1C;MAEA,IAAI,CAACC,eAAe,GAAGC,IAAI,CAACC,KAAK,CAC/B,MAAM1C,iBAAiB,CAAC,SAAS,GAAGqC,mBAAoB,CAC1D,CAAC;MACD,IAAI,CAACM,YAAY,GAAGC,MAAM,CAACC,OAAO,CAACP,SAAS,EAAEF,aAAa,CAAC;MAC5D,IAAI,CAACtB,eAAe,CAAC,IAAI,CAAC;MAC1B,IAAI,CAACgC,OAAO,GAAIC,IAAY,IAAK;QAC/B,IACE,CAACA,IAAI,IACJ9C,cAAc,CAAC+C,SAAS,IAAI,IAAI,CAACR,eAAe,IAC/CO,IAAI,KAAK,IAAI,CAACP,eAAe,CAACS,SAAU,IACzChD,cAAc,CAACiD,SAAS,IAAI,IAAI,CAACV,eAAe,IAC/CO,IAAI,KAAK,IAAI,CAACP,eAAe,CAACW,SAAU,EAC1C;UACA;QACF;QAEA,IAAI,CAACxC,aAAa,CAACoC,IAAI,CAAC;QACxB,IAAI,CAACnC,gBAAgB,CAAC,IAAI,CAACN,SAAS,GAAGyC,IAAI,CAAC;MAC9C,CAAC;IACH,CAAC,CAAC,OAAOK,CAAC,EAAE;MACV,IAAI,CAACtC,eAAe,CAAC,KAAK,CAAC;MAC3B,MAAM,IAAIyB,KAAK,CAAC1C,QAAQ,CAACuD,CAAC,CAAC,CAAC;IAC9B;EACF;EAEOC,gBAAgBA,CAAC1C,aAAsC,EAAE;IAC9D,IAAI,CAACA,aAAa,GAAGA,aAAa;EACpC;EAEO2C,SAASA,CAAC;IACfjD,UAAU;IACVkD;EAIF,CAAC,EAAE;IACD,IAAI,CAAClD,UAAU,GAAG;MAAE,GAAGN,mBAAmB;MAAE,GAAGM;IAAW,CAAC;IAC3D,IAAI,CAACkD,WAAW,GAAGA,WAAW;;IAE9B;IACA,IAAI,CAAC3C,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACuB,qBAAqB,CAAC;IAClE,IAAI,CAACb,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEOyC,MAAMA,CAAA,EAAG;IACd,IAAI,IAAI,CAAChD,aAAa,EAAE;MACtB,MAAM,IAAI+B,KAAK,CACb1C,QAAQ,CAACD,OAAO,CAAC6D,eAAe,CAAC,GAC/B,+DACJ,CAAC;IACH;IACA,IAAI,CAACX,OAAO,GAAG,MAAM,CAAC,CAAC;IACvB,IAAI,CAACH,YAAY,CAACe,MAAM,CAAC,CAAC;IAC1B,IAAI,CAAC5C,eAAe,CAAC,KAAK,CAAC;IAC3B,IAAI,CAACC,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEA,MAAa4C,OAAOA,CAACC,KAAa,EAAE;IAClC,IAAI,CAAC,IAAI,CAACrD,QAAQ,EAAE;MAClB,MAAM,IAAIgC,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAACiE,eAAe,CAAC,CAAC;IACpD;IACA,IAAI,IAAI,CAACrD,aAAa,EAAE;MACtB,MAAM,IAAI+B,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAAC6D,eAAe,CAAC,CAAC;IACpD;IACA,IAAI;MACF,IAAI,CAAC7C,gBAAgB,CAAC,EAAE,CAAC;MACzB,IAAI,CAACG,oBAAoB,CAAC,IAAI,CAAC;MAC/B,MAAM,IAAI,CAAC4B,YAAY,CAACmB,QAAQ,CAACF,KAAK,EAAE,IAAI,CAACd,OAAO,CAAC;IACvD,CAAC,CAAC,OAAOM,CAAC,EAAE;MACV,MAAM,IAAIb,KAAK,CAAC1C,QAAQ,CAACuD,CAAC,CAAC,CAAC;IAC9B,CAAC,SAAS;MACR,IAAI,CAACrC,oBAAoB,CAAC,KAAK,CAAC;IAClC;EACF;EAEOgD,SAASA,CAAA,EAAG;IACjB,IAAI,CAACpB,YAAY,CAACoB,SAAS,CAAC,CAAC;EAC/B;EAEA,MAAaD,QAAQA,CAACE,QAAmB,EAAEC,KAAiB,EAAE;IAC5D,IAAI,CAAC,IAAI,CAAC1D,QAAQ,EAAE;MAClB,MAAM,IAAIgC,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAACiE,eAAe,CAAC,CAAC;IACpD;IACA,IAAIG,QAAQ,CAACE,MAAM,KAAK,CAAC,EAAE;MACzB,MAAM,IAAI3B,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IACA,IAAIyB,QAAQ,CAAC,CAAC,CAAC,IAAIA,QAAQ,CAAC,CAAC,CAAC,CAACG,IAAI,KAAK,QAAQ,EAAE;MAChDhE,MAAM,CAACc,IAAI,CACT,0LACF,CAAC;IACH;IAEA,MAAMmD,YAAoB,GAAG,IAAI,CAACC,iBAAiB,CACjDL,QAAQ,EACR,IAAI,CAACxB,eAAe,EACpByB,KAAK;IACL;IACA;MAAEK,qBAAqB,EAAE,KAAK;MAAEC,qBAAqB,EAAE;IAAK,CAC9D,CAAC;IAED,MAAM,IAAI,CAACZ,OAAO,CAACS,YAAY,CAAC;EAClC;EAEA,MAAaI,WAAWA,CAACC,OAAe,EAAE;IACxC,IAAI,CAAC5D,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;MAAEiE,OAAO,EAAED,OAAO;MAAEN,IAAI,EAAE;IAAO,CAAC,CACnC,CAAC;IAEF,MAAMQ,wBAAmC,GAAG,CAC1C;MAAED,OAAO,EAAE,IAAI,CAACrE,UAAU,CAACuE,YAAY;MAAET,IAAI,EAAE;IAAS,CAAC,EACzD,GAAG,IAAI,CAAC1D,eAAe,CAACoE,KAAK,CAAC,CAAC,IAAI,CAACxE,UAAU,CAACyE,mBAAmB,CAAC,CACpE;IAED,MAAM,IAAI,CAAChB,QAAQ,CAACa,wBAAwB,EAAE,IAAI,CAACpB,WAAW,EAAEU,KAAK,CAAC;IAEtE,IAAI,CAAC,IAAI,CAACV,WAAW,IAAI,IAAI,CAACA,WAAW,CAACwB,gBAAgB,EAAE;MAC1D,IAAI,CAAClE,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;QAAEiE,OAAO,EAAE,IAAI,CAACpE,SAAS;QAAE6D,IAAI,EAAE;MAAY,CAAC,CAC/C,CAAC;IACJ;IACA,IAAI,CAAC,IAAI,CAACZ,WAAW,EAAE;MACrB;IACF;IAEA,MAAMyB,SAAS,GAAG9E,aAAa,CAAC,IAAI,CAACI,SAAS,CAAC;IAE/C,KAAK,MAAM2E,QAAQ,IAAID,SAAS,EAAE;MAChC,IAAI,CAACzB,WAAW,CACb2B,mBAAmB,CAACD,QAAQ,CAAC,CAC7BE,IAAI,CAAEC,YAA2B,IAAK;QACrC,IAAIA,YAAY,EAAE;UAChB,IAAI,CAACvE,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;YAAEiE,OAAO,EAAEU,YAAY;YAAEjB,IAAI,EAAE;UAAY,CAAC,CAC7C,CAAC;QACJ;MACF,CAAC,CAAC;IACN;EACF;EAEOkB,aAAaA,CAACC,KAAa,EAAE;IAClC;IACA;IACA,MAAMC,iBAAiB,GAAG,IAAI,CAAC9E,eAAe,CAACoE,KAAK,CAAC,CAAC,EAAES,KAAK,CAAC;IAE9D,IAAI,CAACzE,sBAAsB,CAAC0E,iBAAiB,CAAC;EAChD;EAEQlB,iBAAiBA,CACvBL,QAAmB,EACnBxB,eAAoB,EACpByB,KAAiB,EACjBuB,aAAsB,EACd;IACR,IAAI,CAAChD,eAAe,CAACiD,aAAa,EAAE;MAClC,MAAMlD,KAAK,CAAC,gDAAgD,CAAC;IAC/D;IACA,MAAMmD,QAAQ,GAAG,IAAI5F,QAAQ,CAAC0C,eAAe,CAACiD,aAAa,CAAC;IAE5D,MAAME,aAAa,GAAGC,MAAM,CAACC,WAAW,CACtCD,MAAM,CAACE,IAAI,CAAC7F,cAAc,CAAC,CACxB8F,MAAM,CAAEC,GAAG,IAAKA,GAAG,IAAIxD,eAAe,CAAC,CACvCyD,GAAG,CAAED,GAAG,IAAK,CAACA,GAAG,EAAExD,eAAe,CAACwD,GAAG,CAAC,CAAC,CAC7C,CAAC;IAED,MAAME,MAAM,GAAGR,QAAQ,CAACS,MAAM,CAAC;MAC7BnC,QAAQ;MACRC,KAAK;MACL,GAAGuB,aAAa;MAChB,GAAGG;IACL,CAAC,CAAC;IACF,OAAOO,MAAM;EACf;AACF","ignoreList":[]}
+{"version":3,"names":["ResourceFetcher","ETError","getError","Template","DEFAULT_CHAT_CONFIG","readAsStringAsync","SPECIAL_TOKENS","parseToolCall","Logger","LLMController","chatConfig","_response","_isReady","_isGenerating","_messageHistory","constructor","tokenCallback","responseCallback","messageHistoryCallback","isReadyCallback","isGeneratingCallback","undefined","warn","token","response","messageHistory","isReady","isGenerating","load","modelSource","tokenizerSource","tokenizerConfigSource","onDownloadProgressCallback","initialMessageHistory","tokenizersPromise","fetch","modelPromise","tokenizersResults","modelResult","Promise","all","tokenizerPath","tokenizerConfigPath","modelPath","Error","tokenizerConfig","JSON","parse","nativeModule","global","loadLLM","onToken","data","EOS_TOKEN","indexOf","eos_token","replaceAll","PAD_TOKEN","pad_token","length","e","setTokenCallback","configure","toolsConfig","generationConfig","outputTokenBatchSize","setCountInterval","batchTimeInterval","setTimeInterval","delete","ModelGenerating","unload","forward","input","ModuleNotLoaded","generate","interrupt","getGeneratedTokenCount","messages","tools","role","renderedChat","applyChatTemplate","tools_in_user_message","add_generation_prompt","sendMessage","message","content","messageHistoryWithPrompt","systemPrompt","slice","contextWindowLength","displayToolCalls","toolCalls","toolCall","executeToolCallback","then","toolResponse","deleteMessage","index","newMessageHistory","templateFlags","chat_template","template","specialTokens","Object","fromEntries","keys","filter","key","map","result","render"],"sourceRoot":"../../../src","sources":["controllers/LLMController.ts"],"mappings":";;AACA,SAASA,eAAe,QAAQ,0BAA0B;AAC1D,SAASC,OAAO,EAAEC,QAAQ,QAAQ,UAAU;AAC5C,SAASC,QAAQ,QAAQ,oBAAoB;AAC7C,SAASC,mBAAmB,QAAQ,0BAA0B;AAC9D,SAASC,iBAAiB,QAAQ,kBAAkB;AACpD,SAKEC,cAAc,QAET,cAAc;AACrB,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,MAAM,QAAQ,kBAAkB;AAEzC,OAAO,MAAMC,aAAa,CAAC;EAEjBC,UAAU,GAAeN,mBAAmB;EAI5CO,SAAS,GAAG,EAAE;EACdC,QAAQ,GAAG,KAAK;EAChBC,aAAa,GAAG,KAAK;EACrBC,eAAe,GAAc,EAAE;;EAEvC;;EAOAC,WAAWA,CAAC;IACVC,aAAa;IACbC,gBAAgB;IAChBC,sBAAsB;IACtBC,eAAe;IACfC;EAOF,CAAC,EAAE;IACD,IAAIH,gBAAgB,KAAKI,SAAS,EAAE;MAClCb,MAAM,CAACc,IAAI,CACT,sEACF,CAAC;IACH;IACA,IAAI,CAACN,aAAa,GAAIO,KAAK,IAAK;MAC9BP,aAAa,GAAGO,KAAK,CAAC;IACxB,CAAC;IACD,IAAI,CAACN,gBAAgB,GAAIO,QAAQ,IAAK;MACpC,IAAI,CAACb,SAAS,GAAGa,QAAQ;MACzBP,gBAAgB,GAAGO,QAAQ,CAAC;IAC9B,CAAC;IACD,IAAI,CAACN,sBAAsB,GAAIO,cAAc,IAAK;MAChD,IAAI,CAACX,eAAe,GAAGW,cAAc;MACrCP,sBAAsB,GAAGO,cAAc,CAAC;IAC1C,CAAC;IACD,IAAI,CAACN,eAAe,GAAIO,OAAO,IAAK;MAClC,IAAI,CAACd,QAAQ,GAAGc,OAAO;MACvBP,eAAe,GAAGO,OAAO,CAAC;IAC5B,CAAC;IACD,IAAI,CAACN,oBAAoB,GAAIO,YAAY,IAAK;MAC5C,IAAI,CAACd,aAAa,GAAGc,YAAY;MACjCP,oBAAoB,GAAGO,YAAY,CAAC;IACtC,CAAC;EACH;EAEA,IAAWH,QAAQA,CAAA,EAAG;IACpB,OAAO,IAAI,CAACb,SAAS;EACvB;EACA,IAAWe,OAAOA,CAAA,EAAG;IACnB,OAAO,IAAI,CAACd,QAAQ;EACtB;EACA,IAAWe,YAAYA,CAAA,EAAG;IACxB,OAAO,IAAI,CAACd,aAAa;EAC3B;EACA,IAAWY,cAAcA,CAAA,EAAG;IAC1B,OAAO,IAAI,CAACX,eAAe;EAC7B;EAEA,MAAac,IAAIA,CAAC;IAChBC,WAAW;IACXC,eAAe;IACfC,qBAAqB;IACrBC;EAMF,CAAC,EAAE;IACD;IACA,IAAI,CAACf,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACuB,qBAAqB,CAAC;IAClE,IAAI,CAACb,oBAAoB,CAAC,KAAK,CAAC;IAChC,IAAI,CAACD,eAAe,CAAC,KAAK,CAAC;IAE3B,IAAI;MACF,MAAMe,iBAAiB,GAAGlC,eAAe,CAACmC,KAAK,CAC7Cd,SAAS,EACTS,eAAe,EACfC,qBACF,CAAC;MAED,MAAMK,YAAY,GAAGpC,eAAe,CAACmC,KAAK,CACxCH,0BAA0B,EAC1BH,WACF,CAAC;MAED,MAAM,CAACQ,iBAAiB,EAAEC,WAAW,CAAC,GAAG,MAAMC,OAAO,CAACC,GAAG,CAAC,CACzDN,iBAAiB,EACjBE,YAAY,CACb,CAAC;MAEF,MAAMK,aAAa,GAAGJ,iBAAiB,GAAG,CAAC,CAAC;MAC5C,MAAMK,mBAAmB,GAAGL,iBAAiB,GAAG,CAAC,CAAC;MAClD,MAAMM,SAAS,GAAGL,WAAW,GAAG,CAAC,CAAC;MAElC,IAAI,CAACG,aAAa,IAAI,CAACC,mBAAmB,IAAI,CAACC,SAAS,EAAE;QACxD,MAAM,IAAIC,KAAK,CAAC,uBAAuB,CAAC;MAC1C;MAEA,IAAI,CAACC,eAAe,GAAGC,IAAI,CAACC,KAAK,CAC/B,MAAM1C,iBAAiB,CAAC,SAAS,GAAGqC,mBAAoB,CAC1D,CAAC;MACD,IAAI,CAACM,YAAY,GAAGC,MAAM,CAACC,OAAO,CAACP,SAAS,EAAEF,aAAa,CAAC;MAC5D,IAAI,CAACtB,eAAe,CAAC,IAAI,CAAC;MAC1B,IAAI,CAACgC,OAAO,GAAIC,IAAY,IAAK;QAC/B,IAAI,CAACA,IAAI,EAAE;UACT;QACF;QAEA,IACE9C,cAAc,CAAC+C,SAAS,IAAI,IAAI,CAACR,eAAe,IAChDO,IAAI,CAACE,OAAO,CAAC,IAAI,CAACT,eAAe,CAACU,SAAS,CAAC,IAAI,CAAC,EACjD;UACAH,IAAI,GAAGA,IAAI,CAACI,UAAU,CAAC,IAAI,CAACX,eAAe,CAACU,SAAS,EAAE,EAAE,CAAC;QAC5D;QACA,IACEjD,cAAc,CAACmD,SAAS,IAAI,IAAI,CAACZ,eAAe,IAChDO,IAAI,CAACE,OAAO,CAAC,IAAI,CAACT,eAAe,CAACa,SAAS,CAAC,IAAI,CAAC,EACjD;UACAN,IAAI,GAAGA,IAAI,CAACI,UAAU,CAAC,IAAI,CAACX,eAAe,CAACa,SAAS,EAAE,EAAE,CAAC;QAC5D;QACA,IAAIN,IAAI,CAACO,MAAM,KAAK,CAAC,EAAE;UACrB;QACF;QAEA,IAAI,CAAC3C,aAAa,CAACoC,IAAI,CAAC;QACxB,IAAI,CAACnC,gBAAgB,CAAC,IAAI,CAACN,SAAS,GAAGyC,IAAI,CAAC;MAC9C,CAAC;IACH,CAAC,CAAC,OAAOQ,CAAC,EAAE;MACV,IAAI,CAACzC,eAAe,CAAC,KAAK,CAAC;MAC3B,MAAM,IAAIyB,KAAK,CAAC1C,QAAQ,CAAC0D,CAAC,CAAC,CAAC;IAC9B;EACF;EAEOC,gBAAgBA,CAAC7C,aAAsC,EAAE;IAC9D,IAAI,CAACA,aAAa,GAAGA,aAAa;EACpC;EAEO8C,SAASA,CAAC;IACfpD,UAAU;IACVqD,WAAW;IACXC;EAKF,CAAC,EAAE;IACD,IAAI,CAACtD,UAAU,GAAG;MAAE,GAAGN,mBAAmB;MAAE,GAAGM;IAAW,CAAC;IAC3D,IAAI,CAACqD,WAAW,GAAGA,WAAW;IAE9B,IAAIC,gBAAgB,EAAEC,oBAAoB,EAAE;MAC1C,IAAI,CAACjB,YAAY,CAACkB,gBAAgB,CAACF,gBAAgB,CAACC,oBAAoB,CAAC;IAC3E;IACA,IAAID,gBAAgB,EAAEG,iBAAiB,EAAE;MACvC,IAAI,CAACnB,YAAY,CAACoB,eAAe,CAACJ,gBAAgB,CAACG,iBAAiB,CAAC;IACvE;;IAEA;IACA,IAAI,CAAClD,gBAAgB,CAAC,EAAE,CAAC;IACzB,IAAI,CAACC,sBAAsB,CAAC,IAAI,CAACR,UAAU,CAACuB,qBAAqB,CAAC;IAClE,IAAI,CAACb,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEOiD,MAAMA,CAAA,EAAG;IACd,IAAI,IAAI,CAACxD,aAAa,EAAE;MACtB,MAAM,IAAI+B,KAAK,CACb1C,QAAQ,CAACD,OAAO,CAACqE,eAAe,CAAC,GAC/B,+DACJ,CAAC;IACH;IACA,IAAI,CAACnB,OAAO,GAAG,MAAM,CAAC,CAAC;IACvB,IAAI,CAACH,YAAY,CAACuB,MAAM,CAAC,CAAC;IAC1B,IAAI,CAACpD,eAAe,CAAC,KAAK,CAAC;IAC3B,IAAI,CAACC,oBAAoB,CAAC,KAAK,CAAC;EAClC;EAEA,MAAaoD,OAAOA,CAACC,KAAa,EAAE;IAClC,IAAI,CAAC,IAAI,CAAC7D,QAAQ,EAAE;MAClB,MAAM,IAAIgC,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAACyE,eAAe,CAAC,CAAC;IACpD;IACA,IAAI,IAAI,CAAC7D,aAAa,EAAE;MACtB,MAAM,IAAI+B,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAACqE,eAAe,CAAC,CAAC;IACpD;IACA,IAAI;MACF,IAAI,CAACrD,gBAAgB,CAAC,EAAE,CAAC;MACzB,IAAI,CAACG,oBAAoB,CAAC,IAAI,CAAC;MAC/B,MAAM,IAAI,CAAC4B,YAAY,CAAC2B,QAAQ,CAACF,KAAK,EAAE,IAAI,CAACtB,OAAO,CAAC;IACvD,CAAC,CAAC,OAAOS,CAAC,EAAE;MACV,MAAM,IAAIhB,KAAK,CAAC1C,QAAQ,CAAC0D,CAAC,CAAC,CAAC;IAC9B,CAAC,SAAS;MACR,IAAI,CAACxC,oBAAoB,CAAC,KAAK,CAAC;IAClC;EACF;EAEOwD,SAASA,CAAA,EAAG;IACjB,IAAI,CAAC5B,YAAY,CAAC4B,SAAS,CAAC,CAAC;EAC/B;EAEOC,sBAAsBA,CAAA,EAAW;IACtC,OAAO,IAAI,CAAC7B,YAAY,CAAC6B,sBAAsB,CAAC,CAAC;EACnD;EAEA,MAAaF,QAAQA,CAACG,QAAmB,EAAEC,KAAiB,EAAE;IAC5D,IAAI,CAAC,IAAI,CAACnE,QAAQ,EAAE;MAClB,MAAM,IAAIgC,KAAK,CAAC1C,QAAQ,CAACD,OAAO,CAACyE,eAAe,CAAC,CAAC;IACpD;IACA,IAAII,QAAQ,CAACnB,MAAM,KAAK,CAAC,EAAE;MACzB,MAAM,IAAIf,KAAK,CAAC,yBAAyB,CAAC;IAC5C;IACA,IAAIkC,QAAQ,CAAC,CAAC,CAAC,IAAIA,QAAQ,CAAC,CAAC,CAAC,CAACE,IAAI,KAAK,QAAQ,EAAE;MAChDxE,MAAM,CAACc,IAAI,CACT,0LACF,CAAC;IACH;IAEA,MAAM2D,YAAoB,GAAG,IAAI,CAACC,iBAAiB,CACjDJ,QAAQ,EACR,IAAI,CAACjC,eAAe,EACpBkC,KAAK;IACL;IACA;MAAEI,qBAAqB,EAAE,KAAK;MAAEC,qBAAqB,EAAE;IAAK,CAC9D,CAAC;IAED,MAAM,IAAI,CAACZ,OAAO,CAACS,YAAY,CAAC;EAClC;EAEA,MAAaI,WAAWA,CAACC,OAAe,EAAE;IACxC,IAAI,CAACpE,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;MAAEyE,OAAO,EAAED,OAAO;MAAEN,IAAI,EAAE;IAAO,CAAC,CACnC,CAAC;IAEF,MAAMQ,wBAAmC,GAAG,CAC1C;MAAED,OAAO,EAAE,IAAI,CAAC7E,UAAU,CAAC+E,YAAY;MAAET,IAAI,EAAE;IAAS,CAAC,EACzD,GAAG,IAAI,CAAClE,eAAe,CAAC4E,KAAK,CAAC,CAAC,IAAI,CAAChF,UAAU,CAACiF,mBAAmB,CAAC,CACpE;IAED,MAAM,IAAI,CAAChB,QAAQ,CAACa,wBAAwB,EAAE,IAAI,CAACzB,WAAW,EAAEgB,KAAK,CAAC;IAEtE,IAAI,CAAC,IAAI,CAAChB,WAAW,IAAI,IAAI,CAACA,WAAW,CAAC6B,gBAAgB,EAAE;MAC1D,IAAI,CAAC1E,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;QAAEyE,OAAO,EAAE,IAAI,CAAC5E,SAAS;QAAEqE,IAAI,EAAE;MAAY,CAAC,CAC/C,CAAC;IACJ;IACA,IAAI,CAAC,IAAI,CAACjB,WAAW,EAAE;MACrB;IACF;IAEA,MAAM8B,SAAS,GAAGtF,aAAa,CAAC,IAAI,CAACI,SAAS,CAAC;IAE/C,KAAK,MAAMmF,QAAQ,IAAID,SAAS,EAAE;MAChC,IAAI,CAAC9B,WAAW,CACbgC,mBAAmB,CAACD,QAAQ,CAAC,CAC7BE,IAAI,CAAEC,YAA2B,IAAK;QACrC,IAAIA,YAAY,EAAE;UAChB,IAAI,CAAC/E,sBAAsB,CAAC,CAC1B,GAAG,IAAI,CAACJ,eAAe,EACvB;YAAEyE,OAAO,EAAEU,YAAY;YAAEjB,IAAI,EAAE;UAAY,CAAC,CAC7C,CAAC;QACJ;MACF,CAAC,CAAC;IACN;EACF;EAEOkB,aAAaA,CAACC,KAAa,EAAE;IAClC;IACA;IACA,MAAMC,iBAAiB,GAAG,IAAI,CAACtF,eAAe,CAAC4E,KAAK,CAAC,CAAC,EAAES,KAAK,CAAC;IAE9D,IAAI,CAACjF,sBAAsB,CAACkF,iBAAiB,CAAC;EAChD;EAEQlB,iBAAiBA,CACvBJ,QAAmB,EACnBjC,eAAoB,EACpBkC,KAAiB,EACjBsB,aAAsB,EACd;IACR,IAAI,CAACxD,eAAe,CAACyD,aAAa,EAAE;MAClC,MAAM1D,KAAK,CAAC,gDAAgD,CAAC;IAC/D;IACA,MAAM2D,QAAQ,GAAG,IAAIpG,QAAQ,CAAC0C,eAAe,CAACyD,aAAa,CAAC;IAE5D,MAAME,aAAa,GAAGC,MAAM,CAACC,WAAW,CACtCD,MAAM,CAACE,IAAI,CAACrG,cAAc,CAAC,CACxBsG,MAAM,CAAEC,GAAG,IAAKA,GAAG,IAAIhE,eAAe,CAAC,CACvCiE,GAAG,CAAED,GAAG,IAAK,CAACA,GAAG,EAAEhE,eAAe,CAACgE,GAAG,CAAC,CAAC,CAC7C,CAAC;IAED,MAAME,MAAM,GAAGR,QAAQ,CAACS,MAAM,CAAC;MAC7BlC,QAAQ;MACRC,KAAK;MACL,GAAGsB,aAAa;MAChB,GAAGG;IACL,CAAC,CAAC;IACF,OAAOO,MAAM;EACf;AACF","ignoreList":[]}

package/lib/module/hooks/natural_language_processing/useLLM.js

@@ -51,10 +51,12 @@ export const useLLM = ({
   // memoization of returned functions
   const configure = useCallback(({
     chatConfig,
-    toolsConfig
+    toolsConfig,
+    generationConfig
   }) => controllerInstance.configure({
     chatConfig,
-    toolsConfig
+    toolsConfig,
+    generationConfig
   }), [controllerInstance]);
   const generate = useCallback((messages, tools) => {
     setResponse('');
@@ -66,6 +68,7 @@ export const useLLM = ({
   }, [controllerInstance]);
   const deleteMessage = useCallback(index => controllerInstance.deleteMessage(index), [controllerInstance]);
   const interrupt = useCallback(() => controllerInstance.interrupt(), [controllerInstance]);
+  const getGeneratedTokenCount = useCallback(() => controllerInstance.getGeneratedTokenCount(), [controllerInstance]);
   return {
     messageHistory,
     response,
@@ -74,6 +77,7 @@ export const useLLM = ({
     isGenerating,
     downloadProgress,
     error,
+    getGeneratedTokenCount: getGeneratedTokenCount,
     configure: configure,
     generate: generate,
     sendMessage: sendMessage,