react-native-executorch 0.5.1-rc.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +132 -0
- package/common/rnexecutorch/models/speech_to_text/SpeechToText.cpp +4 -10
- package/common/rnexecutorch/models/speech_to_text/SpeechToText.h +1 -1
- package/common/rnexecutorch/models/speech_to_text/SpeechToTextStrategy.h +3 -2
- package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.cpp +16 -4
- package/common/rnexecutorch/models/speech_to_text/WhisperStrategy.h +2 -2
- package/ios/RnExecutorch.xcodeproj/project.xcworkspace/contents.xcworkspacedata +7 -0
- package/ios/RnExecutorch.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- package/ios/RnExecutorch.xcodeproj/xcuserdata/jakubchmura.xcuserdatad/xcschemes/xcschememanagement.plist +14 -0
- package/lib/module/constants/modelUrls.js +61 -36
- package/lib/module/constants/modelUrls.js.map +1 -1
- package/lib/module/constants/ocr/models.js +1 -1
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js +71 -34
- package/lib/module/hooks/natural_language_processing/useSpeechToText.js.map +1 -1
- package/lib/module/index.js +2 -3
- package/lib/module/index.js.map +1 -1
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js +72 -31
- package/lib/module/modules/natural_language_processing/SpeechToTextModule.js.map +1 -1
- package/lib/module/types/stt.js +1 -85
- package/lib/module/types/stt.js.map +1 -1
- package/lib/module/utils/SpeechToTextModule/ASR.js +191 -0
- package/lib/module/utils/SpeechToTextModule/ASR.js.map +1 -0
- package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js +73 -0
- package/lib/module/utils/SpeechToTextModule/OnlineProcessor.js.map +1 -0
- package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js +56 -0
- package/lib/module/utils/SpeechToTextModule/hypothesisBuffer.js.map +1 -0
- package/lib/tsconfig.tsbuildinfo +1 -0
- package/lib/typescript/constants/modelUrls.d.ts +24 -7
- package/lib/typescript/constants/modelUrls.d.ts.map +1 -1
- package/lib/typescript/constants/ocr/models.d.ts +126 -126
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts +15 -24
- package/lib/typescript/hooks/natural_language_processing/useSpeechToText.d.ts.map +1 -1
- package/lib/typescript/index.d.ts +2 -3
- package/lib/typescript/index.d.ts.map +1 -1
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts +19 -22
- package/lib/typescript/modules/natural_language_processing/SpeechToTextModule.d.ts.map +1 -1
- package/lib/typescript/types/stt.d.ts +17 -91
- package/lib/typescript/types/stt.d.ts.map +1 -1
- package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts +27 -0
- package/lib/typescript/utils/SpeechToTextModule/ASR.d.ts.map +1 -0
- package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts +23 -0
- package/lib/typescript/utils/SpeechToTextModule/OnlineProcessor.d.ts.map +1 -0
- package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts +13 -0
- package/lib/typescript/utils/SpeechToTextModule/hypothesisBuffer.d.ts.map +1 -0
- package/package.json +5 -3
- package/src/constants/modelUrls.ts +70 -37
- package/src/constants/ocr/models.ts +1 -1
- package/src/hooks/natural_language_processing/useSpeechToText.ts +87 -92
- package/src/index.ts +6 -8
- package/src/modules/natural_language_processing/SpeechToTextModule.ts +81 -69
- package/src/types/stt.ts +97 -92
- package/src/utils/SpeechToTextModule/ASR.ts +303 -0
- package/src/utils/SpeechToTextModule/OnlineProcessor.ts +87 -0
- package/src/utils/SpeechToTextModule/hypothesisBuffer.ts +79 -0
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/jakubchmura.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.cpp +0 -31
- package/common/rnexecutorch/models/speech_to_text/MoonshineStrategy.h +0 -21
- package/lib/common/Logger.d.ts +0 -8
- package/lib/common/Logger.js +0 -19
- package/lib/constants/modelUrls.d.ts +0 -89
- package/lib/constants/modelUrls.js +0 -116
- package/lib/constants/sttDefaults.js +0 -66
- package/lib/controllers/LLMController.js +0 -210
- package/lib/controllers/OCRController.js +0 -65
- package/lib/controllers/SpeechToTextController.d.ts +0 -52
- package/lib/controllers/SpeechToTextController.js +0 -343
- package/lib/hooks/natural_language_processing/useSpeechToText.js +0 -44
- package/lib/index.d.ts +0 -50
- package/lib/index.js +0 -59
- package/lib/module/constants/sttDefaults.js +0 -74
- package/lib/module/constants/sttDefaults.js.map +0 -1
- package/lib/module/controllers/SpeechToTextController.js +0 -320
- package/lib/module/controllers/SpeechToTextController.js.map +0 -1
- package/lib/modules/natural_language_processing/SpeechToTextModule.d.ts +0 -14
- package/lib/modules/natural_language_processing/SpeechToTextModule.js +0 -30
- package/lib/modules/natural_language_processing/TokenizerModule.js +0 -29
- package/lib/native/RnExecutorchModules.d.ts +0 -3
- package/lib/native/RnExecutorchModules.js +0 -16
- package/lib/typescript/constants/sttDefaults.d.ts +0 -29
- package/lib/typescript/constants/sttDefaults.d.ts.map +0 -1
- package/lib/typescript/controllers/SpeechToTextController.d.ts +0 -57
- package/lib/typescript/controllers/SpeechToTextController.d.ts.map +0 -1
- package/lib/utils/ResourceFetcherUtils.js +0 -119
- package/lib/utils/llm.js +0 -72
- package/src/constants/sttDefaults.ts +0 -82
- package/src/controllers/SpeechToTextController.ts +0 -471
- package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/project.xcworkspace/xcuserdata/norbertklockiewicz.xcuserdatad/UserInterfaceState.xcuserstate +0 -0
- /package/third-party/ios/ExecutorchLib/ExecutorchLib.xcodeproj/xcuserdata/{norbertklockiewicz.xcuserdatad → jakubchmura.xcuserdatad}/xcschemes/xcschememanagement.plist +0 -0
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @internal
|
|
3
|
-
*/
|
|
4
|
-
import { getInfoAsync, makeDirectoryAsync, } from 'expo-file-system';
|
|
5
|
-
import { RNEDirectory } from '../constants/directories';
|
|
6
|
-
import { Asset } from 'expo-asset';
|
|
7
|
-
import { Logger } from '../common/Logger';
|
|
8
|
-
export var ResourceFetcherUtils;
|
|
9
|
-
(function (ResourceFetcherUtils) {
|
|
10
|
-
function getType(source) {
|
|
11
|
-
if (typeof source === 'object') {
|
|
12
|
-
return 0 /* SourceType.OBJECT */;
|
|
13
|
-
}
|
|
14
|
-
else if (typeof source === 'number') {
|
|
15
|
-
const uri = Asset.fromModule(source).uri;
|
|
16
|
-
if (!uri.includes('://')) {
|
|
17
|
-
return 2 /* SourceType.RELEASE_MODE_FILE */;
|
|
18
|
-
}
|
|
19
|
-
return 3 /* SourceType.DEV_MODE_FILE */;
|
|
20
|
-
}
|
|
21
|
-
else {
|
|
22
|
-
// typeof source == 'string'
|
|
23
|
-
if (source.startsWith('file://')) {
|
|
24
|
-
return 1 /* SourceType.LOCAL_FILE */;
|
|
25
|
-
}
|
|
26
|
-
return 4 /* SourceType.REMOTE_FILE */;
|
|
27
|
-
}
|
|
28
|
-
}
|
|
29
|
-
ResourceFetcherUtils.getType = getType;
|
|
30
|
-
async function getFilesSizes(sources) {
|
|
31
|
-
const results = [];
|
|
32
|
-
let totalLength = 0;
|
|
33
|
-
let previousFilesTotalLength = 0;
|
|
34
|
-
for (const source of sources) {
|
|
35
|
-
const type = await ResourceFetcherUtils.getType(source);
|
|
36
|
-
let length = 0;
|
|
37
|
-
if (type === 4 /* SourceType.REMOTE_FILE */ && typeof source === 'string') {
|
|
38
|
-
try {
|
|
39
|
-
const response = await fetch(source, { method: 'HEAD' });
|
|
40
|
-
if (!response.ok) {
|
|
41
|
-
Logger.warn(`Failed to fetch HEAD for ${source}: ${response.status}`);
|
|
42
|
-
continue;
|
|
43
|
-
}
|
|
44
|
-
const contentLength = response.headers.get('content-length');
|
|
45
|
-
length = contentLength ? parseInt(contentLength, 10) : 0;
|
|
46
|
-
previousFilesTotalLength = totalLength;
|
|
47
|
-
totalLength += length;
|
|
48
|
-
}
|
|
49
|
-
catch (error) {
|
|
50
|
-
Logger.warn(`Error fetching HEAD for ${source}:`, error);
|
|
51
|
-
continue;
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
results.push({ source, type, length, previousFilesTotalLength });
|
|
55
|
-
}
|
|
56
|
-
return { results, totalLength };
|
|
57
|
-
}
|
|
58
|
-
ResourceFetcherUtils.getFilesSizes = getFilesSizes;
|
|
59
|
-
function removeFilePrefix(uri) {
|
|
60
|
-
return uri.startsWith('file://') ? uri.slice(7) : uri;
|
|
61
|
-
}
|
|
62
|
-
ResourceFetcherUtils.removeFilePrefix = removeFilePrefix;
|
|
63
|
-
function hashObject(jsonString) {
|
|
64
|
-
let hash = 0;
|
|
65
|
-
for (let i = 0; i < jsonString.length; i++) {
|
|
66
|
-
// eslint-disable-next-line no-bitwise
|
|
67
|
-
hash = (hash << 5) - hash + jsonString.charCodeAt(i);
|
|
68
|
-
// eslint-disable-next-line no-bitwise
|
|
69
|
-
hash |= 0;
|
|
70
|
-
}
|
|
71
|
-
// eslint-disable-next-line no-bitwise
|
|
72
|
-
return (hash >>> 0).toString();
|
|
73
|
-
}
|
|
74
|
-
ResourceFetcherUtils.hashObject = hashObject;
|
|
75
|
-
function calculateDownloadProgress(totalLength, previousFilesTotalLength, currentFileLength, setProgress) {
|
|
76
|
-
return (progress) => {
|
|
77
|
-
if (progress === 1 &&
|
|
78
|
-
previousFilesTotalLength === totalLength - currentFileLength) {
|
|
79
|
-
setProgress(1);
|
|
80
|
-
return;
|
|
81
|
-
}
|
|
82
|
-
const baseProgress = previousFilesTotalLength / totalLength;
|
|
83
|
-
const scaledProgress = progress * (currentFileLength / totalLength);
|
|
84
|
-
const updatedProgress = baseProgress + scaledProgress;
|
|
85
|
-
setProgress(updatedProgress);
|
|
86
|
-
};
|
|
87
|
-
}
|
|
88
|
-
ResourceFetcherUtils.calculateDownloadProgress = calculateDownloadProgress;
|
|
89
|
-
/*
|
|
90
|
-
* Increments the Hugging Face download counter if the URI points to a Software Mansion Hugging Face repo.
|
|
91
|
-
* More information: https://huggingface.co/docs/hub/models-download-stats
|
|
92
|
-
*/
|
|
93
|
-
async function triggerHuggingFaceDownloadCounter(uri) {
|
|
94
|
-
const url = new URL(uri);
|
|
95
|
-
if (url.host === 'huggingface.co' &&
|
|
96
|
-
url.pathname.startsWith('/software-mansion/')) {
|
|
97
|
-
const baseUrl = `${url.protocol}//${url.host}${url.pathname.split('resolve')[0]}`;
|
|
98
|
-
fetch(`${baseUrl}resolve/main/config.json`, { method: 'HEAD' });
|
|
99
|
-
}
|
|
100
|
-
}
|
|
101
|
-
ResourceFetcherUtils.triggerHuggingFaceDownloadCounter = triggerHuggingFaceDownloadCounter;
|
|
102
|
-
async function createDirectoryIfNoExists() {
|
|
103
|
-
if (!(await checkFileExists(RNEDirectory))) {
|
|
104
|
-
await makeDirectoryAsync(RNEDirectory, { intermediates: true });
|
|
105
|
-
}
|
|
106
|
-
}
|
|
107
|
-
ResourceFetcherUtils.createDirectoryIfNoExists = createDirectoryIfNoExists;
|
|
108
|
-
async function checkFileExists(fileUri) {
|
|
109
|
-
const fileInfo = await getInfoAsync(fileUri);
|
|
110
|
-
return fileInfo.exists;
|
|
111
|
-
}
|
|
112
|
-
ResourceFetcherUtils.checkFileExists = checkFileExists;
|
|
113
|
-
function getFilenameFromUri(uri) {
|
|
114
|
-
let cleanUri = uri.replace(/^https?:\/\//, '');
|
|
115
|
-
cleanUri = cleanUri.split('#')?.[0] ?? cleanUri;
|
|
116
|
-
return cleanUri.replace(/[^a-zA-Z0-9._-]/g, '_');
|
|
117
|
-
}
|
|
118
|
-
ResourceFetcherUtils.getFilenameFromUri = getFilenameFromUri;
|
|
119
|
-
})(ResourceFetcherUtils || (ResourceFetcherUtils = {}));
|
package/lib/utils/llm.js
DELETED
|
@@ -1,72 +0,0 @@
|
|
|
1
|
-
import * as z from 'zod/v4';
|
|
2
|
-
import { Validator } from 'jsonschema';
|
|
3
|
-
import { jsonrepair } from 'jsonrepair';
|
|
4
|
-
import { DEFAULT_STRUCTURED_OUTPUT_PROMPT } from '../constants/llmDefaults';
|
|
5
|
-
import * as zCore from 'zod/v4/core';
|
|
6
|
-
import { Logger } from '../common/Logger';
|
|
7
|
-
export const parseToolCall = (message) => {
|
|
8
|
-
try {
|
|
9
|
-
const unparsedToolCalls = message.match('\\[(.|\\s)*\\]');
|
|
10
|
-
if (!unparsedToolCalls) {
|
|
11
|
-
throw Error('Regex did not match array.');
|
|
12
|
-
}
|
|
13
|
-
const parsedMessage = JSON.parse(unparsedToolCalls[0]);
|
|
14
|
-
const results = [];
|
|
15
|
-
for (const tool of parsedMessage) {
|
|
16
|
-
if ('name' in tool &&
|
|
17
|
-
typeof tool.name === 'string' &&
|
|
18
|
-
'arguments' in tool &&
|
|
19
|
-
tool.arguments !== null &&
|
|
20
|
-
typeof tool.arguments === 'object') {
|
|
21
|
-
results.push({
|
|
22
|
-
toolName: tool.name,
|
|
23
|
-
arguments: tool.arguments,
|
|
24
|
-
});
|
|
25
|
-
}
|
|
26
|
-
}
|
|
27
|
-
return results;
|
|
28
|
-
}
|
|
29
|
-
catch (e) {
|
|
30
|
-
Logger.error(e);
|
|
31
|
-
return [];
|
|
32
|
-
}
|
|
33
|
-
};
|
|
34
|
-
const filterObjectKeys = (obj, keysToRemove) => {
|
|
35
|
-
const entries = Object.entries(obj);
|
|
36
|
-
const filteredEntries = entries.filter(([key, _]) => !keysToRemove.includes(key));
|
|
37
|
-
return Object.fromEntries(filteredEntries);
|
|
38
|
-
};
|
|
39
|
-
export const getStructuredOutputPrompt = (responseSchema) => {
|
|
40
|
-
const schemaObject = responseSchema instanceof zCore.$ZodType
|
|
41
|
-
? filterObjectKeys(z.toJSONSchema(responseSchema), [
|
|
42
|
-
'$schema',
|
|
43
|
-
'additionalProperties',
|
|
44
|
-
])
|
|
45
|
-
: responseSchema;
|
|
46
|
-
const schemaString = JSON.stringify(schemaObject);
|
|
47
|
-
return DEFAULT_STRUCTURED_OUTPUT_PROMPT(schemaString);
|
|
48
|
-
};
|
|
49
|
-
const extractBetweenBrackets = (text) => {
|
|
50
|
-
const startIndex = text.search(/[\\{\\[]/); // First occurrence of either { or [
|
|
51
|
-
const openingBracket = text[startIndex];
|
|
52
|
-
const closingBracket = openingBracket === '{' ? '}' : ']';
|
|
53
|
-
if (!openingBracket)
|
|
54
|
-
throw Error("Couldn't find JSON in text");
|
|
55
|
-
return text.slice(text.indexOf(openingBracket), text.lastIndexOf(closingBracket) + 1);
|
|
56
|
-
};
|
|
57
|
-
// this is a bit hacky typing
|
|
58
|
-
export const fixAndValidateStructuredOutput = (output, responseSchema) => {
|
|
59
|
-
const extractedOutput = extractBetweenBrackets(output);
|
|
60
|
-
const repairedOutput = jsonrepair(extractedOutput);
|
|
61
|
-
const outputJSON = JSON.parse(repairedOutput);
|
|
62
|
-
if (responseSchema instanceof zCore.$ZodType) {
|
|
63
|
-
return z.parse(responseSchema, outputJSON);
|
|
64
|
-
}
|
|
65
|
-
else {
|
|
66
|
-
const validator = new Validator();
|
|
67
|
-
validator.validate(outputJSON, responseSchema, {
|
|
68
|
-
throwAll: true,
|
|
69
|
-
});
|
|
70
|
-
return outputJSON;
|
|
71
|
-
}
|
|
72
|
-
};
|
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
MOONSHINE_TINY,
|
|
3
|
-
WHISPER_TINY,
|
|
4
|
-
WHISPER_TINY_MULTILINGUAL,
|
|
5
|
-
} from './modelUrls';
|
|
6
|
-
import { AvailableModels, ModelConfig } from '../types/stt';
|
|
7
|
-
|
|
8
|
-
export const SAMPLE_RATE = 16_000;
|
|
9
|
-
export const SECOND = SAMPLE_RATE;
|
|
10
|
-
export const HAMMING_DIST_THRESHOLD = 1;
|
|
11
|
-
|
|
12
|
-
const whisperTinyModelConfig = {
|
|
13
|
-
sources: {
|
|
14
|
-
encoder: WHISPER_TINY.encoderSource,
|
|
15
|
-
decoder: WHISPER_TINY.decoderSource,
|
|
16
|
-
},
|
|
17
|
-
tokenizer: {
|
|
18
|
-
source: WHISPER_TINY.tokenizerSource,
|
|
19
|
-
bos: 50257, // FIXME: this is a placeholder and needs to be changed
|
|
20
|
-
eos: 50256, // FIXME: this is a placeholder and needs to be changed
|
|
21
|
-
},
|
|
22
|
-
isMultilingual: false,
|
|
23
|
-
};
|
|
24
|
-
|
|
25
|
-
const moonshineTinyModelConfig = {
|
|
26
|
-
sources: {
|
|
27
|
-
encoder: MOONSHINE_TINY.encoderSource,
|
|
28
|
-
decoder: MOONSHINE_TINY.decoderSource,
|
|
29
|
-
},
|
|
30
|
-
tokenizer: {
|
|
31
|
-
source: MOONSHINE_TINY.tokenizerSource,
|
|
32
|
-
bos: 1, // FIXME: this is a placeholder and needs to be changed
|
|
33
|
-
eos: 2, // FIXME: this is a placeholder and needs to be changed
|
|
34
|
-
},
|
|
35
|
-
isMultilingual: false,
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
const whisperTinyMultilingualModelConfig = {
|
|
39
|
-
sources: {
|
|
40
|
-
encoder: WHISPER_TINY_MULTILINGUAL.encoderSource,
|
|
41
|
-
decoder: WHISPER_TINY_MULTILINGUAL.decoderSource,
|
|
42
|
-
},
|
|
43
|
-
tokenizer: {
|
|
44
|
-
source: WHISPER_TINY_MULTILINGUAL.tokenizerSource,
|
|
45
|
-
bos: 50258, // FIXME: this is a placeholder and needs to be changed
|
|
46
|
-
eos: 50257, // FIXME: this is a placeholder and needs to be changed
|
|
47
|
-
},
|
|
48
|
-
isMultilingual: true,
|
|
49
|
-
};
|
|
50
|
-
|
|
51
|
-
export const MODEL_CONFIGS: {
|
|
52
|
-
[key in AvailableModels]: ModelConfig;
|
|
53
|
-
} = {
|
|
54
|
-
moonshine: moonshineTinyModelConfig,
|
|
55
|
-
whisper: whisperTinyModelConfig,
|
|
56
|
-
whisperMultilingual: whisperTinyMultilingualModelConfig,
|
|
57
|
-
};
|
|
58
|
-
|
|
59
|
-
export const MODES = {
|
|
60
|
-
fast: {
|
|
61
|
-
windowSize: 5,
|
|
62
|
-
overlapSeconds: 1.2,
|
|
63
|
-
},
|
|
64
|
-
balanced: {
|
|
65
|
-
windowSize: 12,
|
|
66
|
-
overlapSeconds: 2,
|
|
67
|
-
},
|
|
68
|
-
quality: {
|
|
69
|
-
windowSize: 24,
|
|
70
|
-
overlapSeconds: 3,
|
|
71
|
-
},
|
|
72
|
-
};
|
|
73
|
-
|
|
74
|
-
export const NUM_TOKENS_TO_TRIM = 3;
|
|
75
|
-
|
|
76
|
-
export enum STREAMING_ACTION {
|
|
77
|
-
START,
|
|
78
|
-
DATA,
|
|
79
|
-
STOP,
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export { AvailableModels };
|