voicepilot-llm-engine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/index.d.ts +36 -0
- package/dist/actions/index.d.ts.map +1 -0
- package/dist/actions/index.js +82 -0
- package/dist/client.d.ts +29 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +67 -0
- package/dist/fuzzy-matcher.d.ts +41 -0
- package/dist/fuzzy-matcher.d.ts.map +1 -0
- package/dist/fuzzy-matcher.js +147 -0
- package/dist/index.d.ts +12 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +8 -0
- package/dist/intent/classifier.d.ts +20 -0
- package/dist/intent/classifier.d.ts.map +1 -0
- package/dist/intent/classifier.js +198 -0
- package/dist/intent/types.d.ts +62 -0
- package/dist/intent/types.d.ts.map +1 -0
- package/dist/intent/types.js +20 -0
- package/dist/prompts/index.d.ts +12 -0
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/index.js +22 -0
- package/dist/responses/generator.d.ts +81 -0
- package/dist/responses/generator.d.ts.map +1 -0
- package/dist/responses/generator.js +148 -0
- package/package.json +32 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Intent, IntentResult } from '../intent/types.js';
|
|
2
|
+
export interface ActionResult {
|
|
3
|
+
success: boolean;
|
|
4
|
+
message: string;
|
|
5
|
+
data?: any;
|
|
6
|
+
}
|
|
7
|
+
export interface ActionContext {
|
|
8
|
+
pageTitle?: string;
|
|
9
|
+
availableLinks?: {
|
|
10
|
+
text: string;
|
|
11
|
+
href: string;
|
|
12
|
+
}[];
|
|
13
|
+
forms?: any[];
|
|
14
|
+
}
|
|
15
|
+
export declare abstract class ActionHandler {
|
|
16
|
+
abstract canHandle(intent: Intent): boolean;
|
|
17
|
+
abstract execute(intentResult: IntentResult, context: ActionContext): Promise<ActionResult>;
|
|
18
|
+
}
|
|
19
|
+
export declare class NavigateActionHandler extends ActionHandler {
|
|
20
|
+
canHandle(intent: Intent): boolean;
|
|
21
|
+
execute(intentResult: IntentResult, context: ActionContext): Promise<ActionResult>;
|
|
22
|
+
}
|
|
23
|
+
export declare class FormFillActionHandler extends ActionHandler {
|
|
24
|
+
canHandle(intent: Intent): boolean;
|
|
25
|
+
execute(intentResult: IntentResult, context: ActionContext): Promise<ActionResult>;
|
|
26
|
+
}
|
|
27
|
+
export declare class ReadPageActionHandler extends ActionHandler {
|
|
28
|
+
canHandle(intent: Intent): boolean;
|
|
29
|
+
execute(intentResult: IntentResult, context: ActionContext): Promise<ActionResult>;
|
|
30
|
+
}
|
|
31
|
+
export declare class ActionRouter {
|
|
32
|
+
private handlers;
|
|
33
|
+
constructor();
|
|
34
|
+
route(intentResult: IntentResult, context: ActionContext): Promise<ActionResult>;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/actions/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAE1D,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,GAAG,CAAC;CACZ;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAClD,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC;CACf;AAED,8BAAsB,aAAa;IACjC,QAAQ,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAC3C,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;CAC5F;AAED,qBAAa,qBAAsB,SAAQ,aAAa;IACtD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5B,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;CA4BzF;AAED,qBAAa,qBAAsB,SAAQ,aAAa;IACtD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5B,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;CAiBzF;AAED,qBAAa,qBAAsB,SAAQ,aAAa;IACtD,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAI5B,OAAO,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;CAOzF;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAkB;;IAU5B,KAAK,CAAC,YAAY,EAAE,YAAY,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;CAYvF"}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import { Intent } from '../intent/types.js';
|
|
2
|
+
export class ActionHandler {
|
|
3
|
+
}
|
|
4
|
+
export class NavigateActionHandler extends ActionHandler {
|
|
5
|
+
canHandle(intent) {
|
|
6
|
+
return intent === Intent.NAVIGATE;
|
|
7
|
+
}
|
|
8
|
+
async execute(intentResult, context) {
|
|
9
|
+
const target = intentResult.entities?.target;
|
|
10
|
+
if (!target || !context.availableLinks) {
|
|
11
|
+
return {
|
|
12
|
+
success: false,
|
|
13
|
+
message: 'Could not determine navigation target',
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
const link = context.availableLinks.find(l => l.text.toLowerCase().includes(target.toLowerCase()) ||
|
|
17
|
+
l.href.toLowerCase().includes(target.toLowerCase()));
|
|
18
|
+
if (link) {
|
|
19
|
+
return {
|
|
20
|
+
success: true,
|
|
21
|
+
message: `Navigating to ${link.text}`,
|
|
22
|
+
data: { href: link.href, text: link.text },
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
success: false,
|
|
27
|
+
message: `Could not find link matching "${target}"`,
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
export class FormFillActionHandler extends ActionHandler {
|
|
32
|
+
canHandle(intent) {
|
|
33
|
+
return intent === Intent.FORM_FILL;
|
|
34
|
+
}
|
|
35
|
+
async execute(intentResult, context) {
|
|
36
|
+
const fieldName = intentResult.entities?.fieldName;
|
|
37
|
+
const fieldValue = intentResult.entities?.fieldValue;
|
|
38
|
+
if (!fieldName || !fieldValue) {
|
|
39
|
+
return {
|
|
40
|
+
success: false,
|
|
41
|
+
message: 'Could not extract field name or value',
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
return {
|
|
45
|
+
success: true,
|
|
46
|
+
message: `Filling ${fieldName} with ${fieldValue}`,
|
|
47
|
+
data: { fieldName, fieldValue },
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
export class ReadPageActionHandler extends ActionHandler {
|
|
52
|
+
canHandle(intent) {
|
|
53
|
+
return intent === Intent.READ_PAGE;
|
|
54
|
+
}
|
|
55
|
+
async execute(intentResult, context) {
|
|
56
|
+
return {
|
|
57
|
+
success: true,
|
|
58
|
+
message: 'Reading page content',
|
|
59
|
+
data: { action: 'read', section: intentResult.entities?.section },
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
export class ActionRouter {
|
|
64
|
+
handlers;
|
|
65
|
+
constructor() {
|
|
66
|
+
this.handlers = [
|
|
67
|
+
new NavigateActionHandler(),
|
|
68
|
+
new FormFillActionHandler(),
|
|
69
|
+
new ReadPageActionHandler(),
|
|
70
|
+
];
|
|
71
|
+
}
|
|
72
|
+
async route(intentResult, context) {
|
|
73
|
+
const handler = this.handlers.find(h => h.canHandle(intentResult.intent));
|
|
74
|
+
if (!handler) {
|
|
75
|
+
return {
|
|
76
|
+
success: false,
|
|
77
|
+
message: `No handler found for intent: ${intentResult.intent}`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
return handler.execute(intentResult, context);
|
|
81
|
+
}
|
|
82
|
+
}
|
package/dist/client.d.ts
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface AzureOpenAIConfig {
|
|
2
|
+
apiKey: string;
|
|
3
|
+
endpoint: string;
|
|
4
|
+
deployment: string;
|
|
5
|
+
apiVersion?: string;
|
|
6
|
+
}
|
|
7
|
+
export interface ChatMessage {
|
|
8
|
+
role: 'system' | 'user' | 'assistant';
|
|
9
|
+
content: string;
|
|
10
|
+
}
|
|
11
|
+
export interface CompletionOptions {
|
|
12
|
+
messages: ChatMessage[];
|
|
13
|
+
temperature?: number;
|
|
14
|
+
maxTokens?: number;
|
|
15
|
+
functionCall?: any;
|
|
16
|
+
}
|
|
17
|
+
export declare class AzureOpenAIClient {
|
|
18
|
+
private client;
|
|
19
|
+
private deployment;
|
|
20
|
+
constructor(config: AzureOpenAIConfig);
|
|
21
|
+
chat(options: CompletionOptions): Promise<string>;
|
|
22
|
+
chatWithFunctions(options: CompletionOptions & {
|
|
23
|
+
functions: any[];
|
|
24
|
+
functionCall?: any;
|
|
25
|
+
}): Promise<any>;
|
|
26
|
+
streamChat(options: CompletionOptions, onChunk: (text: string) => void): Promise<void>;
|
|
27
|
+
generateEmbedding(text: string): Promise<number[]>;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;IACtC,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,GAAG,CAAC;CACpB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,UAAU,CAAS;gBAEf,MAAM,EAAE,iBAAiB;IAU/B,IAAI,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,MAAM,CAAC;IAWjD,iBAAiB,CAAC,OAAO,EAAE,iBAAiB,GAAG;QAAE,SAAS,EAAE,GAAG,EAAE,CAAC;QAAC,YAAY,CAAC,EAAE,GAAG,CAAA;KAAE,GAAG,OAAO,CAAC,GAAG,CAAC;IA0BtG,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAiBtF,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;CAQzD"}
|
package/dist/client.js
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import OpenAI from 'openai';
|
|
2
|
+
export class AzureOpenAIClient {
|
|
3
|
+
client;
|
|
4
|
+
deployment;
|
|
5
|
+
constructor(config) {
|
|
6
|
+
this.client = new OpenAI({
|
|
7
|
+
apiKey: config.apiKey,
|
|
8
|
+
baseURL: `${config.endpoint}/openai/deployments/${config.deployment}`,
|
|
9
|
+
defaultQuery: { 'api-version': config.apiVersion || '2024-08-01-preview' },
|
|
10
|
+
defaultHeaders: { 'api-key': config.apiKey },
|
|
11
|
+
});
|
|
12
|
+
this.deployment = config.deployment;
|
|
13
|
+
}
|
|
14
|
+
async chat(options) {
|
|
15
|
+
const response = await this.client.chat.completions.create({
|
|
16
|
+
model: this.deployment,
|
|
17
|
+
messages: options.messages,
|
|
18
|
+
temperature: options.temperature ?? 0.7,
|
|
19
|
+
max_tokens: options.maxTokens ?? 500,
|
|
20
|
+
});
|
|
21
|
+
return response.choices[0]?.message?.content || '';
|
|
22
|
+
}
|
|
23
|
+
async chatWithFunctions(options) {
|
|
24
|
+
const response = await this.client.chat.completions.create({
|
|
25
|
+
model: this.deployment,
|
|
26
|
+
messages: options.messages,
|
|
27
|
+
temperature: options.temperature ?? 0.7,
|
|
28
|
+
max_tokens: options.maxTokens ?? 500,
|
|
29
|
+
tools: options.functions.map(f => ({ type: 'function', function: f })),
|
|
30
|
+
});
|
|
31
|
+
const choice = response.choices[0];
|
|
32
|
+
if (choice?.message?.tool_calls?.[0]) {
|
|
33
|
+
const toolCall = choice.message.tool_calls[0];
|
|
34
|
+
return {
|
|
35
|
+
type: 'function',
|
|
36
|
+
name: toolCall.function.name,
|
|
37
|
+
arguments: JSON.parse(toolCall.function.arguments || '{}'),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
return {
|
|
41
|
+
type: 'message',
|
|
42
|
+
content: choice?.message?.content || '',
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
async streamChat(options, onChunk) {
|
|
46
|
+
const stream = await this.client.chat.completions.create({
|
|
47
|
+
model: this.deployment,
|
|
48
|
+
messages: options.messages,
|
|
49
|
+
temperature: options.temperature ?? 0.7,
|
|
50
|
+
max_tokens: options.maxTokens ?? 500,
|
|
51
|
+
stream: true,
|
|
52
|
+
});
|
|
53
|
+
for await (const chunk of stream) {
|
|
54
|
+
const delta = chunk.choices[0]?.delta?.content;
|
|
55
|
+
if (delta) {
|
|
56
|
+
onChunk(delta);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async generateEmbedding(text) {
|
|
61
|
+
const response = await this.client.embeddings.create({
|
|
62
|
+
model: 'text-embedding-3-small',
|
|
63
|
+
input: text,
|
|
64
|
+
});
|
|
65
|
+
return response.data[0]?.embedding || [];
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fuzzy Command Matcher
|
|
3
|
+
* Handles misheard voice commands with intelligent correction
|
|
4
|
+
*/
|
|
5
|
+
export interface FuzzyMatchResult {
|
|
6
|
+
command: string;
|
|
7
|
+
confidence: number;
|
|
8
|
+
original: string;
|
|
9
|
+
}
|
|
10
|
+
export interface CommandSimilarityMap {
|
|
11
|
+
[key: string]: string[];
|
|
12
|
+
}
|
|
13
|
+
export declare class FuzzyMatcher {
|
|
14
|
+
private knownCommands;
|
|
15
|
+
private similarityMap;
|
|
16
|
+
/**
|
|
17
|
+
* Calculate Levenshtein distance between two strings
|
|
18
|
+
*/
|
|
19
|
+
private levenshteinDistance;
|
|
20
|
+
/**
|
|
21
|
+
* Find closest matching command using fuzzy matching
|
|
22
|
+
*/
|
|
23
|
+
findClosestMatch(input: string): FuzzyMatchResult;
|
|
24
|
+
/**
|
|
25
|
+
* Check if input matches a known pattern exactly or closely
|
|
26
|
+
*/
|
|
27
|
+
matchesKnownPattern(input: string): boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Add custom commands to the matcher
|
|
30
|
+
*/
|
|
31
|
+
addCommands(commands: string[]): void;
|
|
32
|
+
/**
|
|
33
|
+
* Add custom similarity mappings
|
|
34
|
+
*/
|
|
35
|
+
addSimilarities(mappings: CommandSimilarityMap): void;
|
|
36
|
+
/**
|
|
37
|
+
* Get all known commands
|
|
38
|
+
*/
|
|
39
|
+
getKnownCommands(): string[];
|
|
40
|
+
}
|
|
41
|
+
//# sourceMappingURL=fuzzy-matcher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fuzzy-matcher.d.ts","sourceRoot":"","sources":["../src/fuzzy-matcher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;CACzB;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,aAAa,CAqBnB;IAEF,OAAO,CAAC,aAAa,CAgBnB;IAEF;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB;IAiDjD;;OAEG;IACH,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAa3C;;OAEG;IACH,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI;IAIrC;;OAEG;IACH,eAAe,CAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI;IAIrD;;OAEG;IACH,gBAAgB,IAAI,MAAM,EAAE;CAG7B"}
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fuzzy Command Matcher
|
|
3
|
+
* Handles misheard voice commands with intelligent correction
|
|
4
|
+
*/
|
|
5
|
+
export class FuzzyMatcher {
|
|
6
|
+
knownCommands = [
|
|
7
|
+
'scroll down',
|
|
8
|
+
'scroll up',
|
|
9
|
+
'scroll to top',
|
|
10
|
+
'scroll to bottom',
|
|
11
|
+
'go to home',
|
|
12
|
+
'go to features',
|
|
13
|
+
'go to about',
|
|
14
|
+
'go to contact',
|
|
15
|
+
'go to pricing',
|
|
16
|
+
'go to documentation',
|
|
17
|
+
'take me to home',
|
|
18
|
+
'take me to features',
|
|
19
|
+
'take me to about',
|
|
20
|
+
'take me to contact',
|
|
21
|
+
'fill my name',
|
|
22
|
+
'fill my email',
|
|
23
|
+
'fill my phone',
|
|
24
|
+
'stop listening',
|
|
25
|
+
'pause',
|
|
26
|
+
'quiet',
|
|
27
|
+
];
|
|
28
|
+
similarityMap = {
|
|
29
|
+
scroll: ['stroll', 'install', 'crawl', 'role', 'scroll', 'role'],
|
|
30
|
+
top: ['dog', 'pop', 'cop', 'hop', 'top', 'tap', 'taco', 'stop'],
|
|
31
|
+
bottom: ['button', 'gotten', 'cotton', 'bottom', 'modern'],
|
|
32
|
+
down: ['town', 'noun', 'gown', 'down', 'done'],
|
|
33
|
+
up: ['cup', 'up', 'app'],
|
|
34
|
+
features: ['teachers', 'creatures', 'pictures', 'features'],
|
|
35
|
+
contact: ['contract', 'context', 'conduct', 'contact'],
|
|
36
|
+
about: ['a boat', 'shout', 'doubt', 'about', 'the boat'],
|
|
37
|
+
home: ['home', 'rome', 'ohm'],
|
|
38
|
+
'go to': ['go to', 'goto', 'go too', 'got to'],
|
|
39
|
+
fill: ['fill', 'feel', 'field'],
|
|
40
|
+
name: ['name', 'aim', 'main'],
|
|
41
|
+
email: ['email', 'e mail', 'meal'],
|
|
42
|
+
phone: ['phone', 'foam', 'zone'],
|
|
43
|
+
stop: ['stop', 'top', 'pause', 'quiet'],
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Calculate Levenshtein distance between two strings
|
|
47
|
+
*/
|
|
48
|
+
levenshteinDistance(a, b) {
|
|
49
|
+
const matrix = [];
|
|
50
|
+
for (let i = 0; i <= b.length; i++) {
|
|
51
|
+
matrix[i] = [i];
|
|
52
|
+
}
|
|
53
|
+
for (let j = 0; j <= a.length; j++) {
|
|
54
|
+
matrix[0][j] = j;
|
|
55
|
+
}
|
|
56
|
+
for (let i = 1; i <= b.length; i++) {
|
|
57
|
+
for (let j = 1; j <= a.length; j++) {
|
|
58
|
+
if (b.charAt(i - 1) === a.charAt(j - 1)) {
|
|
59
|
+
matrix[i][j] = matrix[i - 1][j - 1];
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, // substitution
|
|
63
|
+
matrix[i][j - 1] + 1, // insertion
|
|
64
|
+
matrix[i - 1][j] + 1 // deletion
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return matrix[b.length][a.length];
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Find closest matching command using fuzzy matching
|
|
73
|
+
*/
|
|
74
|
+
findClosestMatch(input) {
|
|
75
|
+
input = input.toLowerCase().trim();
|
|
76
|
+
let bestMatch = '';
|
|
77
|
+
let bestScore = 0;
|
|
78
|
+
// First, try direct Levenshtein distance matching
|
|
79
|
+
for (const cmd of this.knownCommands) {
|
|
80
|
+
const distance = this.levenshteinDistance(input, cmd);
|
|
81
|
+
const maxLength = Math.max(input.length, cmd.length);
|
|
82
|
+
const score = 1 - distance / maxLength;
|
|
83
|
+
if (score > bestScore) {
|
|
84
|
+
bestScore = score;
|
|
85
|
+
bestMatch = cmd;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
// Then, check word-level similarity mapping
|
|
89
|
+
for (const [correct, variants] of Object.entries(this.similarityMap)) {
|
|
90
|
+
for (const variant of variants) {
|
|
91
|
+
if (input.includes(variant) && variant !== correct) {
|
|
92
|
+
// Found a similar word, try correcting it
|
|
93
|
+
const correctedCommand = input.replace(new RegExp(variant, 'gi'), correct);
|
|
94
|
+
// Check if corrected command is closer to a known command
|
|
95
|
+
for (const cmd of this.knownCommands) {
|
|
96
|
+
const distance = this.levenshteinDistance(correctedCommand, cmd);
|
|
97
|
+
const maxLength = Math.max(correctedCommand.length, cmd.length);
|
|
98
|
+
const score = 1 - distance / maxLength;
|
|
99
|
+
// Boost score for similarity map matches
|
|
100
|
+
const boostedScore = Math.min(score * 1.15, 1.0);
|
|
101
|
+
if (boostedScore > bestScore) {
|
|
102
|
+
bestScore = boostedScore;
|
|
103
|
+
bestMatch = cmd;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
return {
|
|
110
|
+
command: bestMatch,
|
|
111
|
+
confidence: bestScore,
|
|
112
|
+
original: input,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Check if input matches a known pattern exactly or closely
|
|
117
|
+
*/
|
|
118
|
+
matchesKnownPattern(input) {
|
|
119
|
+
input = input.toLowerCase();
|
|
120
|
+
const patterns = [
|
|
121
|
+
/\b(go to|take me to|navigate to)\b/,
|
|
122
|
+
/\b(scroll)\b/,
|
|
123
|
+
/\b(name|email|phone)\b/,
|
|
124
|
+
/\b(stop|pause|quiet)\b/,
|
|
125
|
+
/\b(fill|enter)\b/,
|
|
126
|
+
];
|
|
127
|
+
return patterns.some(pattern => pattern.test(input));
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Add custom commands to the matcher
|
|
131
|
+
*/
|
|
132
|
+
addCommands(commands) {
|
|
133
|
+
this.knownCommands.push(...commands);
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Add custom similarity mappings
|
|
137
|
+
*/
|
|
138
|
+
addSimilarities(mappings) {
|
|
139
|
+
this.similarityMap = { ...this.similarityMap, ...mappings };
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Get all known commands
|
|
143
|
+
*/
|
|
144
|
+
getKnownCommands() {
|
|
145
|
+
return [...this.knownCommands];
|
|
146
|
+
}
|
|
147
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export { AzureOpenAIClient } from './client.js';
|
|
2
|
+
export { IntentClassifier } from './intent/classifier.js';
|
|
3
|
+
export { ActionRouter } from './actions/index.js';
|
|
4
|
+
export { FuzzyMatcher } from './fuzzy-matcher.js';
|
|
5
|
+
export { ResponseGenerator } from './responses/generator.js';
|
|
6
|
+
export * from './intent/types.js';
|
|
7
|
+
export * from './actions/index.js';
|
|
8
|
+
export * from './prompts/index.js';
|
|
9
|
+
export type { AzureOpenAIConfig, ChatMessage, CompletionOptions } from './client.js';
|
|
10
|
+
export type { FuzzyMatchResult, CommandSimilarityMap } from './fuzzy-matcher.js';
|
|
11
|
+
export type { ResponseTemplates } from './responses/generator.js';
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAE7D,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AAEnC,YAAY,EAAE,iBAAiB,EAAE,WAAW,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACrF,YAAY,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AACjF,YAAY,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { AzureOpenAIClient } from './client.js';
|
|
2
|
+
export { IntentClassifier } from './intent/classifier.js';
|
|
3
|
+
export { ActionRouter } from './actions/index.js';
|
|
4
|
+
export { FuzzyMatcher } from './fuzzy-matcher.js';
|
|
5
|
+
export { ResponseGenerator } from './responses/generator.js';
|
|
6
|
+
export * from './intent/types.js';
|
|
7
|
+
export * from './actions/index.js';
|
|
8
|
+
export * from './prompts/index.js';
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { AzureOpenAIClient } from '../client.js';
|
|
2
|
+
import { Intent, IntentResult } from './types.js';
|
|
3
|
+
export declare class IntentClassifier {
|
|
4
|
+
private client;
|
|
5
|
+
private fuzzyMatcher;
|
|
6
|
+
constructor(client: AzureOpenAIClient);
|
|
7
|
+
classify(userInput: string, context?: {
|
|
8
|
+
pageTitle?: string;
|
|
9
|
+
availableLinks?: string[];
|
|
10
|
+
}): Promise<IntentResult>;
|
|
11
|
+
/**
|
|
12
|
+
* Parse a fuzzy matched command string into an IntentResult
|
|
13
|
+
*/
|
|
14
|
+
private parseCommandToIntent;
|
|
15
|
+
private buildSystemPrompt;
|
|
16
|
+
private buildUserPrompt;
|
|
17
|
+
private parseIntentResponse;
|
|
18
|
+
extractEntities(userInput: string, intentType: Intent): Promise<Record<string, any>>;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=classifier.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../src/intent/classifier.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAe,MAAM,cAAc,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGlD,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,EAAE,iBAAiB;IAK/B,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAmDrH;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAyD5B,OAAO,CAAC,iBAAiB;IAgCzB,OAAO,CAAC,eAAe;IAIvB,OAAO,CAAC,mBAAmB;IAmCrB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CA4B3F"}
|
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
import { Intent } from './types.js';
|
|
2
|
+
import { FuzzyMatcher } from '../fuzzy-matcher.js';
|
|
3
|
+
export class IntentClassifier {
|
|
4
|
+
client;
|
|
5
|
+
fuzzyMatcher;
|
|
6
|
+
constructor(client) {
|
|
7
|
+
this.client = client;
|
|
8
|
+
this.fuzzyMatcher = new FuzzyMatcher();
|
|
9
|
+
}
|
|
10
|
+
async classify(userInput, context) {
|
|
11
|
+
// First try fuzzy matching for known commands
|
|
12
|
+
const fuzzyMatch = this.fuzzyMatcher.findClosestMatch(userInput);
|
|
13
|
+
// If high confidence match, use it directly
|
|
14
|
+
if (fuzzyMatch.confidence > 0.85) {
|
|
15
|
+
console.log(`[IntentClassifier] Fuzzy match found: ${fuzzyMatch.command} (confidence: ${fuzzyMatch.confidence})`);
|
|
16
|
+
// Parse the fuzzy matched command into an intent
|
|
17
|
+
return this.parseCommandToIntent(fuzzyMatch.command, userInput, fuzzyMatch.confidence);
|
|
18
|
+
}
|
|
19
|
+
// Otherwise, use LLM for classification
|
|
20
|
+
const systemPrompt = this.buildSystemPrompt(context);
|
|
21
|
+
const userPrompt = this.buildUserPrompt(userInput);
|
|
22
|
+
const messages = [
|
|
23
|
+
{ role: 'system', content: systemPrompt },
|
|
24
|
+
{ role: 'user', content: userPrompt },
|
|
25
|
+
];
|
|
26
|
+
try {
|
|
27
|
+
const response = await this.client.chat({ messages, temperature: 0.3, maxTokens: 200 });
|
|
28
|
+
const result = this.parseIntentResponse(response, userInput);
|
|
29
|
+
// If LLM confidence is low, suggest fuzzy match
|
|
30
|
+
if (result.confidence < 0.6 && fuzzyMatch.confidence > 0.6) {
|
|
31
|
+
return {
|
|
32
|
+
...result,
|
|
33
|
+
suggestedCommand: fuzzyMatch.command,
|
|
34
|
+
suggestedConfidence: fuzzyMatch.confidence,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
return result;
|
|
38
|
+
}
|
|
39
|
+
catch (error) {
|
|
40
|
+
console.error('Intent classification failed:', error);
|
|
41
|
+
// Fallback to fuzzy matching
|
|
42
|
+
if (fuzzyMatch.confidence > 0.5) {
|
|
43
|
+
return this.parseCommandToIntent(fuzzyMatch.command, userInput, fuzzyMatch.confidence);
|
|
44
|
+
}
|
|
45
|
+
return {
|
|
46
|
+
intent: Intent.UNKNOWN,
|
|
47
|
+
confidence: 0,
|
|
48
|
+
entities: {},
|
|
49
|
+
rawText: userInput,
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Parse a fuzzy matched command string into an IntentResult
|
|
55
|
+
*/
|
|
56
|
+
parseCommandToIntent(command, rawText, confidence) {
|
|
57
|
+
command = command.toLowerCase();
|
|
58
|
+
// Scroll commands
|
|
59
|
+
if (command.includes('scroll')) {
|
|
60
|
+
let direction = 'down';
|
|
61
|
+
if (command.includes('up'))
|
|
62
|
+
direction = 'up';
|
|
63
|
+
else if (command.includes('top'))
|
|
64
|
+
direction = 'top';
|
|
65
|
+
else if (command.includes('bottom'))
|
|
66
|
+
direction = 'bottom';
|
|
67
|
+
return {
|
|
68
|
+
intent: Intent.SCROLL,
|
|
69
|
+
confidence,
|
|
70
|
+
entities: { direction },
|
|
71
|
+
rawText,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
// Navigation commands
|
|
75
|
+
if (command.includes('go to') || command.includes('take me to')) {
|
|
76
|
+
const target = command.replace(/^(go to|take me to)\s+/, '');
|
|
77
|
+
return {
|
|
78
|
+
intent: Intent.NAVIGATE,
|
|
79
|
+
confidence,
|
|
80
|
+
entities: { target },
|
|
81
|
+
rawText,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
// Form fill commands
|
|
85
|
+
if (command.includes('fill')) {
|
|
86
|
+
return {
|
|
87
|
+
intent: Intent.FORM_FILL,
|
|
88
|
+
confidence,
|
|
89
|
+
entities: {},
|
|
90
|
+
rawText,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
// Stop commands
|
|
94
|
+
if (command.includes('stop') || command.includes('pause') || command.includes('quiet')) {
|
|
95
|
+
return {
|
|
96
|
+
intent: Intent.UNKNOWN, // Special case - handled at UI level
|
|
97
|
+
confidence,
|
|
98
|
+
entities: { action: 'stop' },
|
|
99
|
+
rawText,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
return {
|
|
103
|
+
intent: Intent.UNKNOWN,
|
|
104
|
+
confidence: 0,
|
|
105
|
+
entities: {},
|
|
106
|
+
rawText,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
buildSystemPrompt(context) {
|
|
110
|
+
let prompt = `You are an intent classifier for a voice navigation system. Classify user voice commands into one of these intents:
|
|
111
|
+
|
|
112
|
+
- NAVIGATE: User wants to go to a different page (e.g., "go to products", "take me to about us")
|
|
113
|
+
- FORM_FILL: User wants to fill a form field (e.g., "my name is John", "enter email as test@test.com")
|
|
114
|
+
- QUESTION: User is asking a question about the website (e.g., "what is your refund policy?")
|
|
115
|
+
- READ_PAGE: User wants to hear page content (e.g., "read this page", "what's on this page?")
|
|
116
|
+
- CLICK: User wants to click something (e.g., "click the submit button", "press login")
|
|
117
|
+
- SCROLL: User wants to scroll (e.g., "scroll down", "go to top")
|
|
118
|
+
- SEARCH: User wants to search (e.g., "search for laptops")
|
|
119
|
+
- UNKNOWN: None of the above
|
|
120
|
+
|
|
121
|
+
Respond in this exact JSON format:
|
|
122
|
+
{
|
|
123
|
+
"intent": "NAVIGATE",
|
|
124
|
+
"confidence": 0.95,
|
|
125
|
+
"entities": {
|
|
126
|
+
"target": "products"
|
|
127
|
+
}
|
|
128
|
+
}`;
|
|
129
|
+
if (context?.pageTitle) {
|
|
130
|
+
prompt += `\n\nCurrent page: ${context.pageTitle}`;
|
|
131
|
+
}
|
|
132
|
+
if (context?.availableLinks && context.availableLinks.length > 0) {
|
|
133
|
+
prompt += `\n\nAvailable navigation links: ${context.availableLinks.slice(0, 10).join(', ')}`;
|
|
134
|
+
}
|
|
135
|
+
return prompt;
|
|
136
|
+
}
|
|
137
|
+
buildUserPrompt(userInput) {
|
|
138
|
+
return `Classify this command: "${userInput}"`;
|
|
139
|
+
}
|
|
140
|
+
parseIntentResponse(response, rawText) {
|
|
141
|
+
try {
|
|
142
|
+
const cleaned = response.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
|
143
|
+
const parsed = JSON.parse(cleaned);
|
|
144
|
+
const intentMap = {
|
|
145
|
+
NAVIGATE: Intent.NAVIGATE,
|
|
146
|
+
FORM_FILL: Intent.FORM_FILL,
|
|
147
|
+
QUESTION: Intent.QUESTION,
|
|
148
|
+
READ_PAGE: Intent.READ_PAGE,
|
|
149
|
+
CLICK: Intent.CLICK,
|
|
150
|
+
SCROLL: Intent.SCROLL,
|
|
151
|
+
SEARCH: Intent.SEARCH,
|
|
152
|
+
UNKNOWN: Intent.UNKNOWN,
|
|
153
|
+
};
|
|
154
|
+
const intent = intentMap[parsed.intent?.toUpperCase()] || Intent.UNKNOWN;
|
|
155
|
+
return {
|
|
156
|
+
intent,
|
|
157
|
+
confidence: parsed.confidence || 0.5,
|
|
158
|
+
entities: parsed.entities || {},
|
|
159
|
+
rawText,
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
console.error('Failed to parse intent response:', error);
|
|
164
|
+
return {
|
|
165
|
+
intent: Intent.UNKNOWN,
|
|
166
|
+
confidence: 0,
|
|
167
|
+
entities: {},
|
|
168
|
+
rawText,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
async extractEntities(userInput, intentType) {
|
|
173
|
+
const prompts = {
|
|
174
|
+
[Intent.NAVIGATE]: `Extract the target page/section from: "${userInput}". Return JSON: {"target": "page name"}`,
|
|
175
|
+
[Intent.FORM_FILL]: `Extract field name and value from: "${userInput}". Return JSON: {"fieldName": "name", "fieldValue": "value"}`,
|
|
176
|
+
[Intent.QUESTION]: `Extract the question from: "${userInput}". Return JSON: {"question": "the question"}`,
|
|
177
|
+
[Intent.READ_PAGE]: `Extract section if mentioned: "${userInput}". Return JSON: {"section": "section name or null"}`,
|
|
178
|
+
[Intent.CLICK]: `Extract what to click from: "${userInput}". Return JSON: {"target": "element description"}`,
|
|
179
|
+
[Intent.SCROLL]: `Extract scroll direction from: "${userInput}". Return JSON: {"direction": "up|down|top|bottom"}`,
|
|
180
|
+
[Intent.SEARCH]: `Extract search query from: "${userInput}". Return JSON: {"query": "search terms"}`,
|
|
181
|
+
[Intent.UNKNOWN]: `{}`,
|
|
182
|
+
};
|
|
183
|
+
const prompt = prompts[intentType] || '{}';
|
|
184
|
+
try {
|
|
185
|
+
const response = await this.client.chat({
|
|
186
|
+
messages: [{ role: 'user', content: prompt }],
|
|
187
|
+
temperature: 0.2,
|
|
188
|
+
maxTokens: 100,
|
|
189
|
+
});
|
|
190
|
+
const cleaned = response.replace(/```json\n?/g, '').replace(/```\n?/g, '').trim();
|
|
191
|
+
return JSON.parse(cleaned);
|
|
192
|
+
}
|
|
193
|
+
catch (error) {
|
|
194
|
+
console.error('Entity extraction failed:', error);
|
|
195
|
+
return {};
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare enum Intent {
|
|
3
|
+
NAVIGATE = "navigate",
|
|
4
|
+
FORM_FILL = "form_fill",
|
|
5
|
+
QUESTION = "question",
|
|
6
|
+
READ_PAGE = "read_page",
|
|
7
|
+
SCROLL = "scroll",
|
|
8
|
+
CLICK = "click",
|
|
9
|
+
SEARCH = "search",
|
|
10
|
+
UNKNOWN = "unknown"
|
|
11
|
+
}
|
|
12
|
+
export declare const IntentSchema: z.ZodObject<{
|
|
13
|
+
intent: z.ZodNativeEnum<typeof Intent>;
|
|
14
|
+
confidence: z.ZodNumber;
|
|
15
|
+
entities: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
|
|
16
|
+
rawText: z.ZodString;
|
|
17
|
+
suggestedCommand: z.ZodOptional<z.ZodString>;
|
|
18
|
+
suggestedConfidence: z.ZodOptional<z.ZodNumber>;
|
|
19
|
+
}, "strip", z.ZodTypeAny, {
|
|
20
|
+
intent: Intent;
|
|
21
|
+
confidence: number;
|
|
22
|
+
rawText: string;
|
|
23
|
+
entities?: Record<string, any> | undefined;
|
|
24
|
+
suggestedCommand?: string | undefined;
|
|
25
|
+
suggestedConfidence?: number | undefined;
|
|
26
|
+
}, {
|
|
27
|
+
intent: Intent;
|
|
28
|
+
confidence: number;
|
|
29
|
+
rawText: string;
|
|
30
|
+
entities?: Record<string, any> | undefined;
|
|
31
|
+
suggestedCommand?: string | undefined;
|
|
32
|
+
suggestedConfidence?: number | undefined;
|
|
33
|
+
}>;
|
|
34
|
+
export type IntentResult = z.infer<typeof IntentSchema>;
|
|
35
|
+
export interface NavigateIntent {
|
|
36
|
+
intent: Intent.NAVIGATE;
|
|
37
|
+
targetPage: string;
|
|
38
|
+
linkText?: string;
|
|
39
|
+
}
|
|
40
|
+
export interface FormFillIntent {
|
|
41
|
+
intent: Intent.FORM_FILL;
|
|
42
|
+
fieldName: string;
|
|
43
|
+
fieldValue: string;
|
|
44
|
+
fieldType?: string;
|
|
45
|
+
}
|
|
46
|
+
export interface QuestionIntent {
|
|
47
|
+
intent: Intent.QUESTION;
|
|
48
|
+
question: string;
|
|
49
|
+
}
|
|
50
|
+
export interface ReadPageIntent {
|
|
51
|
+
intent: Intent.READ_PAGE;
|
|
52
|
+
section?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface ClickIntent {
|
|
55
|
+
intent: Intent.CLICK;
|
|
56
|
+
target: string;
|
|
57
|
+
}
|
|
58
|
+
export interface ScrollIntent {
|
|
59
|
+
intent: Intent.SCROLL;
|
|
60
|
+
direction: 'up' | 'down' | 'top' | 'bottom';
|
|
61
|
+
}
|
|
62
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/intent/types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,oBAAY,MAAM;IAChB,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,QAAQ,aAAa;IACrB,SAAS,cAAc;IACvB,MAAM,WAAW;IACjB,KAAK,UAAU;IACf,MAAM,WAAW;IACjB,OAAO,YAAY;CACpB;AAED,eAAO,MAAM,YAAY;;;;;;;;;;;;;;;;;;;;;EAOvB,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,YAAY,CAAC,CAAC;AAExD,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC,SAAS,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC;IACtB,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,CAAC;CAC7C"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export var Intent;
|
|
3
|
+
(function (Intent) {
|
|
4
|
+
Intent["NAVIGATE"] = "navigate";
|
|
5
|
+
Intent["FORM_FILL"] = "form_fill";
|
|
6
|
+
Intent["QUESTION"] = "question";
|
|
7
|
+
Intent["READ_PAGE"] = "read_page";
|
|
8
|
+
Intent["SCROLL"] = "scroll";
|
|
9
|
+
Intent["CLICK"] = "click";
|
|
10
|
+
Intent["SEARCH"] = "search";
|
|
11
|
+
Intent["UNKNOWN"] = "unknown";
|
|
12
|
+
})(Intent || (Intent = {}));
|
|
13
|
+
export const IntentSchema = z.object({
|
|
14
|
+
intent: z.nativeEnum(Intent),
|
|
15
|
+
confidence: z.number().min(0).max(1),
|
|
16
|
+
entities: z.record(z.any()).optional(),
|
|
17
|
+
rawText: z.string(),
|
|
18
|
+
suggestedCommand: z.string().optional(),
|
|
19
|
+
suggestedConfidence: z.number().min(0).max(1).optional(),
|
|
20
|
+
});
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export declare const INTENT_CLASSIFICATION_PROMPT = "You are an AI assistant for voice navigation. Classify user intents and help them navigate websites.";
|
|
2
|
+
export declare const FORM_FILL_PROMPT = "You are helping a user fill out a web form using voice commands. Extract field names and values from their speech.";
|
|
3
|
+
export declare const NAVIGATION_PROMPT = "You are helping a user navigate a website using voice. Match their request to available links or pages.";
|
|
4
|
+
export declare const QA_PROMPT = "You are answering user questions about a website. Provide concise, helpful answers based on the provided context.";
|
|
5
|
+
export declare const PAGE_SUMMARY_PROMPT = "Summarize this webpage content in 2-3 sentences for voice reading. Focus on the main purpose and key information.";
|
|
6
|
+
export declare function buildContextPrompt(context: {
|
|
7
|
+
pageTitle?: string;
|
|
8
|
+
headings?: string[];
|
|
9
|
+
links?: string[];
|
|
10
|
+
content?: string;
|
|
11
|
+
}): string;
|
|
12
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/prompts/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,4BAA4B,yGAAyG,CAAC;AAEnJ,eAAO,MAAM,gBAAgB,uHAAuH,CAAC;AAErJ,eAAO,MAAM,iBAAiB,4GAA4G,CAAC;AAE3I,eAAO,MAAM,SAAS,sHAAsH,CAAC;AAE7I,eAAO,MAAM,mBAAmB,sHAAsH,CAAC;AAEvJ,wBAAgB,kBAAkB,CAAC,OAAO,EAAE;IAC1C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAC;IACpB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,GAAG,MAAM,CAqBT"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export const INTENT_CLASSIFICATION_PROMPT = `You are an AI assistant for voice navigation. Classify user intents and help them navigate websites.`;
|
|
2
|
+
export const FORM_FILL_PROMPT = `You are helping a user fill out a web form using voice commands. Extract field names and values from their speech.`;
|
|
3
|
+
export const NAVIGATION_PROMPT = `You are helping a user navigate a website using voice. Match their request to available links or pages.`;
|
|
4
|
+
export const QA_PROMPT = `You are answering user questions about a website. Provide concise, helpful answers based on the provided context.`;
|
|
5
|
+
export const PAGE_SUMMARY_PROMPT = `Summarize this webpage content in 2-3 sentences for voice reading. Focus on the main purpose and key information.`;
|
|
6
|
+
export function buildContextPrompt(context) {
|
|
7
|
+
let prompt = 'Current page context:\n';
|
|
8
|
+
if (context.pageTitle) {
|
|
9
|
+
prompt += `Page title: ${context.pageTitle}\n`;
|
|
10
|
+
}
|
|
11
|
+
if (context.headings && context.headings.length > 0) {
|
|
12
|
+
prompt += `Headings: ${context.headings.slice(0, 5).join(', ')}\n`;
|
|
13
|
+
}
|
|
14
|
+
if (context.links && context.links.length > 0) {
|
|
15
|
+
prompt += `Available links: ${context.links.slice(0, 10).join(', ')}\n`;
|
|
16
|
+
}
|
|
17
|
+
if (context.content) {
|
|
18
|
+
const truncated = context.content.substring(0, 500);
|
|
19
|
+
prompt += `Content preview: ${truncated}...\n`;
|
|
20
|
+
}
|
|
21
|
+
return prompt;
|
|
22
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversational Response Generator
|
|
3
|
+
* Provides natural, friendly voice responses with variety
|
|
4
|
+
*/
|
|
5
|
+
export interface ResponseTemplates {
|
|
6
|
+
navigation: {
|
|
7
|
+
success: string[];
|
|
8
|
+
notFound: string[];
|
|
9
|
+
};
|
|
10
|
+
scroll: {
|
|
11
|
+
down: string[];
|
|
12
|
+
up: string[];
|
|
13
|
+
top: string[];
|
|
14
|
+
bottom: string[];
|
|
15
|
+
};
|
|
16
|
+
clarification: {
|
|
17
|
+
didYouMean: string;
|
|
18
|
+
notUnderstood: string;
|
|
19
|
+
suggestions: string;
|
|
20
|
+
cancelConfirmation: string;
|
|
21
|
+
};
|
|
22
|
+
formFill: {
|
|
23
|
+
name: string;
|
|
24
|
+
email: string;
|
|
25
|
+
phone: string;
|
|
26
|
+
partial: string;
|
|
27
|
+
};
|
|
28
|
+
stop: string[];
|
|
29
|
+
}
|
|
30
|
+
export declare class ResponseGenerator {
|
|
31
|
+
private templates;
|
|
32
|
+
/**
|
|
33
|
+
* Get a random response from an array for variety
|
|
34
|
+
*/
|
|
35
|
+
private getRandomResponse;
|
|
36
|
+
/**
|
|
37
|
+
* Replace placeholders in template strings
|
|
38
|
+
*/
|
|
39
|
+
private interpolate;
|
|
40
|
+
/**
|
|
41
|
+
* Generate navigation success response
|
|
42
|
+
*/
|
|
43
|
+
navigationSuccess(target: string): string;
|
|
44
|
+
/**
|
|
45
|
+
* Generate navigation not found response
|
|
46
|
+
*/
|
|
47
|
+
navigationNotFound(target: string): string;
|
|
48
|
+
/**
|
|
49
|
+
* Generate scroll response
|
|
50
|
+
*/
|
|
51
|
+
scroll(direction: 'up' | 'down' | 'top' | 'bottom'): string;
|
|
52
|
+
/**
|
|
53
|
+
* Generate "did you mean" clarification
|
|
54
|
+
*/
|
|
55
|
+
didYouMean(suggestion: string): string;
|
|
56
|
+
/**
|
|
57
|
+
* Generate not understood response
|
|
58
|
+
*/
|
|
59
|
+
notUnderstood(): string;
|
|
60
|
+
/**
|
|
61
|
+
* Generate suggestions response
|
|
62
|
+
*/
|
|
63
|
+
suggestions(): string;
|
|
64
|
+
/**
|
|
65
|
+
* Generate cancel confirmation response
|
|
66
|
+
*/
|
|
67
|
+
cancelConfirmation(): string;
|
|
68
|
+
/**
|
|
69
|
+
* Generate form fill response
|
|
70
|
+
*/
|
|
71
|
+
formFill(field: 'name' | 'email' | 'phone' | 'partial'): string;
|
|
72
|
+
/**
|
|
73
|
+
* Generate stop response
|
|
74
|
+
*/
|
|
75
|
+
stop(): string;
|
|
76
|
+
/**
|
|
77
|
+
* Update response templates (for customization)
|
|
78
|
+
*/
|
|
79
|
+
updateTemplates(templates: Partial<ResponseTemplates>): void;
|
|
80
|
+
}
|
|
81
|
+
//# sourceMappingURL=generator.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generator.d.ts","sourceRoot":"","sources":["../../src/responses/generator.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE;QACV,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IACF,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,EAAE,EAAE,MAAM,EAAE,CAAC;QACb,GAAG,EAAE,MAAM,EAAE,CAAC;QACd,MAAM,EAAE,MAAM,EAAE,CAAC;KAClB,CAAC;IACF,aAAa,EAAE;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,WAAW,EAAE,MAAM,CAAC;QACpB,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IACF,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAED,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,SAAS,CA+Df;IAEF;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAQnB;;OAEG;IACH,iBAAiB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAKzC;;OAEG;IACH,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM;IAK1C;;OAEG;IACH,MAAM,CAAC,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM;IAI3D;;OAEG;IACH,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM;IAItC;;OAEG;IACH,aAAa,IAAI,MAAM;IAIvB;;OAEG;IACH,WAAW,IAAI,MAAM;IAIrB;;OAEG;IACH,kBAAkB,IAAI,MAAM;IAI5B;;OAEG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,MAAM;IAI/D;;OAEG;IACH,IAAI,IAAI,MAAM;IAId;;OAEG;IACH,eAAe,CAAC,SAAS,EAAE,OAAO,CAAC,iBAAiB,CAAC,GAAG,IAAI;CAG7D"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Conversational Response Generator
|
|
3
|
+
* Provides natural, friendly voice responses with variety
|
|
4
|
+
*/
|
|
5
|
+
export class ResponseGenerator {
|
|
6
|
+
templates = {
|
|
7
|
+
navigation: {
|
|
8
|
+
success: [
|
|
9
|
+
"Sure, taking you to {target} now",
|
|
10
|
+
"Alright, heading to {target}",
|
|
11
|
+
"On my way to {target}",
|
|
12
|
+
"Got it, navigating to {target}",
|
|
13
|
+
"Let me take you to {target}",
|
|
14
|
+
],
|
|
15
|
+
notFound: [
|
|
16
|
+
"Hmm, I couldn't find {target}. Try saying something like 'go to features' or 'go to contact'",
|
|
17
|
+
"I don't see a {target} section. What else can I help you find?",
|
|
18
|
+
"Sorry, I couldn't locate {target}. You can try other sections available on this page",
|
|
19
|
+
],
|
|
20
|
+
},
|
|
21
|
+
scroll: {
|
|
22
|
+
down: [
|
|
23
|
+
"Scrolling down for you",
|
|
24
|
+
"Going down",
|
|
25
|
+
"Sure, scrolling down",
|
|
26
|
+
"Moving down the page",
|
|
27
|
+
"Let me scroll down",
|
|
28
|
+
],
|
|
29
|
+
up: [
|
|
30
|
+
"Scrolling up",
|
|
31
|
+
"Going back up",
|
|
32
|
+
"Sure, scrolling up",
|
|
33
|
+
"Moving up the page",
|
|
34
|
+
"Let me scroll up",
|
|
35
|
+
],
|
|
36
|
+
top: [
|
|
37
|
+
"Taking you to the top",
|
|
38
|
+
"Back to the top we go",
|
|
39
|
+
"Sure, going to the top",
|
|
40
|
+
"Heading to the top",
|
|
41
|
+
"Let me take you to the top",
|
|
42
|
+
],
|
|
43
|
+
bottom: [
|
|
44
|
+
"Scrolling to the bottom",
|
|
45
|
+
"Taking you all the way down",
|
|
46
|
+
"Sure, going to the bottom",
|
|
47
|
+
"Heading to the bottom",
|
|
48
|
+
"Let me scroll to the bottom",
|
|
49
|
+
],
|
|
50
|
+
},
|
|
51
|
+
clarification: {
|
|
52
|
+
didYouMean: "Did you mean '{suggestion}'? Say yes to confirm or try again",
|
|
53
|
+
notUnderstood: "I didn't quite catch that. Could you try saying it differently?",
|
|
54
|
+
suggestions: "You can try commands like 'scroll down', 'go to contact', or 'fill my name as John'",
|
|
55
|
+
cancelConfirmation: "No problem. What would you like to do instead?",
|
|
56
|
+
},
|
|
57
|
+
formFill: {
|
|
58
|
+
name: "Got it! I've filled in your name",
|
|
59
|
+
email: "Perfect! I've added your email",
|
|
60
|
+
phone: "All set! I've filled in your phone number",
|
|
61
|
+
partial: "I filled what I could. Want to add anything else?",
|
|
62
|
+
},
|
|
63
|
+
stop: [
|
|
64
|
+
"Alright, turning off the voice assistant",
|
|
65
|
+
"No problem, stopping now",
|
|
66
|
+
"Got it, voice assistant off",
|
|
67
|
+
"Sure, I'll stop listening",
|
|
68
|
+
],
|
|
69
|
+
};
|
|
70
|
+
/**
|
|
71
|
+
* Get a random response from an array for variety
|
|
72
|
+
*/
|
|
73
|
+
getRandomResponse(responses) {
|
|
74
|
+
return responses[Math.floor(Math.random() * responses.length)];
|
|
75
|
+
}
|
|
76
|
+
/**
|
|
77
|
+
* Replace placeholders in template strings
|
|
78
|
+
*/
|
|
79
|
+
interpolate(template, values) {
|
|
80
|
+
let result = template;
|
|
81
|
+
for (const [key, value] of Object.entries(values)) {
|
|
82
|
+
result = result.replace(new RegExp(`\\{${key}\\}`, 'g'), value);
|
|
83
|
+
}
|
|
84
|
+
return result;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Generate navigation success response
|
|
88
|
+
*/
|
|
89
|
+
navigationSuccess(target) {
|
|
90
|
+
const template = this.getRandomResponse(this.templates.navigation.success);
|
|
91
|
+
return this.interpolate(template, { target });
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Generate navigation not found response
|
|
95
|
+
*/
|
|
96
|
+
navigationNotFound(target) {
|
|
97
|
+
const template = this.getRandomResponse(this.templates.navigation.notFound);
|
|
98
|
+
return this.interpolate(template, { target });
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Generate scroll response
|
|
102
|
+
*/
|
|
103
|
+
scroll(direction) {
|
|
104
|
+
return this.getRandomResponse(this.templates.scroll[direction]);
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Generate "did you mean" clarification
|
|
108
|
+
*/
|
|
109
|
+
didYouMean(suggestion) {
|
|
110
|
+
return this.interpolate(this.templates.clarification.didYouMean, { suggestion });
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Generate not understood response
|
|
114
|
+
*/
|
|
115
|
+
notUnderstood() {
|
|
116
|
+
return this.templates.clarification.notUnderstood;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Generate suggestions response
|
|
120
|
+
*/
|
|
121
|
+
suggestions() {
|
|
122
|
+
return this.templates.clarification.suggestions;
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Generate cancel confirmation response
|
|
126
|
+
*/
|
|
127
|
+
cancelConfirmation() {
|
|
128
|
+
return this.templates.clarification.cancelConfirmation;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Generate form fill response
|
|
132
|
+
*/
|
|
133
|
+
formFill(field) {
|
|
134
|
+
return this.templates.formFill[field];
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Generate stop response
|
|
138
|
+
*/
|
|
139
|
+
stop() {
|
|
140
|
+
return this.getRandomResponse(this.templates.stop);
|
|
141
|
+
}
|
|
142
|
+
/**
|
|
143
|
+
* Update response templates (for customization)
|
|
144
|
+
*/
|
|
145
|
+
updateTemplates(templates) {
|
|
146
|
+
this.templates = { ...this.templates, ...templates };
|
|
147
|
+
}
|
|
148
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "voicepilot-llm-engine",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "LLM engine with fuzzy matching and conversational responses",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"files": ["dist"],
|
|
9
|
+
"exports": {
|
|
10
|
+
".": {
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"types": "./dist/index.d.ts"
|
|
13
|
+
}
|
|
14
|
+
},
|
|
15
|
+
"scripts": {
|
|
16
|
+
"build": "tsc",
|
|
17
|
+
"dev": "tsc --watch",
|
|
18
|
+
"clean": "rm -rf dist"
|
|
19
|
+
},
|
|
20
|
+
"dependencies": {
|
|
21
|
+
"openai": "^4.77.0",
|
|
22
|
+
"zod": "^3.24.1"
|
|
23
|
+
},
|
|
24
|
+
"devDependencies": {
|
|
25
|
+
"@types/node": "^22.10.2",
|
|
26
|
+
"@voicepilot/typescript-config": "workspace:*",
|
|
27
|
+
"typescript": "^5.7.2"
|
|
28
|
+
},
|
|
29
|
+
"publishConfig": {
|
|
30
|
+
"access": "public"
|
|
31
|
+
}
|
|
32
|
+
}
|