ruvector 0.1.78 ā 0.1.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ruvector/workers/code-analyzer.db +0 -0
- package/bin/cli.js +138 -0
- package/dist/workers/benchmark.d.ts +44 -0
- package/dist/workers/benchmark.d.ts.map +1 -0
- package/dist/workers/benchmark.js +230 -0
- package/dist/workers/index.d.ts +10 -0
- package/dist/workers/index.d.ts.map +1 -0
- package/dist/workers/index.js +25 -0
- package/dist/workers/native-worker.d.ts +76 -0
- package/dist/workers/native-worker.d.ts.map +1 -0
- package/dist/workers/native-worker.js +489 -0
- package/dist/workers/types.d.ts +69 -0
- package/dist/workers/types.d.ts.map +1 -0
- package/dist/workers/types.js +7 -0
- package/package.json +1 -1
|
Binary file
|
package/bin/cli.js
CHANGED
|
@@ -6212,6 +6212,144 @@ workersCmd.command('load-config')
|
|
|
6212
6212
|
|
|
6213
6213
|
console.log && false; // Force registration
|
|
6214
6214
|
|
|
6215
|
+
// Native Workers command group - Deep ruvector integration (no agentic-flow delegation)
|
|
6216
|
+
const nativeCmd = program.command('native').description('Native workers with deep ONNX/VectorDB integration (no external deps)');
|
|
6217
|
+
|
|
6218
|
+
nativeCmd.command('run')
|
|
6219
|
+
.description('Run a native worker type')
|
|
6220
|
+
.argument('<type>', 'Worker type: security, analysis, learning')
|
|
6221
|
+
.option('--path <path>', 'Target path to analyze', '.')
|
|
6222
|
+
.option('--json', 'Output as JSON')
|
|
6223
|
+
.action(async (type, opts) => {
|
|
6224
|
+
try {
|
|
6225
|
+
const { createSecurityWorker, createAnalysisWorker, createLearningWorker } = require('../dist/workers/native-worker.js');
|
|
6226
|
+
|
|
6227
|
+
let worker;
|
|
6228
|
+
switch (type) {
|
|
6229
|
+
case 'security':
|
|
6230
|
+
worker = createSecurityWorker();
|
|
6231
|
+
break;
|
|
6232
|
+
case 'analysis':
|
|
6233
|
+
worker = createAnalysisWorker();
|
|
6234
|
+
break;
|
|
6235
|
+
case 'learning':
|
|
6236
|
+
worker = createLearningWorker();
|
|
6237
|
+
break;
|
|
6238
|
+
default:
|
|
6239
|
+
console.error(chalk.red(`Unknown worker type: ${type}`));
|
|
6240
|
+
console.log(chalk.dim('Available types: security, analysis, learning'));
|
|
6241
|
+
return;
|
|
6242
|
+
}
|
|
6243
|
+
|
|
6244
|
+
console.log(chalk.cyan(`\nš§ Running native ${type} worker on ${opts.path}...\n`));
|
|
6245
|
+
const result = await worker.run(opts.path);
|
|
6246
|
+
|
|
6247
|
+
if (opts.json) {
|
|
6248
|
+
console.log(JSON.stringify(result, null, 2));
|
|
6249
|
+
} else {
|
|
6250
|
+
console.log(chalk.bold(`Worker: ${result.worker}`));
|
|
6251
|
+
console.log(chalk.dim(`Status: ${result.success ? chalk.green('ā Success') : chalk.red('ā Failed')}`));
|
|
6252
|
+
console.log(chalk.dim(`Time: ${result.totalTimeMs.toFixed(0)}ms\n`));
|
|
6253
|
+
|
|
6254
|
+
console.log(chalk.bold('Phases:'));
|
|
6255
|
+
for (const phase of result.phases) {
|
|
6256
|
+
const status = phase.success ? chalk.green('ā') : chalk.red('ā');
|
|
6257
|
+
console.log(` ${status} ${phase.phase} (${phase.timeMs.toFixed(0)}ms)`);
|
|
6258
|
+
if (phase.data) {
|
|
6259
|
+
const dataStr = JSON.stringify(phase.data);
|
|
6260
|
+
if (dataStr.length < 100) {
|
|
6261
|
+
console.log(chalk.dim(` ${dataStr}`));
|
|
6262
|
+
}
|
|
6263
|
+
}
|
|
6264
|
+
}
|
|
6265
|
+
|
|
6266
|
+
if (result.summary) {
|
|
6267
|
+
console.log(chalk.bold('\nSummary:'));
|
|
6268
|
+
console.log(` Files analyzed: ${result.summary.filesAnalyzed}`);
|
|
6269
|
+
console.log(` Patterns found: ${result.summary.patternsFound}`);
|
|
6270
|
+
console.log(` Embeddings: ${result.summary.embeddingsGenerated}`);
|
|
6271
|
+
console.log(` Vectors stored: ${result.summary.vectorsStored}`);
|
|
6272
|
+
|
|
6273
|
+
if (result.summary.findings.length > 0) {
|
|
6274
|
+
console.log(chalk.bold('\nFindings:'));
|
|
6275
|
+
const byType = { info: 0, warning: 0, error: 0, security: 0 };
|
|
6276
|
+
result.summary.findings.forEach(f => byType[f.type]++);
|
|
6277
|
+
if (byType.security > 0) console.log(chalk.red(` š Security: ${byType.security}`));
|
|
6278
|
+
if (byType.error > 0) console.log(chalk.red(` ā Errors: ${byType.error}`));
|
|
6279
|
+
if (byType.warning > 0) console.log(chalk.yellow(` ā ļø Warnings: ${byType.warning}`));
|
|
6280
|
+
if (byType.info > 0) console.log(chalk.blue(` ā¹ļø Info: ${byType.info}`));
|
|
6281
|
+
|
|
6282
|
+
// Show top findings
|
|
6283
|
+
console.log(chalk.dim('\nTop findings:'));
|
|
6284
|
+
result.summary.findings.slice(0, 5).forEach(f => {
|
|
6285
|
+
const icon = f.type === 'security' ? 'š' : f.type === 'warning' ? 'ā ļø' : 'ā¹ļø';
|
|
6286
|
+
console.log(chalk.dim(` ${icon} ${f.message.slice(0, 60)}${f.file ? ` (${path.basename(f.file)})` : ''}`));
|
|
6287
|
+
});
|
|
6288
|
+
}
|
|
6289
|
+
}
|
|
6290
|
+
}
|
|
6291
|
+
} catch (e) {
|
|
6292
|
+
console.error(chalk.red('Native worker failed:'), e.message);
|
|
6293
|
+
if (e.stack) console.error(chalk.dim(e.stack));
|
|
6294
|
+
}
|
|
6295
|
+
});
|
|
6296
|
+
|
|
6297
|
+
nativeCmd.command('benchmark')
|
|
6298
|
+
.description('Run performance benchmark suite')
|
|
6299
|
+
.option('--path <path>', 'Target path for worker benchmarks', '.')
|
|
6300
|
+
.option('--embeddings-only', 'Only benchmark embeddings')
|
|
6301
|
+
.option('--workers-only', 'Only benchmark workers')
|
|
6302
|
+
.action(async (opts) => {
|
|
6303
|
+
try {
|
|
6304
|
+
const benchmark = require('../dist/workers/benchmark.js');
|
|
6305
|
+
|
|
6306
|
+
if (opts.embeddingsOnly) {
|
|
6307
|
+
console.log(chalk.cyan('\nš Benchmarking ONNX Embeddings...\n'));
|
|
6308
|
+
const results = await benchmark.benchmarkEmbeddings(10);
|
|
6309
|
+
console.log(benchmark.formatBenchmarkResults(results));
|
|
6310
|
+
} else if (opts.workersOnly) {
|
|
6311
|
+
console.log(chalk.cyan('\nš§ Benchmarking Native Workers...\n'));
|
|
6312
|
+
const results = await benchmark.benchmarkWorkers(opts.path);
|
|
6313
|
+
console.log(benchmark.formatBenchmarkResults(results));
|
|
6314
|
+
} else {
|
|
6315
|
+
await benchmark.runFullBenchmark(opts.path);
|
|
6316
|
+
}
|
|
6317
|
+
} catch (e) {
|
|
6318
|
+
console.error(chalk.red('Benchmark failed:'), e.message);
|
|
6319
|
+
if (e.stack) console.error(chalk.dim(e.stack));
|
|
6320
|
+
}
|
|
6321
|
+
});
|
|
6322
|
+
|
|
6323
|
+
nativeCmd.command('list')
|
|
6324
|
+
.description('List available native worker types')
|
|
6325
|
+
.action(() => {
|
|
6326
|
+
console.log(chalk.cyan('\nš§ Native Worker Types\n'));
|
|
6327
|
+
console.log(chalk.bold('security'));
|
|
6328
|
+
console.log(chalk.dim(' Security vulnerability scanner'));
|
|
6329
|
+
console.log(chalk.dim(' Phases: file-discovery ā security-scan ā summarization'));
|
|
6330
|
+
console.log(chalk.dim(' No ONNX/VectorDB required\n'));
|
|
6331
|
+
|
|
6332
|
+
console.log(chalk.bold('analysis'));
|
|
6333
|
+
console.log(chalk.dim(' Full code analysis with embeddings'));
|
|
6334
|
+
console.log(chalk.dim(' Phases: file-discovery ā pattern-extraction ā embedding-generation'));
|
|
6335
|
+
console.log(chalk.dim(' ā vector-storage ā complexity-analysis ā summarization'));
|
|
6336
|
+
console.log(chalk.dim(' Requires: ONNX embedder, VectorDB\n'));
|
|
6337
|
+
|
|
6338
|
+
console.log(chalk.bold('learning'));
|
|
6339
|
+
console.log(chalk.dim(' Pattern learning with vector storage'));
|
|
6340
|
+
console.log(chalk.dim(' Phases: file-discovery ā pattern-extraction ā embedding-generation'));
|
|
6341
|
+
console.log(chalk.dim(' ā vector-storage ā summarization'));
|
|
6342
|
+
console.log(chalk.dim(' Requires: ONNX embedder, VectorDB, Intelligence memory\n'));
|
|
6343
|
+
|
|
6344
|
+
console.log(chalk.bold('Available Phases:'));
|
|
6345
|
+
const phases = [
|
|
6346
|
+
'file-discovery', 'pattern-extraction', 'embedding-generation',
|
|
6347
|
+
'vector-storage', 'similarity-search', 'security-scan',
|
|
6348
|
+
'complexity-analysis', 'summarization'
|
|
6349
|
+
];
|
|
6350
|
+
phases.forEach(p => console.log(chalk.dim(` ⢠${p}`)));
|
|
6351
|
+
});
|
|
6352
|
+
|
|
6215
6353
|
// MCP Server command
|
|
6216
6354
|
const mcpCmd = program.command('mcp').description('MCP (Model Context Protocol) server for Claude Code integration');
|
|
6217
6355
|
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Worker Benchmark Suite for RuVector
|
|
3
|
+
*
|
|
4
|
+
* Measures performance of:
|
|
5
|
+
* - ONNX embedding generation (single vs batch)
|
|
6
|
+
* - Vector storage and search
|
|
7
|
+
* - Phase execution times
|
|
8
|
+
* - Worker end-to-end throughput
|
|
9
|
+
*/
|
|
10
|
+
import { BenchmarkResult } from './types';
|
|
11
|
+
/**
|
|
12
|
+
* Benchmark ONNX embedding generation
|
|
13
|
+
*/
|
|
14
|
+
export declare function benchmarkEmbeddings(iterations?: number): Promise<BenchmarkResult[]>;
|
|
15
|
+
/**
|
|
16
|
+
* Benchmark worker execution
|
|
17
|
+
*/
|
|
18
|
+
export declare function benchmarkWorkers(targetPath?: string): Promise<BenchmarkResult[]>;
|
|
19
|
+
/**
|
|
20
|
+
* Benchmark individual phases
|
|
21
|
+
*/
|
|
22
|
+
export declare function benchmarkPhases(targetPath?: string): Promise<BenchmarkResult[]>;
|
|
23
|
+
/**
|
|
24
|
+
* Format benchmark results as table
|
|
25
|
+
*/
|
|
26
|
+
export declare function formatBenchmarkResults(results: BenchmarkResult[]): string;
|
|
27
|
+
/**
|
|
28
|
+
* Run full benchmark suite
|
|
29
|
+
*/
|
|
30
|
+
export declare function runFullBenchmark(targetPath?: string): Promise<{
|
|
31
|
+
embeddings: BenchmarkResult[];
|
|
32
|
+
phases: BenchmarkResult[];
|
|
33
|
+
workers: BenchmarkResult[];
|
|
34
|
+
summary: string;
|
|
35
|
+
}>;
|
|
36
|
+
declare const _default: {
|
|
37
|
+
benchmarkEmbeddings: typeof benchmarkEmbeddings;
|
|
38
|
+
benchmarkWorkers: typeof benchmarkWorkers;
|
|
39
|
+
benchmarkPhases: typeof benchmarkPhases;
|
|
40
|
+
runFullBenchmark: typeof runFullBenchmark;
|
|
41
|
+
formatBenchmarkResults: typeof formatBenchmarkResults;
|
|
42
|
+
};
|
|
43
|
+
export default _default;
|
|
44
|
+
//# sourceMappingURL=benchmark.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"benchmark.d.ts","sourceRoot":"","sources":["../../src/workers/benchmark.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AA4C1C;;GAEG;AACH,wBAAsB,mBAAmB,CAAC,UAAU,GAAE,MAAW,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAwE7F;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CA4B3F;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAsD1F;AAED;;GAEG;AACH,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,MAAM,CAwBzE;AAED;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC;IACxE,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,MAAM,EAAE,eAAe,EAAE,CAAC;IAC1B,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC,CAyCD;;;;;;;;AAED,wBAME"}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Worker Benchmark Suite for RuVector
|
|
4
|
+
*
|
|
5
|
+
* Measures performance of:
|
|
6
|
+
* - ONNX embedding generation (single vs batch)
|
|
7
|
+
* - Vector storage and search
|
|
8
|
+
* - Phase execution times
|
|
9
|
+
* - Worker end-to-end throughput
|
|
10
|
+
*/
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.benchmarkEmbeddings = benchmarkEmbeddings;
|
|
13
|
+
exports.benchmarkWorkers = benchmarkWorkers;
|
|
14
|
+
exports.benchmarkPhases = benchmarkPhases;
|
|
15
|
+
exports.formatBenchmarkResults = formatBenchmarkResults;
|
|
16
|
+
exports.runFullBenchmark = runFullBenchmark;
|
|
17
|
+
const perf_hooks_1 = require("perf_hooks");
|
|
18
|
+
const native_worker_1 = require("./native-worker");
|
|
19
|
+
const onnx_embedder_1 = require("../core/onnx-embedder");
|
|
20
|
+
/**
|
|
21
|
+
* Run a benchmark function multiple times and collect stats
|
|
22
|
+
*/
|
|
23
|
+
async function runBenchmark(name, fn, iterations = 10, warmup = 2) {
|
|
24
|
+
// Warmup runs
|
|
25
|
+
for (let i = 0; i < warmup; i++) {
|
|
26
|
+
await fn();
|
|
27
|
+
}
|
|
28
|
+
// Actual benchmark runs
|
|
29
|
+
const times = [];
|
|
30
|
+
for (let i = 0; i < iterations; i++) {
|
|
31
|
+
const start = perf_hooks_1.performance.now();
|
|
32
|
+
await fn();
|
|
33
|
+
times.push(perf_hooks_1.performance.now() - start);
|
|
34
|
+
}
|
|
35
|
+
// Calculate statistics
|
|
36
|
+
times.sort((a, b) => a - b);
|
|
37
|
+
const sum = times.reduce((a, b) => a + b, 0);
|
|
38
|
+
return {
|
|
39
|
+
name,
|
|
40
|
+
iterations,
|
|
41
|
+
results: {
|
|
42
|
+
min: times[0],
|
|
43
|
+
max: times[times.length - 1],
|
|
44
|
+
avg: sum / times.length,
|
|
45
|
+
p50: times[Math.floor(times.length * 0.5)],
|
|
46
|
+
p95: times[Math.floor(times.length * 0.95)],
|
|
47
|
+
p99: times[Math.floor(times.length * 0.99)],
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Benchmark ONNX embedding generation
|
|
53
|
+
*/
|
|
54
|
+
async function benchmarkEmbeddings(iterations = 10) {
|
|
55
|
+
const results = [];
|
|
56
|
+
// Initialize embedder
|
|
57
|
+
await (0, onnx_embedder_1.initOnnxEmbedder)();
|
|
58
|
+
const stats = (0, onnx_embedder_1.getStats)();
|
|
59
|
+
console.log(`\nš ONNX Embedder: ${stats.dimension}d, SIMD: ${stats.simd}`);
|
|
60
|
+
// Single embedding benchmark
|
|
61
|
+
const singleResult = await runBenchmark('Single embedding (short text)', async () => {
|
|
62
|
+
await (0, onnx_embedder_1.embed)('This is a test sentence for embedding.');
|
|
63
|
+
}, iterations);
|
|
64
|
+
results.push(singleResult);
|
|
65
|
+
// Single embedding - long text
|
|
66
|
+
const longText = 'This is a much longer text that contains more content. '.repeat(20);
|
|
67
|
+
const singleLongResult = await runBenchmark('Single embedding (long text)', async () => {
|
|
68
|
+
await (0, onnx_embedder_1.embed)(longText);
|
|
69
|
+
}, iterations);
|
|
70
|
+
results.push(singleLongResult);
|
|
71
|
+
// Batch embedding - small batch
|
|
72
|
+
const smallBatch = Array(4).fill(0).map((_, i) => `Test sentence number ${i}`);
|
|
73
|
+
const batchSmallResult = await runBenchmark('Batch embedding (4 texts)', async () => {
|
|
74
|
+
await (0, onnx_embedder_1.embedBatch)(smallBatch);
|
|
75
|
+
}, iterations);
|
|
76
|
+
batchSmallResult.throughput = {
|
|
77
|
+
itemsPerSecond: (4 * 1000) / batchSmallResult.results.avg,
|
|
78
|
+
};
|
|
79
|
+
results.push(batchSmallResult);
|
|
80
|
+
// Batch embedding - medium batch
|
|
81
|
+
const mediumBatch = Array(16).fill(0).map((_, i) => `Test sentence number ${i} with some content`);
|
|
82
|
+
const batchMediumResult = await runBenchmark('Batch embedding (16 texts)', async () => {
|
|
83
|
+
await (0, onnx_embedder_1.embedBatch)(mediumBatch);
|
|
84
|
+
}, iterations);
|
|
85
|
+
batchMediumResult.throughput = {
|
|
86
|
+
itemsPerSecond: (16 * 1000) / batchMediumResult.results.avg,
|
|
87
|
+
};
|
|
88
|
+
results.push(batchMediumResult);
|
|
89
|
+
// Batch embedding - large batch
|
|
90
|
+
const largeBatch = Array(64).fill(0).map((_, i) => `Test sentence number ${i} with additional content here`);
|
|
91
|
+
const batchLargeResult = await runBenchmark('Batch embedding (64 texts)', async () => {
|
|
92
|
+
await (0, onnx_embedder_1.embedBatch)(largeBatch);
|
|
93
|
+
}, Math.min(iterations, 5) // Fewer iterations for large batches
|
|
94
|
+
);
|
|
95
|
+
batchLargeResult.throughput = {
|
|
96
|
+
itemsPerSecond: (64 * 1000) / batchLargeResult.results.avg,
|
|
97
|
+
};
|
|
98
|
+
results.push(batchLargeResult);
|
|
99
|
+
return results;
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Benchmark worker execution
|
|
103
|
+
*/
|
|
104
|
+
async function benchmarkWorkers(targetPath = '.') {
|
|
105
|
+
const results = [];
|
|
106
|
+
// Security worker (no embeddings - fastest)
|
|
107
|
+
const securityWorker = (0, native_worker_1.createSecurityWorker)();
|
|
108
|
+
const securityResult = await runBenchmark('Security worker (no embeddings)', async () => {
|
|
109
|
+
await securityWorker.run(targetPath);
|
|
110
|
+
}, 5, 1);
|
|
111
|
+
results.push(securityResult);
|
|
112
|
+
// Analysis worker (with embeddings)
|
|
113
|
+
const analysisWorker = (0, native_worker_1.createAnalysisWorker)();
|
|
114
|
+
const analysisResult = await runBenchmark('Analysis worker (with embeddings)', async () => {
|
|
115
|
+
await analysisWorker.run(targetPath);
|
|
116
|
+
}, 3, 1);
|
|
117
|
+
results.push(analysisResult);
|
|
118
|
+
return results;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Benchmark individual phases
|
|
122
|
+
*/
|
|
123
|
+
async function benchmarkPhases(targetPath = '.') {
|
|
124
|
+
const results = [];
|
|
125
|
+
// File discovery phase only
|
|
126
|
+
const discoveryWorker = new native_worker_1.NativeWorker({
|
|
127
|
+
name: 'discovery-only',
|
|
128
|
+
phases: [{ type: 'file-discovery' }],
|
|
129
|
+
capabilities: {},
|
|
130
|
+
});
|
|
131
|
+
const discoveryResult = await runBenchmark('Phase: file-discovery', async () => {
|
|
132
|
+
await discoveryWorker.run(targetPath);
|
|
133
|
+
}, 10);
|
|
134
|
+
results.push(discoveryResult);
|
|
135
|
+
// Pattern extraction phase
|
|
136
|
+
const patternWorker = new native_worker_1.NativeWorker({
|
|
137
|
+
name: 'pattern-only',
|
|
138
|
+
phases: [{ type: 'file-discovery' }, { type: 'pattern-extraction' }],
|
|
139
|
+
capabilities: {},
|
|
140
|
+
});
|
|
141
|
+
const patternResult = await runBenchmark('Phase: pattern-extraction', async () => {
|
|
142
|
+
await patternWorker.run(targetPath);
|
|
143
|
+
}, 5);
|
|
144
|
+
results.push(patternResult);
|
|
145
|
+
// Embedding generation phase
|
|
146
|
+
const embeddingWorker = new native_worker_1.NativeWorker({
|
|
147
|
+
name: 'embedding-only',
|
|
148
|
+
phases: [
|
|
149
|
+
{ type: 'file-discovery', config: { patterns: ['**/*.ts'], exclude: ['**/node_modules/**'] } },
|
|
150
|
+
{ type: 'pattern-extraction' },
|
|
151
|
+
{ type: 'embedding-generation' },
|
|
152
|
+
],
|
|
153
|
+
capabilities: { onnxEmbeddings: true },
|
|
154
|
+
});
|
|
155
|
+
const embeddingResult = await runBenchmark('Phase: embedding-generation', async () => {
|
|
156
|
+
await embeddingWorker.run(targetPath);
|
|
157
|
+
}, 3, 1);
|
|
158
|
+
results.push(embeddingResult);
|
|
159
|
+
return results;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Format benchmark results as table
|
|
163
|
+
*/
|
|
164
|
+
function formatBenchmarkResults(results) {
|
|
165
|
+
const lines = [];
|
|
166
|
+
lines.push('');
|
|
167
|
+
lines.push('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā¬āāāāāāāāāāā¬āāāāāāāāāāā¬āāāāāāāāāāā¬āāāāāāāāāāā¬āāāāāāāāāāāāāāā');
|
|
168
|
+
lines.push('ā Benchmark ā Min (ms) ā Avg (ms) ā P95 (ms) ā Max (ms) ā Throughput ā');
|
|
169
|
+
lines.push('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā¼āāāāāāāāāāā¼āāāāāāāāāāā¼āāāāāāāāāāā¼āāāāāāāāāāā¼āāāāāāāāāāāāāāā¤');
|
|
170
|
+
for (const result of results) {
|
|
171
|
+
const name = result.name.padEnd(35).slice(0, 35);
|
|
172
|
+
const min = result.results.min.toFixed(1).padStart(8);
|
|
173
|
+
const avg = result.results.avg.toFixed(1).padStart(8);
|
|
174
|
+
const p95 = result.results.p95.toFixed(1).padStart(8);
|
|
175
|
+
const max = result.results.max.toFixed(1).padStart(8);
|
|
176
|
+
const throughput = result.throughput
|
|
177
|
+
? `${result.throughput.itemsPerSecond.toFixed(1)}/s`.padStart(12)
|
|
178
|
+
: ' -';
|
|
179
|
+
lines.push(`ā ${name} ā ${min} ā ${avg} ā ${p95} ā ${max} ā ${throughput} ā`);
|
|
180
|
+
}
|
|
181
|
+
lines.push('āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā“āāāāāāāāāāā“āāāāāāāāāāā“āāāāāāāāāāā“āāāāāāāāāāā“āāāāāāāāāāāāāāā');
|
|
182
|
+
lines.push('');
|
|
183
|
+
return lines.join('\n');
|
|
184
|
+
}
|
|
185
|
+
/**
|
|
186
|
+
* Run full benchmark suite
|
|
187
|
+
*/
|
|
188
|
+
async function runFullBenchmark(targetPath = '.') {
|
|
189
|
+
console.log('š RuVector Native Worker Benchmark Suite\n');
|
|
190
|
+
console.log('='.repeat(60));
|
|
191
|
+
// Embeddings benchmark
|
|
192
|
+
console.log('\nš Benchmarking ONNX Embeddings...');
|
|
193
|
+
const embeddings = await benchmarkEmbeddings(10);
|
|
194
|
+
console.log(formatBenchmarkResults(embeddings));
|
|
195
|
+
// Phases benchmark
|
|
196
|
+
console.log('\nā” Benchmarking Individual Phases...');
|
|
197
|
+
const phases = await benchmarkPhases(targetPath);
|
|
198
|
+
console.log(formatBenchmarkResults(phases));
|
|
199
|
+
// Workers benchmark
|
|
200
|
+
console.log('\nš§ Benchmarking Full Workers...');
|
|
201
|
+
const workers = await benchmarkWorkers(targetPath);
|
|
202
|
+
console.log(formatBenchmarkResults(workers));
|
|
203
|
+
// Summary
|
|
204
|
+
const stats = (0, onnx_embedder_1.getStats)();
|
|
205
|
+
const summary = `
|
|
206
|
+
RuVector Native Worker Benchmark Summary
|
|
207
|
+
========================================
|
|
208
|
+
ONNX Model: all-MiniLM-L6-v2 (${stats.dimension}d)
|
|
209
|
+
SIMD: ${stats.simd ? 'Enabled ā' : 'Disabled'}
|
|
210
|
+
Parallel Workers: ${stats.parallel ? `${stats.parallelWorkers} workers` : 'Disabled'}
|
|
211
|
+
|
|
212
|
+
Embedding Performance:
|
|
213
|
+
Single: ${embeddings[0].results.avg.toFixed(1)}ms avg
|
|
214
|
+
Batch (16): ${embeddings[3].results.avg.toFixed(1)}ms avg (${embeddings[3].throughput?.itemsPerSecond.toFixed(0)}/s)
|
|
215
|
+
Batch (64): ${embeddings[4].results.avg.toFixed(1)}ms avg (${embeddings[4].throughput?.itemsPerSecond.toFixed(0)}/s)
|
|
216
|
+
|
|
217
|
+
Worker Performance:
|
|
218
|
+
Security scan: ${workers[0].results.avg.toFixed(0)}ms avg
|
|
219
|
+
Full analysis: ${workers[1].results.avg.toFixed(0)}ms avg
|
|
220
|
+
`;
|
|
221
|
+
console.log(summary);
|
|
222
|
+
return { embeddings, phases, workers, summary };
|
|
223
|
+
}
|
|
224
|
+
exports.default = {
|
|
225
|
+
benchmarkEmbeddings,
|
|
226
|
+
benchmarkWorkers,
|
|
227
|
+
benchmarkPhases,
|
|
228
|
+
runFullBenchmark,
|
|
229
|
+
formatBenchmarkResults,
|
|
230
|
+
};
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RuVector Native Workers
|
|
3
|
+
*
|
|
4
|
+
* Deep integration with ONNX embeddings, VectorDB, and intelligence engine.
|
|
5
|
+
* No external dependencies - pure ruvector execution.
|
|
6
|
+
*/
|
|
7
|
+
export * from './types';
|
|
8
|
+
export * from './native-worker';
|
|
9
|
+
export * from './benchmark';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/workers/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,cAAc,SAAS,CAAC;AACxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* RuVector Native Workers
|
|
4
|
+
*
|
|
5
|
+
* Deep integration with ONNX embeddings, VectorDB, and intelligence engine.
|
|
6
|
+
* No external dependencies - pure ruvector execution.
|
|
7
|
+
*/
|
|
8
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
|
+
if (k2 === undefined) k2 = k;
|
|
10
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
11
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
12
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
13
|
+
}
|
|
14
|
+
Object.defineProperty(o, k2, desc);
|
|
15
|
+
}) : (function(o, m, k, k2) {
|
|
16
|
+
if (k2 === undefined) k2 = k;
|
|
17
|
+
o[k2] = m[k];
|
|
18
|
+
}));
|
|
19
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
20
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
21
|
+
};
|
|
22
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
|
+
__exportStar(require("./types"), exports);
|
|
24
|
+
__exportStar(require("./native-worker"), exports);
|
|
25
|
+
__exportStar(require("./benchmark"), exports);
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native Worker Runner for RuVector
|
|
3
|
+
*
|
|
4
|
+
* Direct integration with:
|
|
5
|
+
* - ONNX embedder (384d, SIMD-accelerated)
|
|
6
|
+
* - VectorDB (HNSW indexing)
|
|
7
|
+
* - Intelligence engine (Q-learning, memory)
|
|
8
|
+
*
|
|
9
|
+
* No delegation to external tools - pure ruvector execution.
|
|
10
|
+
*/
|
|
11
|
+
import { WorkerConfig, WorkerResult } from './types';
|
|
12
|
+
/**
|
|
13
|
+
* Native Worker Runner
|
|
14
|
+
*/
|
|
15
|
+
export declare class NativeWorker {
|
|
16
|
+
private config;
|
|
17
|
+
private vectorDb;
|
|
18
|
+
private findings;
|
|
19
|
+
private stats;
|
|
20
|
+
constructor(config: WorkerConfig);
|
|
21
|
+
/**
|
|
22
|
+
* Initialize worker with capabilities
|
|
23
|
+
*/
|
|
24
|
+
init(): Promise<void>;
|
|
25
|
+
/**
|
|
26
|
+
* Run all phases in sequence
|
|
27
|
+
*/
|
|
28
|
+
run(targetPath?: string): Promise<WorkerResult>;
|
|
29
|
+
/**
|
|
30
|
+
* Execute a single phase
|
|
31
|
+
*/
|
|
32
|
+
private executePhase;
|
|
33
|
+
/**
|
|
34
|
+
* Phase: File Discovery
|
|
35
|
+
*/
|
|
36
|
+
private phaseFileDiscovery;
|
|
37
|
+
/**
|
|
38
|
+
* Phase: Pattern Extraction
|
|
39
|
+
*/
|
|
40
|
+
private phasePatternExtraction;
|
|
41
|
+
/**
|
|
42
|
+
* Phase: Embedding Generation (ONNX)
|
|
43
|
+
*/
|
|
44
|
+
private phaseEmbeddingGeneration;
|
|
45
|
+
/**
|
|
46
|
+
* Phase: Vector Storage
|
|
47
|
+
*/
|
|
48
|
+
private phaseVectorStorage;
|
|
49
|
+
/**
|
|
50
|
+
* Phase: Similarity Search
|
|
51
|
+
*/
|
|
52
|
+
private phaseSimilaritySearch;
|
|
53
|
+
/**
|
|
54
|
+
* Phase: Security Scan
|
|
55
|
+
*/
|
|
56
|
+
private phaseSecurityScan;
|
|
57
|
+
/**
|
|
58
|
+
* Phase: Complexity Analysis
|
|
59
|
+
*/
|
|
60
|
+
private phaseComplexityAnalysis;
|
|
61
|
+
/**
|
|
62
|
+
* Phase: Summarization
|
|
63
|
+
*/
|
|
64
|
+
private phaseSummarization;
|
|
65
|
+
/**
|
|
66
|
+
* Summarize phase data for results
|
|
67
|
+
*/
|
|
68
|
+
private summarizePhaseData;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Quick worker factory functions
|
|
72
|
+
*/
|
|
73
|
+
export declare function createSecurityWorker(name?: string): NativeWorker;
|
|
74
|
+
export declare function createAnalysisWorker(name?: string): NativeWorker;
|
|
75
|
+
export declare function createLearningWorker(name?: string): NativeWorker;
|
|
76
|
+
//# sourceMappingURL=native-worker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"native-worker.d.ts","sourceRoot":"","sources":["../../src/workers/native-worker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAKH,OAAO,EACL,YAAY,EACZ,YAAY,EAKb,MAAM,SAAS,CAAC;AAuBjB;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAiB;IACjC,OAAO,CAAC,KAAK,CAKX;gBAEU,MAAM,EAAE,YAAY;IAIhC;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAmB3B;;OAEG;IACG,GAAG,CAAC,UAAU,GAAE,MAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAiD1D;;OAEG;YACW,YAAY;IAmC1B;;OAEG;YACW,kBAAkB;IAqBhC;;OAEG;YACW,sBAAsB;IAoDpC;;OAEG;YACW,wBAAwB;IAgDtC;;OAEG;YACW,kBAAkB;IA8BhC;;OAEG;YACW,qBAAqB;IAoBnC;;OAEG;YACW,iBAAiB;IAyC/B;;OAEG;YACW,uBAAuB;IA0CrC;;OAEG;YACW,kBAAkB;IAqBhC;;OAEG;IACH,OAAO,CAAC,kBAAkB;CAsB3B;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,SAAqB,GAAG,YAAY,CAW5E;AAED,wBAAgB,oBAAoB,CAAC,IAAI,SAAkB,GAAG,YAAY,CAczE;AAED,wBAAgB,oBAAoB,CAAC,IAAI,SAAoB,GAAG,YAAY,CAa3E"}
|
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Native Worker Runner for RuVector
|
|
4
|
+
*
|
|
5
|
+
* Direct integration with:
|
|
6
|
+
* - ONNX embedder (384d, SIMD-accelerated)
|
|
7
|
+
* - VectorDB (HNSW indexing)
|
|
8
|
+
* - Intelligence engine (Q-learning, memory)
|
|
9
|
+
*
|
|
10
|
+
* No delegation to external tools - pure ruvector execution.
|
|
11
|
+
*/
|
|
12
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
13
|
+
if (k2 === undefined) k2 = k;
|
|
14
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
15
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
16
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
17
|
+
}
|
|
18
|
+
Object.defineProperty(o, k2, desc);
|
|
19
|
+
}) : (function(o, m, k, k2) {
|
|
20
|
+
if (k2 === undefined) k2 = k;
|
|
21
|
+
o[k2] = m[k];
|
|
22
|
+
}));
|
|
23
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
24
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
25
|
+
}) : function(o, v) {
|
|
26
|
+
o["default"] = v;
|
|
27
|
+
});
|
|
28
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
29
|
+
var ownKeys = function(o) {
|
|
30
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
31
|
+
var ar = [];
|
|
32
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
33
|
+
return ar;
|
|
34
|
+
};
|
|
35
|
+
return ownKeys(o);
|
|
36
|
+
};
|
|
37
|
+
return function (mod) {
|
|
38
|
+
if (mod && mod.__esModule) return mod;
|
|
39
|
+
var result = {};
|
|
40
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
41
|
+
__setModuleDefault(result, mod);
|
|
42
|
+
return result;
|
|
43
|
+
};
|
|
44
|
+
})();
|
|
45
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
46
|
+
exports.NativeWorker = void 0;
|
|
47
|
+
exports.createSecurityWorker = createSecurityWorker;
|
|
48
|
+
exports.createAnalysisWorker = createAnalysisWorker;
|
|
49
|
+
exports.createLearningWorker = createLearningWorker;
|
|
50
|
+
const fs = __importStar(require("fs"));
|
|
51
|
+
const path = __importStar(require("path"));
|
|
52
|
+
const glob_1 = require("glob");
|
|
53
|
+
const onnx_embedder_1 = require("../core/onnx-embedder");
|
|
54
|
+
// Lazy imports for optional dependencies
|
|
55
|
+
let VectorDb = null;
|
|
56
|
+
let intelligence = null;
|
|
57
|
+
async function loadOptionalDeps() {
|
|
58
|
+
try {
|
|
59
|
+
const core = await Promise.resolve().then(() => __importStar(require('@ruvector/core')));
|
|
60
|
+
VectorDb = core.VectorDb;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
// VectorDB not available
|
|
64
|
+
}
|
|
65
|
+
try {
|
|
66
|
+
const intel = await Promise.resolve().then(() => __importStar(require('../core/intelligence-engine')));
|
|
67
|
+
intelligence = intel;
|
|
68
|
+
}
|
|
69
|
+
catch {
|
|
70
|
+
// Intelligence not available
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Native Worker Runner
|
|
75
|
+
*/
|
|
76
|
+
class NativeWorker {
|
|
77
|
+
constructor(config) {
|
|
78
|
+
this.vectorDb = null;
|
|
79
|
+
this.findings = [];
|
|
80
|
+
this.stats = {
|
|
81
|
+
filesAnalyzed: 0,
|
|
82
|
+
patternsFound: 0,
|
|
83
|
+
embeddingsGenerated: 0,
|
|
84
|
+
vectorsStored: 0,
|
|
85
|
+
};
|
|
86
|
+
this.config = config;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Initialize worker with capabilities
|
|
90
|
+
*/
|
|
91
|
+
async init() {
|
|
92
|
+
await loadOptionalDeps();
|
|
93
|
+
// Initialize ONNX embedder if needed
|
|
94
|
+
if (this.config.capabilities?.onnxEmbeddings) {
|
|
95
|
+
await (0, onnx_embedder_1.initOnnxEmbedder)();
|
|
96
|
+
}
|
|
97
|
+
// Initialize VectorDB if needed
|
|
98
|
+
if (this.config.capabilities?.vectorDb && VectorDb) {
|
|
99
|
+
const dbPath = path.join(process.cwd(), '.ruvector', 'workers', `${this.config.name}.db`);
|
|
100
|
+
fs.mkdirSync(path.dirname(dbPath), { recursive: true });
|
|
101
|
+
this.vectorDb = new VectorDb({
|
|
102
|
+
dimensions: 384,
|
|
103
|
+
storagePath: dbPath,
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Run all phases in sequence
|
|
109
|
+
*/
|
|
110
|
+
async run(targetPath = '.') {
|
|
111
|
+
const startTime = performance.now();
|
|
112
|
+
const phaseResults = [];
|
|
113
|
+
await this.init();
|
|
114
|
+
let context = { targetPath, files: [], patterns: [], embeddings: [] };
|
|
115
|
+
for (const phaseConfig of this.config.phases) {
|
|
116
|
+
const phaseStart = performance.now();
|
|
117
|
+
try {
|
|
118
|
+
context = await this.executePhase(phaseConfig.type, context, phaseConfig.config);
|
|
119
|
+
phaseResults.push({
|
|
120
|
+
phase: phaseConfig.type,
|
|
121
|
+
success: true,
|
|
122
|
+
data: this.summarizePhaseData(phaseConfig.type, context),
|
|
123
|
+
timeMs: performance.now() - phaseStart,
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
catch (error) {
|
|
127
|
+
phaseResults.push({
|
|
128
|
+
phase: phaseConfig.type,
|
|
129
|
+
success: false,
|
|
130
|
+
data: null,
|
|
131
|
+
timeMs: performance.now() - phaseStart,
|
|
132
|
+
error: error.message,
|
|
133
|
+
});
|
|
134
|
+
// Continue to next phase on error (fault-tolerant)
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const totalTimeMs = performance.now() - startTime;
|
|
138
|
+
return {
|
|
139
|
+
worker: this.config.name,
|
|
140
|
+
success: phaseResults.every(p => p.success),
|
|
141
|
+
phases: phaseResults,
|
|
142
|
+
totalTimeMs,
|
|
143
|
+
summary: {
|
|
144
|
+
filesAnalyzed: this.stats.filesAnalyzed,
|
|
145
|
+
patternsFound: this.stats.patternsFound,
|
|
146
|
+
embeddingsGenerated: this.stats.embeddingsGenerated,
|
|
147
|
+
vectorsStored: this.stats.vectorsStored,
|
|
148
|
+
findings: this.findings,
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* Execute a single phase
|
|
154
|
+
*/
|
|
155
|
+
async executePhase(type, context, config) {
|
|
156
|
+
switch (type) {
|
|
157
|
+
case 'file-discovery':
|
|
158
|
+
return this.phaseFileDiscovery(context, config);
|
|
159
|
+
case 'pattern-extraction':
|
|
160
|
+
return this.phasePatternExtraction(context, config);
|
|
161
|
+
case 'embedding-generation':
|
|
162
|
+
return this.phaseEmbeddingGeneration(context, config);
|
|
163
|
+
case 'vector-storage':
|
|
164
|
+
return this.phaseVectorStorage(context, config);
|
|
165
|
+
case 'similarity-search':
|
|
166
|
+
return this.phaseSimilaritySearch(context, config);
|
|
167
|
+
case 'security-scan':
|
|
168
|
+
return this.phaseSecurityScan(context, config);
|
|
169
|
+
case 'complexity-analysis':
|
|
170
|
+
return this.phaseComplexityAnalysis(context, config);
|
|
171
|
+
case 'summarization':
|
|
172
|
+
return this.phaseSummarization(context, config);
|
|
173
|
+
default:
|
|
174
|
+
throw new Error(`Unknown phase: ${type}`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Phase: File Discovery
|
|
179
|
+
*/
|
|
180
|
+
async phaseFileDiscovery(context, config) {
|
|
181
|
+
const patterns = config?.patterns || ['**/*.ts', '**/*.js', '**/*.tsx', '**/*.jsx'];
|
|
182
|
+
const exclude = config?.exclude || ['**/node_modules/**', '**/dist/**', '**/.git/**'];
|
|
183
|
+
const files = [];
|
|
184
|
+
for (const pattern of patterns) {
|
|
185
|
+
const matches = await (0, glob_1.glob)(pattern, {
|
|
186
|
+
cwd: context.targetPath,
|
|
187
|
+
ignore: exclude,
|
|
188
|
+
nodir: true,
|
|
189
|
+
});
|
|
190
|
+
files.push(...matches.map(f => path.join(context.targetPath, f)));
|
|
191
|
+
}
|
|
192
|
+
this.stats.filesAnalyzed = files.length;
|
|
193
|
+
return { ...context, files };
|
|
194
|
+
}
|
|
195
|
+
/**
|
|
196
|
+
* Phase: Pattern Extraction
|
|
197
|
+
*/
|
|
198
|
+
async phasePatternExtraction(context, config) {
|
|
199
|
+
const patterns = [];
|
|
200
|
+
const patternTypes = config?.types || ['function', 'class', 'import', 'export', 'todo'];
|
|
201
|
+
for (const file of context.files.slice(0, 100)) {
|
|
202
|
+
try {
|
|
203
|
+
const content = fs.readFileSync(file, 'utf-8');
|
|
204
|
+
// Extract functions
|
|
205
|
+
if (patternTypes.includes('function')) {
|
|
206
|
+
const funcMatches = content.match(/(?:async\s+)?function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s*)?\(/g) || [];
|
|
207
|
+
patterns.push(...funcMatches.map(m => ({ type: 'function', match: m, file })));
|
|
208
|
+
}
|
|
209
|
+
// Extract classes
|
|
210
|
+
if (patternTypes.includes('class')) {
|
|
211
|
+
const classMatches = content.match(/class\s+(\w+)/g) || [];
|
|
212
|
+
patterns.push(...classMatches.map(m => ({ type: 'class', match: m, file })));
|
|
213
|
+
}
|
|
214
|
+
// Extract imports
|
|
215
|
+
if (patternTypes.includes('import')) {
|
|
216
|
+
const importMatches = content.match(/import\s+.*?from\s+['"][^'"]+['"]/g) || [];
|
|
217
|
+
patterns.push(...importMatches.map(m => ({ type: 'import', match: m, file })));
|
|
218
|
+
}
|
|
219
|
+
// Extract TODOs
|
|
220
|
+
if (patternTypes.includes('todo')) {
|
|
221
|
+
const todoMatches = content.match(/\/\/\s*(TODO|FIXME|HACK|XXX):.*/gi) || [];
|
|
222
|
+
patterns.push(...todoMatches.map(m => ({ type: 'todo', match: m, file })));
|
|
223
|
+
// Add findings for TODOs
|
|
224
|
+
todoMatches.forEach(m => {
|
|
225
|
+
this.findings.push({
|
|
226
|
+
type: 'info',
|
|
227
|
+
message: m,
|
|
228
|
+
file,
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
catch {
|
|
234
|
+
// Skip unreadable files
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
this.stats.patternsFound = patterns.length;
|
|
238
|
+
return { ...context, patterns };
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Phase: Embedding Generation (ONNX)
|
|
242
|
+
*/
|
|
243
|
+
async phaseEmbeddingGeneration(context, config) {
|
|
244
|
+
if (!(0, onnx_embedder_1.isReady)()) {
|
|
245
|
+
await (0, onnx_embedder_1.initOnnxEmbedder)();
|
|
246
|
+
}
|
|
247
|
+
const embeddings = [];
|
|
248
|
+
const batchSize = config?.batchSize || 32;
|
|
249
|
+
// Collect texts to embed
|
|
250
|
+
const texts = [];
|
|
251
|
+
// Embed file content summaries
|
|
252
|
+
for (const file of context.files.slice(0, 50)) {
|
|
253
|
+
try {
|
|
254
|
+
const content = fs.readFileSync(file, 'utf-8');
|
|
255
|
+
const summary = content.slice(0, 512); // First 512 chars
|
|
256
|
+
texts.push({ text: summary, file });
|
|
257
|
+
}
|
|
258
|
+
catch {
|
|
259
|
+
// Skip
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
// Embed patterns
|
|
263
|
+
for (const pattern of context.patterns.slice(0, 100)) {
|
|
264
|
+
texts.push({ text: pattern.match, file: pattern.file });
|
|
265
|
+
}
|
|
266
|
+
// Batch embed
|
|
267
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
268
|
+
const batch = texts.slice(i, i + batchSize);
|
|
269
|
+
const results = await (0, onnx_embedder_1.embedBatch)(batch.map(t => t.text));
|
|
270
|
+
for (let j = 0; j < results.length; j++) {
|
|
271
|
+
embeddings.push({
|
|
272
|
+
text: batch[j].text,
|
|
273
|
+
embedding: results[j].embedding,
|
|
274
|
+
file: batch[j].file,
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
this.stats.embeddingsGenerated = embeddings.length;
|
|
279
|
+
return { ...context, embeddings };
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Phase: Vector Storage
|
|
283
|
+
*/
|
|
284
|
+
async phaseVectorStorage(context, config) {
|
|
285
|
+
if (!this.vectorDb) {
|
|
286
|
+
return context;
|
|
287
|
+
}
|
|
288
|
+
let stored = 0;
|
|
289
|
+
for (const item of context.embeddings) {
|
|
290
|
+
try {
|
|
291
|
+
await this.vectorDb.insert({
|
|
292
|
+
vector: new Float32Array(item.embedding),
|
|
293
|
+
metadata: {
|
|
294
|
+
text: item.text.slice(0, 200),
|
|
295
|
+
file: item.file,
|
|
296
|
+
worker: this.config.name,
|
|
297
|
+
timestamp: Date.now(),
|
|
298
|
+
},
|
|
299
|
+
});
|
|
300
|
+
stored++;
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
303
|
+
// Skip duplicates/errors
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
this.stats.vectorsStored = stored;
|
|
307
|
+
return context;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Phase: Similarity Search
|
|
311
|
+
*/
|
|
312
|
+
async phaseSimilaritySearch(context, config) {
|
|
313
|
+
if (!this.vectorDb || context.embeddings.length === 0) {
|
|
314
|
+
return context;
|
|
315
|
+
}
|
|
316
|
+
const query = config?.query || context.embeddings[0]?.text;
|
|
317
|
+
if (!query)
|
|
318
|
+
return context;
|
|
319
|
+
const queryResult = await (0, onnx_embedder_1.embed)(query);
|
|
320
|
+
const results = await this.vectorDb.search({
|
|
321
|
+
vector: new Float32Array(queryResult.embedding),
|
|
322
|
+
k: config?.k || 5,
|
|
323
|
+
});
|
|
324
|
+
return { ...context, searchResults: results };
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* Phase: Security Scan
|
|
328
|
+
*/
|
|
329
|
+
async phaseSecurityScan(context, config) {
|
|
330
|
+
const securityPatterns = [
|
|
331
|
+
{ pattern: /password\s*=\s*['"][^'"]+['"]/gi, severity: 'high', type: 'Hardcoded password' },
|
|
332
|
+
{ pattern: /api[_-]?key\s*=\s*['"][^'"]+['"]/gi, severity: 'high', type: 'Hardcoded API key' },
|
|
333
|
+
{ pattern: /secret\s*=\s*['"][^'"]+['"]/gi, severity: 'high', type: 'Hardcoded secret' },
|
|
334
|
+
{ pattern: /eval\s*\(/g, severity: 'medium', type: 'eval() usage' },
|
|
335
|
+
{ pattern: /dangerouslySetInnerHTML/g, severity: 'medium', type: 'XSS risk' },
|
|
336
|
+
{ pattern: /SELECT\s+.*\s+FROM.*\+/gi, severity: 'high', type: 'SQL injection risk' },
|
|
337
|
+
{ pattern: /exec\s*\(/g, severity: 'medium', type: 'Command execution' },
|
|
338
|
+
];
|
|
339
|
+
for (const file of context.files.slice(0, 100)) {
|
|
340
|
+
try {
|
|
341
|
+
const content = fs.readFileSync(file, 'utf-8');
|
|
342
|
+
const lines = content.split('\n');
|
|
343
|
+
for (const { pattern, severity, type } of securityPatterns) {
|
|
344
|
+
let match;
|
|
345
|
+
const regex = new RegExp(pattern.source, pattern.flags);
|
|
346
|
+
while ((match = regex.exec(content)) !== null) {
|
|
347
|
+
const lineNum = content.slice(0, match.index).split('\n').length;
|
|
348
|
+
this.findings.push({
|
|
349
|
+
type: 'security',
|
|
350
|
+
message: `${type}: ${match[0].slice(0, 50)}...`,
|
|
351
|
+
file,
|
|
352
|
+
line: lineNum,
|
|
353
|
+
severity: severity === 'high' ? 3 : 2,
|
|
354
|
+
});
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
catch {
|
|
359
|
+
// Skip
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return context;
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Phase: Complexity Analysis
|
|
366
|
+
*/
|
|
367
|
+
async phaseComplexityAnalysis(context, config) {
|
|
368
|
+
const complexityThreshold = config?.threshold || 10;
|
|
369
|
+
const complexFiles = [];
|
|
370
|
+
for (const file of context.files.slice(0, 50)) {
|
|
371
|
+
try {
|
|
372
|
+
const content = fs.readFileSync(file, 'utf-8');
|
|
373
|
+
// Simple cyclomatic complexity approximation
|
|
374
|
+
const branches = (content.match(/\bif\b/g)?.length || 0) +
|
|
375
|
+
(content.match(/\belse\b/g)?.length || 0) +
|
|
376
|
+
(content.match(/\bfor\b/g)?.length || 0) +
|
|
377
|
+
(content.match(/\bwhile\b/g)?.length || 0) +
|
|
378
|
+
(content.match(/\bswitch\b/g)?.length || 0) +
|
|
379
|
+
(content.match(/\bcase\b/g)?.length || 0) +
|
|
380
|
+
(content.match(/\bcatch\b/g)?.length || 0) +
|
|
381
|
+
(content.match(/\?\?/g)?.length || 0) +
|
|
382
|
+
(content.match(/\?/g)?.length || 0);
|
|
383
|
+
const complexity = branches + 1;
|
|
384
|
+
if (complexity > complexityThreshold) {
|
|
385
|
+
complexFiles.push({ file, complexity });
|
|
386
|
+
this.findings.push({
|
|
387
|
+
type: 'warning',
|
|
388
|
+
message: `High complexity: ${complexity} (threshold: ${complexityThreshold})`,
|
|
389
|
+
file,
|
|
390
|
+
severity: complexity > 20 ? 3 : 2,
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
catch {
|
|
395
|
+
// Skip
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
return { ...context, complexFiles };
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* Phase: Summarization
|
|
402
|
+
*/
|
|
403
|
+
async phaseSummarization(context, config) {
|
|
404
|
+
const summary = {
|
|
405
|
+
filesAnalyzed: context.files?.length || 0,
|
|
406
|
+
patternsFound: context.patterns?.length || 0,
|
|
407
|
+
embeddingsGenerated: context.embeddings?.length || 0,
|
|
408
|
+
findingsCount: this.findings.length,
|
|
409
|
+
findingsByType: {
|
|
410
|
+
info: this.findings.filter(f => f.type === 'info').length,
|
|
411
|
+
warning: this.findings.filter(f => f.type === 'warning').length,
|
|
412
|
+
error: this.findings.filter(f => f.type === 'error').length,
|
|
413
|
+
security: this.findings.filter(f => f.type === 'security').length,
|
|
414
|
+
},
|
|
415
|
+
topFindings: this.findings.slice(0, 10),
|
|
416
|
+
};
|
|
417
|
+
return { ...context, summary };
|
|
418
|
+
}
|
|
419
|
+
/**
|
|
420
|
+
* Summarize phase data for results
|
|
421
|
+
*/
|
|
422
|
+
summarizePhaseData(type, context) {
|
|
423
|
+
switch (type) {
|
|
424
|
+
case 'file-discovery':
|
|
425
|
+
return { filesFound: context.files?.length || 0 };
|
|
426
|
+
case 'pattern-extraction':
|
|
427
|
+
return { patternsFound: context.patterns?.length || 0 };
|
|
428
|
+
case 'embedding-generation':
|
|
429
|
+
return { embeddingsGenerated: context.embeddings?.length || 0 };
|
|
430
|
+
case 'vector-storage':
|
|
431
|
+
return { vectorsStored: this.stats.vectorsStored };
|
|
432
|
+
case 'similarity-search':
|
|
433
|
+
return { resultsFound: context.searchResults?.length || 0 };
|
|
434
|
+
case 'security-scan':
|
|
435
|
+
return { securityFindings: this.findings.filter(f => f.type === 'security').length };
|
|
436
|
+
case 'complexity-analysis':
|
|
437
|
+
return { complexFiles: context.complexFiles?.length || 0 };
|
|
438
|
+
case 'summarization':
|
|
439
|
+
return context.summary;
|
|
440
|
+
default:
|
|
441
|
+
return {};
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
exports.NativeWorker = NativeWorker;
|
|
446
|
+
/**
|
|
447
|
+
* Quick worker factory functions
|
|
448
|
+
*/
|
|
449
|
+
function createSecurityWorker(name = 'security-scanner') {
|
|
450
|
+
return new NativeWorker({
|
|
451
|
+
name,
|
|
452
|
+
description: 'Security vulnerability scanner',
|
|
453
|
+
phases: [
|
|
454
|
+
{ type: 'file-discovery', config: { patterns: ['**/*.ts', '**/*.js', '**/*.tsx', '**/*.jsx'] } },
|
|
455
|
+
{ type: 'security-scan' },
|
|
456
|
+
{ type: 'summarization' },
|
|
457
|
+
],
|
|
458
|
+
capabilities: { onnxEmbeddings: false, vectorDb: false },
|
|
459
|
+
});
|
|
460
|
+
}
|
|
461
|
+
function createAnalysisWorker(name = 'code-analyzer') {
|
|
462
|
+
return new NativeWorker({
|
|
463
|
+
name,
|
|
464
|
+
description: 'Code analysis with embeddings',
|
|
465
|
+
phases: [
|
|
466
|
+
{ type: 'file-discovery' },
|
|
467
|
+
{ type: 'pattern-extraction' },
|
|
468
|
+
{ type: 'embedding-generation' },
|
|
469
|
+
{ type: 'vector-storage' },
|
|
470
|
+
{ type: 'complexity-analysis' },
|
|
471
|
+
{ type: 'summarization' },
|
|
472
|
+
],
|
|
473
|
+
capabilities: { onnxEmbeddings: true, vectorDb: true },
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
function createLearningWorker(name = 'pattern-learner') {
|
|
477
|
+
return new NativeWorker({
|
|
478
|
+
name,
|
|
479
|
+
description: 'Pattern learning with vector storage',
|
|
480
|
+
phases: [
|
|
481
|
+
{ type: 'file-discovery' },
|
|
482
|
+
{ type: 'pattern-extraction' },
|
|
483
|
+
{ type: 'embedding-generation' },
|
|
484
|
+
{ type: 'vector-storage' },
|
|
485
|
+
{ type: 'summarization' },
|
|
486
|
+
],
|
|
487
|
+
capabilities: { onnxEmbeddings: true, vectorDb: true, intelligenceMemory: true },
|
|
488
|
+
});
|
|
489
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Native Worker Types for RuVector
|
|
3
|
+
*
|
|
4
|
+
* Deep integration with ONNX embeddings, VectorDB, and intelligence engine.
|
|
5
|
+
*/
|
|
6
|
+
export interface WorkerConfig {
|
|
7
|
+
name: string;
|
|
8
|
+
description?: string;
|
|
9
|
+
phases: PhaseConfig[];
|
|
10
|
+
capabilities?: WorkerCapabilities;
|
|
11
|
+
timeout?: number;
|
|
12
|
+
parallel?: boolean;
|
|
13
|
+
}
|
|
14
|
+
export interface PhaseConfig {
|
|
15
|
+
type: PhaseType;
|
|
16
|
+
config?: Record<string, any>;
|
|
17
|
+
}
|
|
18
|
+
export type PhaseType = 'file-discovery' | 'pattern-extraction' | 'embedding-generation' | 'vector-storage' | 'similarity-search' | 'security-scan' | 'complexity-analysis' | 'summarization';
|
|
19
|
+
export interface WorkerCapabilities {
|
|
20
|
+
onnxEmbeddings?: boolean;
|
|
21
|
+
vectorDb?: boolean;
|
|
22
|
+
intelligenceMemory?: boolean;
|
|
23
|
+
parallelProcessing?: boolean;
|
|
24
|
+
}
|
|
25
|
+
export interface PhaseResult {
|
|
26
|
+
phase: PhaseType;
|
|
27
|
+
success: boolean;
|
|
28
|
+
data: any;
|
|
29
|
+
timeMs: number;
|
|
30
|
+
error?: string;
|
|
31
|
+
}
|
|
32
|
+
export interface WorkerResult {
|
|
33
|
+
worker: string;
|
|
34
|
+
success: boolean;
|
|
35
|
+
phases: PhaseResult[];
|
|
36
|
+
totalTimeMs: number;
|
|
37
|
+
summary?: WorkerSummary;
|
|
38
|
+
}
|
|
39
|
+
export interface WorkerSummary {
|
|
40
|
+
filesAnalyzed: number;
|
|
41
|
+
patternsFound: number;
|
|
42
|
+
embeddingsGenerated: number;
|
|
43
|
+
vectorsStored: number;
|
|
44
|
+
findings: Finding[];
|
|
45
|
+
}
|
|
46
|
+
export interface Finding {
|
|
47
|
+
type: 'info' | 'warning' | 'error' | 'security';
|
|
48
|
+
message: string;
|
|
49
|
+
file?: string;
|
|
50
|
+
line?: number;
|
|
51
|
+
severity?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface BenchmarkResult {
|
|
54
|
+
name: string;
|
|
55
|
+
iterations: number;
|
|
56
|
+
results: {
|
|
57
|
+
min: number;
|
|
58
|
+
max: number;
|
|
59
|
+
avg: number;
|
|
60
|
+
p50: number;
|
|
61
|
+
p95: number;
|
|
62
|
+
p99: number;
|
|
63
|
+
};
|
|
64
|
+
throughput?: {
|
|
65
|
+
itemsPerSecond: number;
|
|
66
|
+
mbPerSecond?: number;
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/workers/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,YAAY,CAAC,EAAE,kBAAkB,CAAC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,SAAS,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,SAAS,GACjB,gBAAgB,GAChB,oBAAoB,GACpB,sBAAsB,GACtB,gBAAgB,GAChB,mBAAmB,GACnB,eAAe,GACf,qBAAqB,GACrB,eAAe,CAAC;AAEpB,MAAM,WAAW,kBAAkB;IACjC,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,KAAK,EAAE,SAAS,CAAC;IACjB,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,GAAG,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,aAAa,CAAC;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,MAAM,WAAW,OAAO;IACtB,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,CAAC;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE;QACP,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;QACZ,GAAG,EAAE,MAAM,CAAC;KACb,CAAC;IACF,UAAU,CAAC,EAAE;QACX,cAAc,EAAE,MAAM,CAAC;QACvB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC;CACH"}
|