viberag 0.4.2 → 0.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -77,7 +77,7 @@ function deriveIndexingDisplay(indexing) {
|
|
|
77
77
|
percent: indexing.percent,
|
|
78
78
|
stage: indexing.stage,
|
|
79
79
|
chunkInfo,
|
|
80
|
-
throttleInfo: indexing.throttleMessage,
|
|
80
|
+
throttleInfo: indexing.throttleMessage !== null ? 'Requests retrying...' : null,
|
|
81
81
|
color,
|
|
82
82
|
};
|
|
83
83
|
}
|
|
@@ -74,6 +74,7 @@ export declare class IndexingService extends TypedEmitter<IndexingServiceEvents>
|
|
|
74
74
|
private logger;
|
|
75
75
|
private debugLogger;
|
|
76
76
|
private readonly externalStorage;
|
|
77
|
+
private suppressEvents;
|
|
77
78
|
constructor(projectRoot: string, options?: IndexingServiceOptions | Logger);
|
|
78
79
|
/**
|
|
79
80
|
* Run the indexing pipeline.
|
|
@@ -107,6 +107,12 @@ export class IndexingService extends TypedEmitter {
|
|
|
107
107
|
writable: true,
|
|
108
108
|
value: void 0
|
|
109
109
|
});
|
|
110
|
+
Object.defineProperty(this, "suppressEvents", {
|
|
111
|
+
enumerable: true,
|
|
112
|
+
configurable: true,
|
|
113
|
+
writable: true,
|
|
114
|
+
value: false
|
|
115
|
+
});
|
|
110
116
|
this.projectRoot = projectRoot;
|
|
111
117
|
this.debugLogger = createServiceLogger(projectRoot, 'indexer');
|
|
112
118
|
// Handle both old (logger) and new (options) signatures for backward compatibility
|
|
@@ -157,6 +163,7 @@ export class IndexingService extends TypedEmitter {
|
|
|
157
163
|
const { force = false } = options;
|
|
158
164
|
const failedFilesThisRun = new Set();
|
|
159
165
|
const failedBatches = [];
|
|
166
|
+
this.suppressEvents = false;
|
|
160
167
|
// Emit start event
|
|
161
168
|
this.emit('start');
|
|
162
169
|
this.emit('slots-reset');
|
|
@@ -265,6 +272,9 @@ export class IndexingService extends TypedEmitter {
|
|
|
265
272
|
let chunksProcessed = 0;
|
|
266
273
|
// Emit progress helper
|
|
267
274
|
const emitProgress = () => {
|
|
275
|
+
if (this.suppressEvents) {
|
|
276
|
+
return;
|
|
277
|
+
}
|
|
268
278
|
this.emit('progress', {
|
|
269
279
|
current: chunksProcessed,
|
|
270
280
|
total: totalChunks,
|
|
@@ -275,9 +285,10 @@ export class IndexingService extends TypedEmitter {
|
|
|
275
285
|
// Wire throttle callback for rate limit feedback
|
|
276
286
|
if ('onThrottle' in embeddings) {
|
|
277
287
|
embeddings.onThrottle = message => {
|
|
278
|
-
if (
|
|
279
|
-
|
|
288
|
+
if (this.suppressEvents) {
|
|
289
|
+
return;
|
|
280
290
|
}
|
|
291
|
+
this.emit('throttle', { message });
|
|
281
292
|
};
|
|
282
293
|
}
|
|
283
294
|
// Wire slot progress callbacks for API providers
|
|
@@ -319,6 +330,9 @@ export class IndexingService extends TypedEmitter {
|
|
|
319
330
|
// Wire up per-chunk progress callback
|
|
320
331
|
if ('onBatchProgress' in embeddings) {
|
|
321
332
|
embeddings.onBatchProgress = (processed, _total) => {
|
|
333
|
+
if (this.suppressEvents) {
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
322
336
|
const delta = processed - lastReportedWithinBatch;
|
|
323
337
|
if (delta > 0) {
|
|
324
338
|
chunksProcessed += delta;
|
|
@@ -426,6 +440,7 @@ export class IndexingService extends TypedEmitter {
|
|
|
426
440
|
return stats;
|
|
427
441
|
}
|
|
428
442
|
catch (error) {
|
|
443
|
+
this.suppressEvents = true;
|
|
429
444
|
this.log('error', 'Indexing failed', error);
|
|
430
445
|
this.emit('error', {
|
|
431
446
|
error: error instanceof Error ? error : new Error(String(error)),
|
|
@@ -440,21 +455,33 @@ export class IndexingService extends TypedEmitter {
|
|
|
440
455
|
const provider = embeddings;
|
|
441
456
|
if ('onSlotProcessing' in provider) {
|
|
442
457
|
provider.onSlotProcessing = (slot, batchInfo) => {
|
|
458
|
+
if (this.suppressEvents) {
|
|
459
|
+
return;
|
|
460
|
+
}
|
|
443
461
|
this.emit('slot-processing', { slot, batchInfo });
|
|
444
462
|
};
|
|
445
463
|
}
|
|
446
464
|
if ('onSlotRateLimited' in provider) {
|
|
447
465
|
provider.onSlotRateLimited = (slot, _batchInfo, retryInfo) => {
|
|
466
|
+
if (this.suppressEvents) {
|
|
467
|
+
return;
|
|
468
|
+
}
|
|
448
469
|
this.emit('slot-rate-limited', { slot, retryInfo });
|
|
449
470
|
};
|
|
450
471
|
}
|
|
451
472
|
if ('onSlotIdle' in provider) {
|
|
452
473
|
provider.onSlotIdle = slot => {
|
|
474
|
+
if (this.suppressEvents) {
|
|
475
|
+
return;
|
|
476
|
+
}
|
|
453
477
|
this.emit('slot-idle', { slot });
|
|
454
478
|
};
|
|
455
479
|
}
|
|
456
480
|
if ('onSlotFailure' in provider) {
|
|
457
481
|
provider.onSlotFailure = data => {
|
|
482
|
+
if (this.suppressEvents) {
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
458
485
|
const failure = {
|
|
459
486
|
batchInfo: data.batchInfo,
|
|
460
487
|
files: data.files,
|
|
@@ -474,6 +501,9 @@ export class IndexingService extends TypedEmitter {
|
|
|
474
501
|
}
|
|
475
502
|
if ('onResetSlots' in provider) {
|
|
476
503
|
provider.onResetSlots = () => {
|
|
504
|
+
if (this.suppressEvents) {
|
|
505
|
+
return;
|
|
506
|
+
}
|
|
477
507
|
this.emit('slots-reset');
|
|
478
508
|
};
|
|
479
509
|
}
|
|
@@ -614,6 +644,7 @@ export class IndexingService extends TypedEmitter {
|
|
|
614
644
|
* Close all resources.
|
|
615
645
|
*/
|
|
616
646
|
close() {
|
|
647
|
+
this.suppressEvents = true;
|
|
617
648
|
// Only close storage if we created it (not external)
|
|
618
649
|
if (!this.externalStorage) {
|
|
619
650
|
this.storage?.close();
|
|
@@ -296,7 +296,12 @@ export class Storage {
|
|
|
296
296
|
this.ensureConnected();
|
|
297
297
|
if (entries.length === 0)
|
|
298
298
|
return;
|
|
299
|
-
|
|
299
|
+
// Deduplicate by content hash to avoid merge-insert ambiguity.
|
|
300
|
+
const uniqueEntries = new Map();
|
|
301
|
+
for (const entry of entries) {
|
|
302
|
+
uniqueEntries.set(entry.contentHash, entry);
|
|
303
|
+
}
|
|
304
|
+
const rows = Array.from(uniqueEntries.values()).map(embeddingToRow);
|
|
300
305
|
// Use merge insert for upsert behavior
|
|
301
306
|
await this.getCacheTable()
|
|
302
307
|
.mergeInsert('content_hash')
|
|
@@ -45,9 +45,9 @@ export interface IndexingEvents {
|
|
|
45
45
|
'chunk-progress': [data: {
|
|
46
46
|
chunksProcessed: number;
|
|
47
47
|
}];
|
|
48
|
-
/** Rate limiting message from embedding provider */
|
|
48
|
+
/** Rate limiting message from embedding provider (null clears) */
|
|
49
49
|
throttle: [data: {
|
|
50
|
-
message: string;
|
|
50
|
+
message: string | null;
|
|
51
51
|
}];
|
|
52
52
|
/** Indexing completed successfully */
|
|
53
53
|
complete: [data: {
|