recallx 1.0.0 → 1.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/app/cli/src/cli.js +8 -1
- package/app/cli/src/format.js +54 -0
- package/app/cli/src/http.js +1 -1
- package/app/server/app.js +65 -3
- package/app/server/observability.js +381 -1
- package/app/server/repositories.js +241 -41
- package/app/server/workspace-session.js +1 -0
- package/app/shared/version.js +1 -1
- package/dist/renderer/assets/{ProjectGraphCanvas-BMvz9DmE.js → ProjectGraphCanvas-BVlOgk7l.js} +1 -1
- package/dist/renderer/assets/{index-CrDu22h7.js → index-JtUcLLhP.js} +3 -3
- package/dist/renderer/index.html +1 -1
- package/package.json +2 -2
package/README.md
CHANGED
|
@@ -160,7 +160,7 @@ At `/`, the headless runtime returns a small runtime notice instead of the rende
|
|
|
160
160
|
|
|
161
161
|
Node requirements:
|
|
162
162
|
|
|
163
|
-
- npm packages: Node
|
|
163
|
+
- npm packages: Node 22.13+
|
|
164
164
|
- local source development: Node 25+ is recommended because the backend uses `node:sqlite`
|
|
165
165
|
|
|
166
166
|
## Use From Other Coding Agents
|
package/app/cli/src/cli.js
CHANGED
|
@@ -726,7 +726,14 @@ function buildMcpConfigPayload(launcherPath) {
|
|
|
726
726
|
}
|
|
727
727
|
async function installMcpLauncher(launcherPath, commandParts) {
|
|
728
728
|
await mkdir(path.dirname(launcherPath), { recursive: true });
|
|
729
|
-
await writeFile(launcherPath, `#!/bin/sh
|
|
729
|
+
await writeFile(launcherPath, `#!/bin/sh
|
|
730
|
+
NODE_BIN="\${RECALLX_NODE_BIN:-$(command -v node 2>/dev/null || true)}"
|
|
731
|
+
if [ -z "$NODE_BIN" ]; then
|
|
732
|
+
echo "recallx-mcp launcher could not find a node executable in PATH." >&2
|
|
733
|
+
exit 1
|
|
734
|
+
fi
|
|
735
|
+
exec "$NODE_BIN" ${commandParts.slice(1).map(quoteShellArg).join(" ")} "$@"
|
|
736
|
+
`, "utf8");
|
|
730
737
|
await chmod(launcherPath, 0o755);
|
|
731
738
|
}
|
|
732
739
|
function quoteShellArg(value) {
|
package/app/cli/src/format.js
CHANGED
|
@@ -202,11 +202,65 @@ export function renderTelemetrySummary(data) {
|
|
|
202
202
|
`fts fallback: ${data.ftsFallbackRate.fallbackCount}/${data.ftsFallbackRate.sampleCount} (${data.ftsFallbackRate.ratio ?? "n/a"})`
|
|
203
203
|
);
|
|
204
204
|
}
|
|
205
|
+
if (data?.searchHitRate) {
|
|
206
|
+
lines.push(
|
|
207
|
+
`search hit rate: ${data.searchHitRate.hitCount}/${data.searchHitRate.sampleCount} (${data.searchHitRate.ratio ?? "n/a"})`
|
|
208
|
+
);
|
|
209
|
+
const searchHitOps = Array.isArray(data.searchHitRate.operations) ? data.searchHitRate.operations : [];
|
|
210
|
+
for (const item of searchHitOps.slice(0, 5)) {
|
|
211
|
+
lines.push(
|
|
212
|
+
`- [${item.surface}] ${item.operation}: ${item.hitCount}/${item.sampleCount} hits (${item.ratio ?? "n/a"})`
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
if (data?.searchLexicalQualityRate) {
|
|
217
|
+
lines.push(
|
|
218
|
+
`lexical quality: strong=${data.searchLexicalQualityRate.strongCount}, weak=${data.searchLexicalQualityRate.weakCount}, none=${data.searchLexicalQualityRate.noneCount}`
|
|
219
|
+
);
|
|
220
|
+
}
|
|
221
|
+
if (data?.workspaceResultCompositionRate) {
|
|
222
|
+
lines.push(
|
|
223
|
+
`workspace composition: node_only=${data.workspaceResultCompositionRate.nodeOnlyCount}, semantic_node_only=${data.workspaceResultCompositionRate.semanticNodeOnlyCount}, activity_only=${data.workspaceResultCompositionRate.activityOnlyCount}, mixed=${data.workspaceResultCompositionRate.mixedCount}, semantic_mixed=${data.workspaceResultCompositionRate.semanticMixedCount}, empty=${data.workspaceResultCompositionRate.emptyCount}`
|
|
224
|
+
);
|
|
225
|
+
}
|
|
226
|
+
if (data?.workspaceFallbackModeRate) {
|
|
227
|
+
lines.push(
|
|
228
|
+
`workspace fallback modes: strict_zero=${data.workspaceFallbackModeRate.strictZeroCount}, no_strong_node_hit=${data.workspaceFallbackModeRate.noStrongNodeHitCount}`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
if (data?.searchFeedbackRate) {
|
|
232
|
+
lines.push(
|
|
233
|
+
`search feedback: useful=${data.searchFeedbackRate.usefulCount}/${data.searchFeedbackRate.sampleCount} (${data.searchFeedbackRate.usefulRatio ?? "n/a"}), top1=${data.searchFeedbackRate.top1UsefulCount}/${data.searchFeedbackRate.top1SampleCount} (${data.searchFeedbackRate.top1UsefulRatio ?? "n/a"}), top3=${data.searchFeedbackRate.top3UsefulCount}/${data.searchFeedbackRate.top3SampleCount} (${data.searchFeedbackRate.top3UsefulRatio ?? "n/a"}), semantic_fp=${data.searchFeedbackRate.semanticFalsePositiveRatio ?? "n/a"}`
|
|
234
|
+
);
|
|
235
|
+
const qualityBuckets = Array.isArray(data.searchFeedbackRate.byLexicalQuality) ? data.searchFeedbackRate.byLexicalQuality : [];
|
|
236
|
+
for (const item of qualityBuckets) {
|
|
237
|
+
lines.push(
|
|
238
|
+
`- feedback [${item.lexicalQuality}]: ${item.usefulCount}/${item.sampleCount} useful (${item.usefulRatio ?? "n/a"})`
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
const fallbackModeBuckets = Array.isArray(data.searchFeedbackRate.byFallbackMode) ? data.searchFeedbackRate.byFallbackMode : [];
|
|
242
|
+
for (const item of fallbackModeBuckets) {
|
|
243
|
+
lines.push(
|
|
244
|
+
`- feedback mode [${item.fallbackMode}]: ${item.usefulCount}/${item.sampleCount} useful (${item.usefulRatio ?? "n/a"})`
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
205
248
|
if (data?.semanticAugmentationRate) {
|
|
206
249
|
lines.push(
|
|
207
250
|
`semantic augmentation: ${data.semanticAugmentationRate.usedCount}/${data.semanticAugmentationRate.sampleCount} (${data.semanticAugmentationRate.ratio ?? "n/a"})`
|
|
208
251
|
);
|
|
209
252
|
}
|
|
253
|
+
if (data?.semanticFallbackRate) {
|
|
254
|
+
lines.push(
|
|
255
|
+
`semantic fallback: eligible=${data.semanticFallbackRate.eligibleCount}, attempted=${data.semanticFallbackRate.attemptedCount}, hit=${data.semanticFallbackRate.hitCount}, hit_ratio=${data.semanticFallbackRate.hitRatio ?? "n/a"}`
|
|
256
|
+
);
|
|
257
|
+
const fallbackModes = Array.isArray(data.semanticFallbackRate.modes) ? data.semanticFallbackRate.modes : [];
|
|
258
|
+
for (const item of fallbackModes) {
|
|
259
|
+
lines.push(
|
|
260
|
+
`- fallback mode [${item.fallbackMode}]: attempted=${item.attemptedCount}/${item.eligibleCount} (${item.attemptRatio ?? "n/a"}), hit=${item.hitCount}/${item.attemptedCount} (${item.hitRatio ?? "n/a"})`
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
210
264
|
|
|
211
265
|
return `${lines.join("\n")}\n`;
|
|
212
266
|
}
|
package/app/cli/src/http.js
CHANGED
|
@@ -6,7 +6,7 @@ export function getApiBase(argvOptions = {}, env = process.env) {
|
|
|
6
6
|
DEFAULT_API_BASE);
|
|
7
7
|
}
|
|
8
8
|
export function getAuthToken(argvOptions = {}, env = process.env) {
|
|
9
|
-
return argvOptions.token || env.RECALLX_TOKEN || "";
|
|
9
|
+
return argvOptions.token || env.RECALLX_API_TOKEN || env.RECALLX_TOKEN || "";
|
|
10
10
|
}
|
|
11
11
|
export async function requestJson(apiBase, path, { method = "GET", token, body } = {}) {
|
|
12
12
|
const response = await fetch(buildApiUrl(apiBase, path), buildApiRequestInit({ method, token, body }));
|
package/app/server/app.js
CHANGED
|
@@ -8,7 +8,7 @@ import { activitySearchSchema, appendActivitySchema, appendRelationUsageEventSch
|
|
|
8
8
|
import { AppError } from "./errors.js";
|
|
9
9
|
import { isShortLogLikeAgentNodeInput, maybeCreatePromotionCandidate, recomputeAutomaticGovernance, resolveGovernancePolicy, resolveNodeGovernance, resolveRelationStatus, shouldPromoteActivitySummary } from "./governance.js";
|
|
10
10
|
import { refreshAutomaticInferredRelationsForNode, reindexAutomaticInferredRelations } from "./inferred-relations.js";
|
|
11
|
-
import { createObservabilityWriter, summarizePayloadShape } from "./observability.js";
|
|
11
|
+
import { appendCurrentTelemetryDetails, createObservabilityWriter, summarizePayloadShape } from "./observability.js";
|
|
12
12
|
import { buildSemanticCandidateBonusMap, buildCandidateRelationBonusMap, buildContextBundle, buildNeighborhoodItems, buildTargetRelatedRetrievalItems, bundleAsMarkdown, computeRankCandidateScore, shouldUseSemanticCandidateAugmentation } from "./retrieval.js";
|
|
13
13
|
import { buildProjectGraph } from "./project-graph.js";
|
|
14
14
|
import { createId, isPathWithinRoot } from "./utils.js";
|
|
@@ -172,6 +172,9 @@ function parseBooleanSetting(value, fallback) {
|
|
|
172
172
|
function parseNumberSetting(value, fallback) {
|
|
173
173
|
return typeof value === "number" && Number.isFinite(value) ? value : fallback;
|
|
174
174
|
}
|
|
175
|
+
function readWorkspaceSemanticFallbackMode(value) {
|
|
176
|
+
return value === "strict_zero" || value === "no_strong_node_hit" ? value : null;
|
|
177
|
+
}
|
|
175
178
|
function normalizeApiRequestPath(value) {
|
|
176
179
|
return value
|
|
177
180
|
.replace(/^\/artifacts\/.+$/g, "/artifacts/:path")
|
|
@@ -1406,6 +1409,12 @@ export function createRecallXApp(params) {
|
|
|
1406
1409
|
}, (span) => {
|
|
1407
1410
|
const searchResult = currentRepository().searchNodes(input);
|
|
1408
1411
|
span.addDetails({
|
|
1412
|
+
searchHit: searchResult.items.length > 0,
|
|
1413
|
+
bestLexicalQuality: searchResult.items.some((item) => item.lexicalQuality === "strong")
|
|
1414
|
+
? "strong"
|
|
1415
|
+
: searchResult.items.some((item) => item.lexicalQuality === "weak")
|
|
1416
|
+
? "weak"
|
|
1417
|
+
: "none",
|
|
1409
1418
|
resultCount: searchResult.items.length,
|
|
1410
1419
|
totalCount: searchResult.total
|
|
1411
1420
|
});
|
|
@@ -1423,6 +1432,12 @@ export function createRecallXApp(params) {
|
|
|
1423
1432
|
}, (span) => {
|
|
1424
1433
|
const searchResult = currentRepository().searchActivities(input);
|
|
1425
1434
|
span.addDetails({
|
|
1435
|
+
searchHit: searchResult.items.length > 0,
|
|
1436
|
+
bestLexicalQuality: searchResult.items.some((item) => item.lexicalQuality === "strong")
|
|
1437
|
+
? "strong"
|
|
1438
|
+
: searchResult.items.some((item) => item.lexicalQuality === "weak")
|
|
1439
|
+
? "weak"
|
|
1440
|
+
: "none",
|
|
1426
1441
|
resultCount: searchResult.items.length,
|
|
1427
1442
|
totalCount: searchResult.total
|
|
1428
1443
|
});
|
|
@@ -1457,6 +1472,7 @@ export function createRecallXApp(params) {
|
|
|
1457
1472
|
})
|
|
1458
1473
|
});
|
|
1459
1474
|
span.addDetails({
|
|
1475
|
+
searchHit: searchResult.items.length > 0,
|
|
1460
1476
|
resultCount: searchResult.items.length,
|
|
1461
1477
|
totalCount: searchResult.total
|
|
1462
1478
|
});
|
|
@@ -1861,9 +1877,55 @@ export function createRecallXApp(params) {
|
|
|
1861
1877
|
governance: buildGovernancePayload(repository, "relation", event.relationId, governanceResult.items[0] ?? repository.getGovernanceStateNullable("relation", event.relationId))
|
|
1862
1878
|
}));
|
|
1863
1879
|
});
|
|
1864
|
-
app.post("/api/v1/search-feedback-events", (request, response) => {
|
|
1880
|
+
app.post("/api/v1/search-feedback-events", handleAsyncRoute(async (request, response) => {
|
|
1865
1881
|
const repository = currentRepository();
|
|
1866
1882
|
const event = repository.appendSearchFeedbackEvent(appendSearchFeedbackSchema.parse(request.body ?? {}));
|
|
1883
|
+
const metadata = event.metadata ?? {};
|
|
1884
|
+
const fallbackMode = readWorkspaceSemanticFallbackMode(metadata.semanticFallbackMode);
|
|
1885
|
+
const lexicalQuality = metadata.lexicalQuality === "strong" || metadata.lexicalQuality === "weak" || metadata.lexicalQuality === "none"
|
|
1886
|
+
? metadata.lexicalQuality
|
|
1887
|
+
: null;
|
|
1888
|
+
const feedbackRank = typeof metadata.rank === "number" && Number.isFinite(metadata.rank)
|
|
1889
|
+
? metadata.rank
|
|
1890
|
+
: typeof metadata.resultRank === "number" && Number.isFinite(metadata.resultRank)
|
|
1891
|
+
? metadata.resultRank
|
|
1892
|
+
: null;
|
|
1893
|
+
const matchStrategy = metadata.matchStrategy === "fts" ||
|
|
1894
|
+
metadata.matchStrategy === "like" ||
|
|
1895
|
+
metadata.matchStrategy === "fallback_token" ||
|
|
1896
|
+
metadata.matchStrategy === "semantic" ||
|
|
1897
|
+
metadata.matchStrategy === "browse"
|
|
1898
|
+
? metadata.matchStrategy
|
|
1899
|
+
: null;
|
|
1900
|
+
await observability.recordEvent({
|
|
1901
|
+
surface: "api",
|
|
1902
|
+
operation: "search.feedback",
|
|
1903
|
+
details: {
|
|
1904
|
+
feedbackVerdict: event.verdict,
|
|
1905
|
+
feedbackResultType: event.resultType,
|
|
1906
|
+
feedbackLexicalQuality: lexicalQuality,
|
|
1907
|
+
feedbackConfidence: event.confidence,
|
|
1908
|
+
feedbackRank,
|
|
1909
|
+
feedbackMatchStrategy: matchStrategy,
|
|
1910
|
+
feedbackSemanticLifted: metadata.semanticLifted === true,
|
|
1911
|
+
feedbackSemanticFallbackMode: fallbackMode ?? undefined,
|
|
1912
|
+
feedbackHasQuery: Boolean(event.query?.trim()),
|
|
1913
|
+
feedbackHasRunId: Boolean(event.runId),
|
|
1914
|
+
feedbackHasSessionId: Boolean(event.sessionId)
|
|
1915
|
+
}
|
|
1916
|
+
});
|
|
1917
|
+
appendCurrentTelemetryDetails({
|
|
1918
|
+
feedbackVerdict: event.verdict,
|
|
1919
|
+
feedbackResultType: event.resultType,
|
|
1920
|
+
feedbackLexicalQuality: lexicalQuality,
|
|
1921
|
+
feedbackMatchStrategy: matchStrategy,
|
|
1922
|
+
feedbackRank,
|
|
1923
|
+
feedbackSemanticLifted: metadata.semanticLifted === true,
|
|
1924
|
+
feedbackSemanticFallbackMode: fallbackMode ?? undefined,
|
|
1925
|
+
feedbackHasQuery: Boolean(event.query?.trim()),
|
|
1926
|
+
feedbackHasRunId: Boolean(event.runId),
|
|
1927
|
+
feedbackHasSessionId: Boolean(event.sessionId)
|
|
1928
|
+
});
|
|
1867
1929
|
const governanceResult = event.resultType === "node"
|
|
1868
1930
|
? recomputeGovernanceForEntities("node", [event.resultId])
|
|
1869
1931
|
: { items: [] };
|
|
@@ -1878,7 +1940,7 @@ export function createRecallXApp(params) {
|
|
|
1878
1940
|
? buildGovernancePayload(repository, "node", event.resultId, governanceResult.items[0] ?? repository.getGovernanceStateNullable("node", event.resultId))
|
|
1879
1941
|
: null
|
|
1880
1942
|
}));
|
|
1881
|
-
});
|
|
1943
|
+
}));
|
|
1882
1944
|
app.post("/api/v1/inferred-relations/recompute", handleAsyncRoute(async (request, response) => {
|
|
1883
1945
|
const input = recomputeInferredRelationsSchema.parse(request.body ?? {});
|
|
1884
1946
|
const isFullMaintenancePass = !input.generator && !input.relationIds?.length;
|
|
@@ -340,8 +340,44 @@ export class ObservabilityWriter {
|
|
|
340
340
|
const buckets = new Map();
|
|
341
341
|
const mcpFailures = new Map();
|
|
342
342
|
const autoJobs = new Map();
|
|
343
|
+
const searchHitBuckets = new Map();
|
|
344
|
+
const lexicalQualityBuckets = new Map();
|
|
345
|
+
const workspaceFallbackModeBuckets = new Map();
|
|
346
|
+
const feedbackByLexicalQuality = new Map();
|
|
347
|
+
const feedbackByFallbackMode = new Map();
|
|
348
|
+
const semanticFallbackByMode = new Map();
|
|
343
349
|
let ftsFallbackCount = 0;
|
|
344
350
|
let ftsSampleCount = 0;
|
|
351
|
+
let searchHitCount = 0;
|
|
352
|
+
let searchMissCount = 0;
|
|
353
|
+
let searchSampleCount = 0;
|
|
354
|
+
let strongLexicalCount = 0;
|
|
355
|
+
let weakLexicalCount = 0;
|
|
356
|
+
let noLexicalCount = 0;
|
|
357
|
+
let lexicalSampleCount = 0;
|
|
358
|
+
let emptyCompositionCount = 0;
|
|
359
|
+
let nodeOnlyCompositionCount = 0;
|
|
360
|
+
let activityOnlyCompositionCount = 0;
|
|
361
|
+
let mixedCompositionCount = 0;
|
|
362
|
+
let semanticNodeOnlyCompositionCount = 0;
|
|
363
|
+
let semanticMixedCompositionCount = 0;
|
|
364
|
+
let compositionSampleCount = 0;
|
|
365
|
+
let strictZeroFallbackModeCount = 0;
|
|
366
|
+
let noStrongNodeHitFallbackModeCount = 0;
|
|
367
|
+
let workspaceFallbackModeSampleCount = 0;
|
|
368
|
+
let feedbackUsefulCount = 0;
|
|
369
|
+
let feedbackNotUsefulCount = 0;
|
|
370
|
+
let feedbackUncertainCount = 0;
|
|
371
|
+
let feedbackSampleCount = 0;
|
|
372
|
+
let feedbackTop1UsefulCount = 0;
|
|
373
|
+
let feedbackTop1SampleCount = 0;
|
|
374
|
+
let feedbackTop3UsefulCount = 0;
|
|
375
|
+
let feedbackTop3SampleCount = 0;
|
|
376
|
+
let feedbackSemanticUsefulCount = 0;
|
|
377
|
+
let feedbackSemanticNotUsefulCount = 0;
|
|
378
|
+
let feedbackSemanticSampleCount = 0;
|
|
379
|
+
let feedbackSemanticLiftUsefulCount = 0;
|
|
380
|
+
let feedbackSemanticLiftSampleCount = 0;
|
|
345
381
|
let semanticUsedCount = 0;
|
|
346
382
|
let semanticSampleCount = 0;
|
|
347
383
|
let semanticFallbackEligibleCount = 0;
|
|
@@ -354,6 +390,209 @@ export class ObservabilityWriter {
|
|
|
354
390
|
ftsFallbackCount += 1;
|
|
355
391
|
}
|
|
356
392
|
}
|
|
393
|
+
if (typeof event.details.searchHit === "boolean") {
|
|
394
|
+
searchSampleCount += 1;
|
|
395
|
+
if (event.details.searchHit) {
|
|
396
|
+
searchHitCount += 1;
|
|
397
|
+
}
|
|
398
|
+
else {
|
|
399
|
+
searchMissCount += 1;
|
|
400
|
+
}
|
|
401
|
+
const bucketKey = `${event.surface}:${event.operation}`;
|
|
402
|
+
const current = searchHitBuckets.get(bucketKey) ?? {
|
|
403
|
+
surface: event.surface,
|
|
404
|
+
operation: event.operation,
|
|
405
|
+
hitCount: 0,
|
|
406
|
+
missCount: 0
|
|
407
|
+
};
|
|
408
|
+
if (event.details.searchHit) {
|
|
409
|
+
current.hitCount += 1;
|
|
410
|
+
}
|
|
411
|
+
else {
|
|
412
|
+
current.missCount += 1;
|
|
413
|
+
}
|
|
414
|
+
searchHitBuckets.set(bucketKey, current);
|
|
415
|
+
}
|
|
416
|
+
if (event.details.bestLexicalQuality === "strong" ||
|
|
417
|
+
event.details.bestLexicalQuality === "weak" ||
|
|
418
|
+
event.details.bestLexicalQuality === "none" ||
|
|
419
|
+
event.details.bestNodeLexicalQuality === "strong" ||
|
|
420
|
+
event.details.bestNodeLexicalQuality === "weak" ||
|
|
421
|
+
event.details.bestNodeLexicalQuality === "none") {
|
|
422
|
+
const quality = (event.details.bestNodeLexicalQuality ??
|
|
423
|
+
event.details.bestLexicalQuality);
|
|
424
|
+
lexicalSampleCount += 1;
|
|
425
|
+
if (quality === "strong") {
|
|
426
|
+
strongLexicalCount += 1;
|
|
427
|
+
}
|
|
428
|
+
else if (quality === "weak") {
|
|
429
|
+
weakLexicalCount += 1;
|
|
430
|
+
}
|
|
431
|
+
else {
|
|
432
|
+
noLexicalCount += 1;
|
|
433
|
+
}
|
|
434
|
+
const bucketKey = `${event.surface}:${event.operation}`;
|
|
435
|
+
const current = lexicalQualityBuckets.get(bucketKey) ?? {
|
|
436
|
+
surface: event.surface,
|
|
437
|
+
operation: event.operation,
|
|
438
|
+
strongCount: 0,
|
|
439
|
+
weakCount: 0,
|
|
440
|
+
noneCount: 0
|
|
441
|
+
};
|
|
442
|
+
if (quality === "strong") {
|
|
443
|
+
current.strongCount += 1;
|
|
444
|
+
}
|
|
445
|
+
else if (quality === "weak") {
|
|
446
|
+
current.weakCount += 1;
|
|
447
|
+
}
|
|
448
|
+
else {
|
|
449
|
+
current.noneCount += 1;
|
|
450
|
+
}
|
|
451
|
+
lexicalQualityBuckets.set(bucketKey, current);
|
|
452
|
+
}
|
|
453
|
+
if (typeof event.details.resultComposition === "string") {
|
|
454
|
+
compositionSampleCount += 1;
|
|
455
|
+
switch (event.details.resultComposition) {
|
|
456
|
+
case "node_only":
|
|
457
|
+
nodeOnlyCompositionCount += 1;
|
|
458
|
+
break;
|
|
459
|
+
case "activity_only":
|
|
460
|
+
activityOnlyCompositionCount += 1;
|
|
461
|
+
break;
|
|
462
|
+
case "mixed":
|
|
463
|
+
mixedCompositionCount += 1;
|
|
464
|
+
break;
|
|
465
|
+
case "semantic_node_only":
|
|
466
|
+
semanticNodeOnlyCompositionCount += 1;
|
|
467
|
+
break;
|
|
468
|
+
case "semantic_mixed":
|
|
469
|
+
semanticMixedCompositionCount += 1;
|
|
470
|
+
break;
|
|
471
|
+
default:
|
|
472
|
+
emptyCompositionCount += 1;
|
|
473
|
+
break;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
if (event.operation === "workspace.search" &&
|
|
477
|
+
(event.details.semanticFallbackMode === "strict_zero" || event.details.semanticFallbackMode === "no_strong_node_hit")) {
|
|
478
|
+
workspaceFallbackModeSampleCount += 1;
|
|
479
|
+
if (event.details.semanticFallbackMode === "strict_zero") {
|
|
480
|
+
strictZeroFallbackModeCount += 1;
|
|
481
|
+
}
|
|
482
|
+
else {
|
|
483
|
+
noStrongNodeHitFallbackModeCount += 1;
|
|
484
|
+
}
|
|
485
|
+
const bucketKey = `${event.surface}:${event.operation}`;
|
|
486
|
+
const current = workspaceFallbackModeBuckets.get(bucketKey) ?? {
|
|
487
|
+
surface: event.surface,
|
|
488
|
+
operation: event.operation,
|
|
489
|
+
strictZeroCount: 0,
|
|
490
|
+
noStrongNodeHitCount: 0
|
|
491
|
+
};
|
|
492
|
+
if (event.details.semanticFallbackMode === "strict_zero") {
|
|
493
|
+
current.strictZeroCount += 1;
|
|
494
|
+
}
|
|
495
|
+
else {
|
|
496
|
+
current.noStrongNodeHitCount += 1;
|
|
497
|
+
}
|
|
498
|
+
workspaceFallbackModeBuckets.set(bucketKey, current);
|
|
499
|
+
}
|
|
500
|
+
if (event.details.feedbackVerdict === "useful" ||
|
|
501
|
+
event.details.feedbackVerdict === "not_useful" ||
|
|
502
|
+
event.details.feedbackVerdict === "uncertain") {
|
|
503
|
+
feedbackSampleCount += 1;
|
|
504
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
505
|
+
feedbackUsefulCount += 1;
|
|
506
|
+
}
|
|
507
|
+
else if (event.details.feedbackVerdict === "not_useful") {
|
|
508
|
+
feedbackNotUsefulCount += 1;
|
|
509
|
+
}
|
|
510
|
+
else {
|
|
511
|
+
feedbackUncertainCount += 1;
|
|
512
|
+
}
|
|
513
|
+
const feedbackRank = typeof event.details.feedbackRank === "number" && Number.isFinite(event.details.feedbackRank)
|
|
514
|
+
? event.details.feedbackRank
|
|
515
|
+
: null;
|
|
516
|
+
if (feedbackRank != null) {
|
|
517
|
+
if (feedbackRank <= 1) {
|
|
518
|
+
feedbackTop1SampleCount += 1;
|
|
519
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
520
|
+
feedbackTop1UsefulCount += 1;
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
if (feedbackRank <= 3) {
|
|
524
|
+
feedbackTop3SampleCount += 1;
|
|
525
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
526
|
+
feedbackTop3UsefulCount += 1;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
const feedbackMatchStrategy = event.details.feedbackMatchStrategy === "fts" ||
|
|
531
|
+
event.details.feedbackMatchStrategy === "like" ||
|
|
532
|
+
event.details.feedbackMatchStrategy === "fallback_token" ||
|
|
533
|
+
event.details.feedbackMatchStrategy === "semantic" ||
|
|
534
|
+
event.details.feedbackMatchStrategy === "browse"
|
|
535
|
+
? event.details.feedbackMatchStrategy
|
|
536
|
+
: null;
|
|
537
|
+
if (feedbackMatchStrategy === "semantic") {
|
|
538
|
+
feedbackSemanticSampleCount += 1;
|
|
539
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
540
|
+
feedbackSemanticUsefulCount += 1;
|
|
541
|
+
}
|
|
542
|
+
else if (event.details.feedbackVerdict === "not_useful") {
|
|
543
|
+
feedbackSemanticNotUsefulCount += 1;
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
if (event.details.feedbackSemanticLifted === true) {
|
|
547
|
+
feedbackSemanticLiftSampleCount += 1;
|
|
548
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
549
|
+
feedbackSemanticLiftUsefulCount += 1;
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
const lexicalQuality = event.details.feedbackLexicalQuality === "strong" ||
|
|
553
|
+
event.details.feedbackLexicalQuality === "weak" ||
|
|
554
|
+
event.details.feedbackLexicalQuality === "none"
|
|
555
|
+
? event.details.feedbackLexicalQuality
|
|
556
|
+
: null;
|
|
557
|
+
if (lexicalQuality) {
|
|
558
|
+
const current = feedbackByLexicalQuality.get(lexicalQuality) ?? {
|
|
559
|
+
usefulCount: 0,
|
|
560
|
+
notUsefulCount: 0,
|
|
561
|
+
uncertainCount: 0
|
|
562
|
+
};
|
|
563
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
564
|
+
current.usefulCount += 1;
|
|
565
|
+
}
|
|
566
|
+
else if (event.details.feedbackVerdict === "not_useful") {
|
|
567
|
+
current.notUsefulCount += 1;
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
current.uncertainCount += 1;
|
|
571
|
+
}
|
|
572
|
+
feedbackByLexicalQuality.set(lexicalQuality, current);
|
|
573
|
+
}
|
|
574
|
+
const feedbackFallbackMode = event.details.feedbackSemanticFallbackMode === "strict_zero" ||
|
|
575
|
+
event.details.feedbackSemanticFallbackMode === "no_strong_node_hit"
|
|
576
|
+
? event.details.feedbackSemanticFallbackMode
|
|
577
|
+
: null;
|
|
578
|
+
if (feedbackFallbackMode) {
|
|
579
|
+
const current = feedbackByFallbackMode.get(feedbackFallbackMode) ?? {
|
|
580
|
+
usefulCount: 0,
|
|
581
|
+
notUsefulCount: 0,
|
|
582
|
+
uncertainCount: 0
|
|
583
|
+
};
|
|
584
|
+
if (event.details.feedbackVerdict === "useful") {
|
|
585
|
+
current.usefulCount += 1;
|
|
586
|
+
}
|
|
587
|
+
else if (event.details.feedbackVerdict === "not_useful") {
|
|
588
|
+
current.notUsefulCount += 1;
|
|
589
|
+
}
|
|
590
|
+
else {
|
|
591
|
+
current.uncertainCount += 1;
|
|
592
|
+
}
|
|
593
|
+
feedbackByFallbackMode.set(feedbackFallbackMode, current);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
357
596
|
if (typeof event.details.semanticUsed === "boolean") {
|
|
358
597
|
semanticSampleCount += 1;
|
|
359
598
|
if (event.details.semanticUsed) {
|
|
@@ -375,6 +614,29 @@ export class ObservabilityWriter {
|
|
|
375
614
|
semanticFallbackHitCount += 1;
|
|
376
615
|
}
|
|
377
616
|
}
|
|
617
|
+
const semanticFallbackMode = event.operation === "workspace.search" &&
|
|
618
|
+
(event.details.semanticFallbackMode === "strict_zero" || event.details.semanticFallbackMode === "no_strong_node_hit")
|
|
619
|
+
? event.details.semanticFallbackMode
|
|
620
|
+
: null;
|
|
621
|
+
if (semanticFallbackMode) {
|
|
622
|
+
const current = semanticFallbackByMode.get(semanticFallbackMode) ?? {
|
|
623
|
+
eligibleCount: 0,
|
|
624
|
+
attemptedCount: 0,
|
|
625
|
+
hitCount: 0,
|
|
626
|
+
sampleCount: 0
|
|
627
|
+
};
|
|
628
|
+
current.sampleCount += 1;
|
|
629
|
+
if (event.details.semanticFallbackEligible === true) {
|
|
630
|
+
current.eligibleCount += 1;
|
|
631
|
+
}
|
|
632
|
+
if (event.details.semanticFallbackAttempted === true) {
|
|
633
|
+
current.attemptedCount += 1;
|
|
634
|
+
}
|
|
635
|
+
if (event.details.semanticFallbackUsed === true) {
|
|
636
|
+
current.hitCount += 1;
|
|
637
|
+
}
|
|
638
|
+
semanticFallbackByMode.set(semanticFallbackMode, current);
|
|
639
|
+
}
|
|
378
640
|
if (event.durationMs != null) {
|
|
379
641
|
const bucketKey = `${event.surface}:${event.operation}`;
|
|
380
642
|
const current = buckets.get(bucketKey) ??
|
|
@@ -440,6 +702,108 @@ export class ObservabilityWriter {
|
|
|
440
702
|
sampleCount: ftsSampleCount,
|
|
441
703
|
ratio: ftsSampleCount > 0 ? Number((ftsFallbackCount / ftsSampleCount).toFixed(4)) : null
|
|
442
704
|
},
|
|
705
|
+
searchHitRate: {
|
|
706
|
+
hitCount: searchHitCount,
|
|
707
|
+
missCount: searchMissCount,
|
|
708
|
+
sampleCount: searchSampleCount,
|
|
709
|
+
ratio: searchSampleCount > 0 ? Number((searchHitCount / searchSampleCount).toFixed(4)) : null,
|
|
710
|
+
operations: [...searchHitBuckets.values()]
|
|
711
|
+
.map((item) => ({
|
|
712
|
+
...item,
|
|
713
|
+
sampleCount: item.hitCount + item.missCount,
|
|
714
|
+
ratio: item.hitCount + item.missCount > 0
|
|
715
|
+
? Number((item.hitCount / (item.hitCount + item.missCount)).toFixed(4))
|
|
716
|
+
: null
|
|
717
|
+
}))
|
|
718
|
+
.sort((left, right) => right.sampleCount - left.sampleCount || left.operation.localeCompare(right.operation))
|
|
719
|
+
},
|
|
720
|
+
searchLexicalQualityRate: {
|
|
721
|
+
strongCount: strongLexicalCount,
|
|
722
|
+
weakCount: weakLexicalCount,
|
|
723
|
+
noneCount: noLexicalCount,
|
|
724
|
+
sampleCount: lexicalSampleCount,
|
|
725
|
+
operations: [...lexicalQualityBuckets.values()]
|
|
726
|
+
.map((item) => ({
|
|
727
|
+
...item,
|
|
728
|
+
sampleCount: item.strongCount + item.weakCount + item.noneCount
|
|
729
|
+
}))
|
|
730
|
+
.sort((left, right) => right.sampleCount - left.sampleCount || left.operation.localeCompare(right.operation))
|
|
731
|
+
},
|
|
732
|
+
workspaceResultCompositionRate: {
|
|
733
|
+
emptyCount: emptyCompositionCount,
|
|
734
|
+
nodeOnlyCount: nodeOnlyCompositionCount,
|
|
735
|
+
activityOnlyCount: activityOnlyCompositionCount,
|
|
736
|
+
mixedCount: mixedCompositionCount,
|
|
737
|
+
semanticNodeOnlyCount: semanticNodeOnlyCompositionCount,
|
|
738
|
+
semanticMixedCount: semanticMixedCompositionCount,
|
|
739
|
+
sampleCount: compositionSampleCount
|
|
740
|
+
},
|
|
741
|
+
workspaceFallbackModeRate: {
|
|
742
|
+
strictZeroCount: strictZeroFallbackModeCount,
|
|
743
|
+
noStrongNodeHitCount: noStrongNodeHitFallbackModeCount,
|
|
744
|
+
sampleCount: workspaceFallbackModeSampleCount,
|
|
745
|
+
operations: [...workspaceFallbackModeBuckets.values()]
|
|
746
|
+
.map((item) => ({
|
|
747
|
+
...item,
|
|
748
|
+
sampleCount: item.strictZeroCount + item.noStrongNodeHitCount
|
|
749
|
+
}))
|
|
750
|
+
.sort((left, right) => right.sampleCount - left.sampleCount || left.operation.localeCompare(right.operation))
|
|
751
|
+
},
|
|
752
|
+
searchFeedbackRate: {
|
|
753
|
+
usefulCount: feedbackUsefulCount,
|
|
754
|
+
notUsefulCount: feedbackNotUsefulCount,
|
|
755
|
+
uncertainCount: feedbackUncertainCount,
|
|
756
|
+
sampleCount: feedbackSampleCount,
|
|
757
|
+
usefulRatio: feedbackSampleCount > 0 ? Number((feedbackUsefulCount / feedbackSampleCount).toFixed(4)) : null,
|
|
758
|
+
top1UsefulCount: feedbackTop1UsefulCount,
|
|
759
|
+
top1SampleCount: feedbackTop1SampleCount,
|
|
760
|
+
top1UsefulRatio: feedbackTop1SampleCount > 0 ? Number((feedbackTop1UsefulCount / feedbackTop1SampleCount).toFixed(4)) : null,
|
|
761
|
+
top3UsefulCount: feedbackTop3UsefulCount,
|
|
762
|
+
top3SampleCount: feedbackTop3SampleCount,
|
|
763
|
+
top3UsefulRatio: feedbackTop3SampleCount > 0 ? Number((feedbackTop3UsefulCount / feedbackTop3SampleCount).toFixed(4)) : null,
|
|
764
|
+
semanticUsefulCount: feedbackSemanticUsefulCount,
|
|
765
|
+
semanticNotUsefulCount: feedbackSemanticNotUsefulCount,
|
|
766
|
+
semanticSampleCount: feedbackSemanticSampleCount,
|
|
767
|
+
semanticUsefulRatio: feedbackSemanticSampleCount > 0 ? Number((feedbackSemanticUsefulCount / feedbackSemanticSampleCount).toFixed(4)) : null,
|
|
768
|
+
semanticFalsePositiveRatio: feedbackSemanticSampleCount > 0 ? Number((feedbackSemanticNotUsefulCount / feedbackSemanticSampleCount).toFixed(4)) : null,
|
|
769
|
+
semanticLiftUsefulCount: feedbackSemanticLiftUsefulCount,
|
|
770
|
+
semanticLiftSampleCount: feedbackSemanticLiftSampleCount,
|
|
771
|
+
semanticLiftUsefulRatio: feedbackSemanticLiftSampleCount > 0
|
|
772
|
+
? Number((feedbackSemanticLiftUsefulCount / feedbackSemanticLiftSampleCount).toFixed(4))
|
|
773
|
+
: null,
|
|
774
|
+
byLexicalQuality: ["strong", "weak", "none"]
|
|
775
|
+
.map((lexicalQuality) => {
|
|
776
|
+
const counts = feedbackByLexicalQuality.get(lexicalQuality) ?? {
|
|
777
|
+
usefulCount: 0,
|
|
778
|
+
notUsefulCount: 0,
|
|
779
|
+
uncertainCount: 0
|
|
780
|
+
};
|
|
781
|
+
const sampleCount = counts.usefulCount + counts.notUsefulCount + counts.uncertainCount;
|
|
782
|
+
return {
|
|
783
|
+
lexicalQuality,
|
|
784
|
+
...counts,
|
|
785
|
+
sampleCount,
|
|
786
|
+
usefulRatio: sampleCount > 0 ? Number((counts.usefulCount / sampleCount).toFixed(4)) : null
|
|
787
|
+
};
|
|
788
|
+
})
|
|
789
|
+
.filter((item) => item.sampleCount > 0),
|
|
790
|
+
byFallbackMode: ["strict_zero", "no_strong_node_hit"]
|
|
791
|
+
.map((fallbackMode) => {
|
|
792
|
+
const counts = feedbackByFallbackMode.get(fallbackMode) ?? {
|
|
793
|
+
usefulCount: 0,
|
|
794
|
+
notUsefulCount: 0,
|
|
795
|
+
uncertainCount: 0
|
|
796
|
+
};
|
|
797
|
+
const sampleCount = counts.usefulCount + counts.notUsefulCount + counts.uncertainCount;
|
|
798
|
+
return {
|
|
799
|
+
fallbackMode,
|
|
800
|
+
...counts,
|
|
801
|
+
sampleCount,
|
|
802
|
+
usefulRatio: sampleCount > 0 ? Number((counts.usefulCount / sampleCount).toFixed(4)) : null
|
|
803
|
+
};
|
|
804
|
+
})
|
|
805
|
+
.filter((item) => item.sampleCount > 0)
|
|
806
|
+
},
|
|
443
807
|
semanticAugmentationRate: {
|
|
444
808
|
usedCount: semanticUsedCount,
|
|
445
809
|
sampleCount: semanticSampleCount,
|
|
@@ -454,7 +818,23 @@ export class ObservabilityWriter {
|
|
|
454
818
|
: null,
|
|
455
819
|
hitRatio: semanticFallbackAttemptedCount > 0
|
|
456
820
|
? Number((semanticFallbackHitCount / semanticFallbackAttemptedCount).toFixed(4))
|
|
457
|
-
: null
|
|
821
|
+
: null,
|
|
822
|
+
modes: ["strict_zero", "no_strong_node_hit"]
|
|
823
|
+
.map((fallbackMode) => {
|
|
824
|
+
const counts = semanticFallbackByMode.get(fallbackMode) ?? {
|
|
825
|
+
eligibleCount: 0,
|
|
826
|
+
attemptedCount: 0,
|
|
827
|
+
hitCount: 0,
|
|
828
|
+
sampleCount: 0
|
|
829
|
+
};
|
|
830
|
+
return {
|
|
831
|
+
fallbackMode,
|
|
832
|
+
...counts,
|
|
833
|
+
attemptRatio: counts.eligibleCount > 0 ? Number((counts.attemptedCount / counts.eligibleCount).toFixed(4)) : null,
|
|
834
|
+
hitRatio: counts.attemptedCount > 0 ? Number((counts.hitCount / counts.attemptedCount).toFixed(4)) : null
|
|
835
|
+
};
|
|
836
|
+
})
|
|
837
|
+
.filter((item) => item.sampleCount > 0)
|
|
458
838
|
},
|
|
459
839
|
autoJobStats: [...autoJobs.entries()].map(([operation, durations]) => ({
|
|
460
840
|
operation,
|