recallx 1.0.7 → 1.1.0

package/app/server/app.js

@@ -9,7 +9,7 @@ import { AppError } from "./errors.js";
 import { isShortLogLikeAgentNodeInput, maybeCreatePromotionCandidate, recomputeAutomaticGovernance, resolveGovernancePolicy, resolveNodeGovernance, resolveRelationStatus, shouldPromoteActivitySummary } from "./governance.js";
 import { refreshAutomaticInferredRelationsForNode, reindexAutomaticInferredRelations } from "./inferred-relations.js";
 import { appendCurrentTelemetryDetails, createObservabilityWriter, summarizePayloadShape } from "./observability.js";
-import { buildSemanticCandidateBonusMap, buildCandidateRelationBonusMap, buildContextBundle, buildNeighborhoodItems, buildTargetRelatedRetrievalItems, bundleAsMarkdown, computeRankCandidateScore, shouldUseSemanticCandidateAugmentation } from "./retrieval.js";
+import { buildSemanticCandidateBonusMap, buildCandidateRelationBonusMap, buildContextBundle, buildNeighborhoodItems, buildNeighborhoodItemsBatch, buildTargetRelatedRetrievalItems, bundleAsMarkdown, computeRankCandidateScore, selectSemanticCandidateIds, shouldUseSemanticCandidateAugmentation } from "./retrieval.js";
 import { buildProjectGraph } from "./project-graph.js";
 import { createId, isPathWithinRoot } from "./utils.js";
 const relationTypeSet = new Set(relationTypes);
@@ -546,7 +546,7 @@ export function createRecallXApp(params) {
             workspaceRoot: currentWorkspaceRoot(),
             workspaceName: currentWorkspaceInfo().workspaceName,
             retentionDays: Math.max(1, parseNumberSetting(settings["observability.retentionDays"], 14)),
-            slowRequestMs: Math.max(1, parseNumberSetting(settings["observability.slowRequestMs"], 250)),
+            slowRequestMs: Math.max(1, parseNumberSetting(settings["observability.slowRequestMs"], 50)),
             capturePayloadShape: parseBooleanSetting(settings["observability.capturePayloadShape"], true)
         };
     };
@@ -1779,19 +1779,17 @@ export function createRecallXApp(params) {
         }, (span) => {
             const repository = currentRepository();
             const nodeId = readRequestParam(request.params.id);
-            const result = buildNeighborhoodItems(repository, nodeId, {
+            const neighborhoodOptions = {
                 relationTypes: types,
                 includeInferred,
                 maxInferred
-            });
+            };
+            const result = buildNeighborhoodItems(repository, nodeId, neighborhoodOptions);
             const expanded = depth === 2
                 ? (() => {
                     const seen = new Set(result.map((item) => `${item.edge.relationId}:${item.node.id}:1`));
-                    const secondHop = result.flatMap((item) => buildNeighborhoodItems(repository, item.node.id, {
-                        relationTypes: types,
-                        includeInferred,
-                        maxInferred
-                    })
+                    const secondHopByNodeId = buildNeighborhoodItemsBatch(repository, result.map((item) => item.node.id), neighborhoodOptions);
+                    const secondHop = result.flatMap((item) => (secondHopByNodeId.get(item.node.id) ?? [])
                         .filter((nested) => nested.node.id !== nodeId)
                         .map((nested) => ({
                         ...nested,
@@ -1814,7 +1812,8 @@ export function createRecallXApp(params) {
                 })()
                 : result;
             span.addDetails({
-                resultCount: expanded.length
+                resultCount: expanded.length,
+                firstHopCount: result.length
             });
             return expanded;
         });
@@ -2154,8 +2153,9 @@ export function createRecallXApp(params) {
             });
             const semanticAugmentation = repository.getSemanticAugmentationSettings();
             const semanticEnabled = shouldUseSemanticCandidateAugmentation(query, candidates);
+            const semanticCandidateIds = selectSemanticCandidateIds(query, candidates);
             const semanticBonuses = semanticEnabled
-                ? buildSemanticCandidateBonusMap(await repository.rankSemanticCandidates(query, candidateNodeIds), semanticAugmentation)
+                ? buildSemanticCandidateBonusMap(await repository.rankSemanticCandidates(query, semanticCandidateIds), semanticAugmentation)
                 : new Map();
             const result = candidates
                 .map((node) => {
@@ -2179,7 +2179,9 @@ export function createRecallXApp(params) {
                 .sort((left, right) => right.score - left.score);
             span.addDetails({
                 resultCount: result.length,
-                semanticUsed: semanticEnabled
+                semanticUsed: semanticEnabled,
+                semanticCandidateCount: candidates.length,
+                semanticRankedCandidateCount: semanticCandidateIds.length
             });
             return result;
         });

package/app/server/inferred-relations.js

@@ -14,6 +14,9 @@ function normalizeText(value) {
 function normalizeTags(tags) {
     return Array.from(new Set(tags.map((tag) => normalizeText(tag)).filter(Boolean)));
 }
+function normalizeTexts(values) {
+    return values.map(normalizeText).filter(Boolean);
+}
 function escapeRegExp(value) {
     return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
 }
@@ -33,6 +36,10 @@ function mentionsNode(haystacks, candidate) {
         : false;
     return { idMention, titleMention };
 }
+function hasCheapMention(normalizedHaystacks, candidate) {
+    const normalizedTitle = normalizeText(candidate.title);
+    return normalizedHaystacks.some((haystack) => haystack.includes(candidate.id.toLowerCase()) || (normalizedTitle.length >= 5 && haystack.includes(normalizedTitle)));
+}
 function sortPair(left, right) {
     return left.localeCompare(right) <= 0 ? [left, right] : [right, left];
 }
@@ -154,36 +161,42 @@ function collectGeneratedCandidates(repository, target, trigger) {
         projectIds: new Set(projectMembershipsByNodeId.get(target.id) ?? []),
         artifactKeys: artifactKeysByNodeId.get(target.id) ?? { exactPaths: [], baseNames: [] }
     };
+    const normalizedTargetTexts = normalizeTexts([target.title, target.body, target.summary]);
     const activityBodies = trigger === "activity-append" || trigger === "reindex"
         ? repository
             .listNodeActivities(target.id, MAX_ACTIVITY_BODIES)
             .map((activity) => activity.body ?? "")
             .filter(Boolean)
         : [];
+    const normalizedActivityBodies = activityBodies.map(normalizeText).filter(Boolean);
     return candidates.flatMap((candidate) => {
         const generated = [];
+        const candidateProjectIds = projectMembershipsByNodeId.get(candidate.id) ?? [];
+        const candidateArtifacts = artifactKeysByNodeId.get(candidate.id) ?? { exactPaths: [], baseNames: [] };
         const tagOverlapCandidate = buildTagOverlapCandidate(target, candidate, targetContext.normalizedTags);
         if (tagOverlapCandidate) {
             generated.push(tagOverlapCandidate);
         }
-        const bodyReferenceCandidate = buildBodyReferenceCandidate(target, candidate);
+        const projectMembershipCandidate = buildProjectMembershipCandidate(target, candidate, candidateProjectIds, targetContext.projectIds);
+        if (projectMembershipCandidate) {
+            generated.push(projectMembershipCandidate);
+        }
+        const sharedArtifactCandidate = buildSharedArtifactCandidate(target, candidate, candidateArtifacts, targetContext.artifactKeys);
+        if (sharedArtifactCandidate) {
+            generated.push(sharedArtifactCandidate);
+        }
+        const candidateTexts = normalizeTexts([candidate.title, candidate.body, candidate.summary]);
+        const shouldCheckBodyReference = hasCheapMention(normalizedTargetTexts, candidate) || hasCheapMention(candidateTexts, target);
+        const bodyReferenceCandidate = shouldCheckBodyReference ? buildBodyReferenceCandidate(target, candidate) : null;
         if (bodyReferenceCandidate) {
             generated.push(bodyReferenceCandidate);
         }
-        if (activityBodies.length) {
+        if (normalizedActivityBodies.length && hasCheapMention(normalizedActivityBodies, candidate)) {
             const activityReferenceCandidate = buildActivityReferenceCandidate(target, candidate, activityBodies);
             if (activityReferenceCandidate) {
                 generated.push(activityReferenceCandidate);
             }
         }
-        const projectMembershipCandidate = buildProjectMembershipCandidate(target, candidate, projectMembershipsByNodeId.get(candidate.id) ?? [], targetContext.projectIds);
-        if (projectMembershipCandidate) {
-            generated.push(projectMembershipCandidate);
-        }
-        const sharedArtifactCandidate = buildSharedArtifactCandidate(target, candidate, artifactKeysByNodeId.get(candidate.id) ?? { exactPaths: [], baseNames: [] }, targetContext.artifactKeys);
-        if (sharedArtifactCandidate) {
-            generated.push(sharedArtifactCandidate);
-        }
         return generated;
     });
 }

package/app/server/observability.js

@@ -1,6 +1,8 @@
 import { AsyncLocalStorage } from "node:async_hooks";
-import { appendFile, mkdir, readFile, readdir, unlink } from "node:fs/promises";
+import { createReadStream } from "node:fs";
+import { appendFile, mkdir, readdir, unlink } from "node:fs/promises";
 import path from "node:path";
+import { createInterface } from "node:readline";
 import { createId } from "./utils.js";
 const telemetryStorage = new AsyncLocalStorage();
 function nowIso() {
@@ -45,6 +47,10 @@ function roundDuration(value) {
 function dateStamp(value) {
     return value.slice(0, 10);
 }
+function telemetryLogDate(entry) {
+    const match = /^telemetry-(\d{4}-\d{2}-\d{2})\.ndjson$/.exec(entry);
+    return match ? match[1] : null;
+}
 function normalizeRetentionDays(value) {
     return Math.max(1, Math.trunc(value || 14));
 }
@@ -341,24 +347,41 @@ export class ObservabilityWriter {
         }
         const files = entries
             .filter((entry) => /^telemetry-\d{4}-\d{2}-\d{2}\.ndjson$/.test(entry))
+            .filter((entry) => {
+            const stamp = telemetryLogDate(entry);
+            return stamp !== null && stamp >= dateStamp(new Date(sinceMs).toISOString());
+        })
             .sort();
         const events = [];
         for (const file of files) {
             const filePath = path.join(logsDir, file);
-            const content = await readFile(filePath, "utf8").catch(() => "");
-            for (const line of content.split("\n")) {
-                const event = parseJsonLine(line);
-                if (!event) {
-                    continue;
-                }
-                const eventMs = Date.parse(event.ts);
-                if (!Number.isFinite(eventMs) || eventMs < sinceMs) {
-                    continue;
-                }
-                if (options.surface && options.surface !== "all" && event.surface !== options.surface) {
-                    continue;
+            const stream = createReadStream(filePath, { encoding: "utf8" });
+            const lines = createInterface({
+                input: stream,
+                crlfDelay: Number.POSITIVE_INFINITY
+            });
+            try {
+                for await (const line of lines) {
+                    const event = parseJsonLine(line);
+                    if (!event) {
+                        continue;
+                    }
+                    const eventMs = Date.parse(event.ts);
+                    if (!Number.isFinite(eventMs) || eventMs < sinceMs) {
+                        continue;
+                    }
+                    if (options.surface && options.surface !== "all" && event.surface !== options.surface) {
+                        continue;
+                    }
+                    events.push(event);
                 }
-                events.push(event);
+            }
+            catch {
+                // Ignore individual file read failures so observability endpoints stay resilient.
+            }
+            finally {
+                lines.close();
+                stream.destroy();
             }
         }
         return {
@@ -724,7 +747,9 @@ export class ObservabilityWriter {
             generatedAt: nowIso(),
             logsPath,
             totalEvents: events.length,
+            slowRequestThresholdMs: state.slowRequestMs,
             operationSummaries,
+            hotOperations: operationSummaries.slice(0, 10),
             slowOperations: operationSummaries
                 .filter((item) => (item.p95DurationMs ?? 0) >= state.slowRequestMs)
                 .slice(0, 10),

package/app/server/repositories.js

@@ -59,12 +59,14 @@ function collectSearchFieldSignals(matcher, candidates) {
             matchedFields: [],
             exactFields: [],
             matchedTermCount: 0,
+            matchedTermCounts: {},
             totalTermCount: 0
         };
     }
     const matchedFields = new Set();
     const exactFields = new Set();
     const matchedTerms = new Set();
+    const matchedTermCounts = {};
     for (const candidate of candidates) {
         const haystack = normalizeSearchText(candidate.value);
         if (!haystack) {
@@ -76,6 +78,7 @@ function collectSearchFieldSignals(matcher, candidates) {
             continue;
         }
         matchedFields.add(candidate.field);
+        matchedTermCounts[candidate.field] = termMatches.length;
         if (exactMatch) {
             exactFields.add(candidate.field);
         }
@@ -87,6 +90,7 @@ function collectSearchFieldSignals(matcher, candidates) {
         matchedFields: [...matchedFields],
         exactFields: [...exactFields],
         matchedTermCount: matchedTerms.size,
+        matchedTermCounts,
         totalTermCount: matcher.matchTerms.length
     };
 }
@@ -94,14 +98,18 @@ function classifyNodeLexicalQuality(strategy, signals) {
     if (strategy === "browse" || strategy === "semantic" || !signals.matchedFields.length) {
         return "none";
     }
-    if (strategy === "fallback_token") {
-        return "weak";
-    }
-    const strongExactFields = new Set(["title", "summary", "tags"]);
+    const strongExactFields = new Set(["title", "summary", "tags", "body"]);
     if (signals.exactFields.some((field) => strongExactFields.has(field))) {
         return "strong";
     }
     const termCoverage = signals.totalTermCount > 0 ? signals.matchedTermCount / signals.totalTermCount : 0;
+    const titleCoverage = signals.totalTermCount > 0 ? (signals.matchedTermCounts.title ?? 0) / signals.totalTermCount : 0;
+    if (strategy === "fallback_token") {
+        return titleCoverage >= 0.5 ? "strong" : "weak";
+    }
+    if (strategy === "fts" && titleCoverage >= 0.5) {
+        return "strong";
+    }
     if (strategy === "fts" && termCoverage >= 0.6 && signals.matchedFields.some((field) => strongExactFields.has(field))) {
         return "strong";
     }
@@ -2312,25 +2320,59 @@ export class RecallXRepository {
             : "";
         const rows = this.db
             .prepare(`SELECT
-           r.*,
-           CASE WHEN r.from_node_id = ? THEN r.to_node_id ELSE r.from_node_id END AS related_id
+           r.id,
+           r.from_node_id,
+           r.to_node_id,
+           r.relation_type,
+           r.status,
+           r.created_by,
+           r.source_type,
+           r.source_label,
+           r.created_at,
+           r.metadata_json,
+           n.id AS node_id,
+           n.type AS node_type,
+           n.status AS node_status,
+           n.canonicality AS node_canonicality,
+           n.visibility AS node_visibility,
+           n.title AS node_title,
+           n.body AS node_body,
+           n.summary AS node_summary,
+           n.created_by AS node_created_by,
+           n.source_type AS node_source_type,
+           n.source_label AS node_source_label,
+           n.created_at AS node_created_at,
+           n.updated_at AS node_updated_at,
+           n.tags_json AS node_tags_json,
+           n.metadata_json AS node_metadata_json
          FROM relations r
+         JOIN nodes n
+           ON n.id = CASE WHEN r.from_node_id = ? THEN r.to_node_id ELSE r.from_node_id END
          WHERE (r.from_node_id = ? OR r.to_node_id = ?)
            AND r.status != 'archived'
            ${relationWhere}
          ORDER BY r.created_at DESC`)
             .all(nodeId, nodeId, nodeId, ...(relationFilter ?? []));
-        const relatedNodes = this.getNodesByIds(rows.map((row) => String(row.related_id)));
-        return rows.flatMap((row) => {
-            const node = relatedNodes.get(String(row.related_id));
-            if (!node) {
-                return [];
+        return rows.map((row) => ({
+            relation: mapRelation(row),
+            node: {
+                id: String(row.node_id),
+                type: row.node_type,
+                status: row.node_status,
+                canonicality: row.node_canonicality,
+                visibility: String(row.node_visibility),
+                title: row.node_title ? String(row.node_title) : null,
+                body: row.node_body ? String(row.node_body) : null,
+                summary: row.node_summary ? String(row.node_summary) : null,
+                createdBy: row.node_created_by ? String(row.node_created_by) : null,
+                sourceType: row.node_source_type ? String(row.node_source_type) : null,
+                sourceLabel: row.node_source_label ? String(row.node_source_label) : null,
+                createdAt: String(row.node_created_at),
+                updatedAt: String(row.node_updated_at),
+                tags: parseJson(row.node_tags_json, []),
+                metadata: parseJson(row.node_metadata_json, {})
             }
-            return [{
-                    relation: mapRelation(row),
-                    node
-                }];
-        });
+        }));
     }
     listProjectMemberNodes(projectId, limit) {
         const rows = this.db

package/app/server/retrieval.js

@@ -18,6 +18,7 @@ const boostedRelationRankWeights = {
 };
 const semanticCandidateMinSimilarity = 0.2;
 const semanticCandidateMaxBonus = 18;
+const maxSemanticPrefilterCandidates = 256;
 function resolveSemanticAugmentationSettings(settings) {
     return {
         minSimilarity: typeof settings?.minSimilarity === "number" && Number.isFinite(settings.minSimilarity)
@@ -38,7 +39,15 @@ function prioritizeItems(items, preset, maxItems, bonuses) {
         .map(({ item }) => item);
     return weighted.slice(0, maxItems);
 }
-function buildNeighborhoodResult(repository, nodeId, options) {
+function normalizeRetrievalText(value) {
+    return (value ?? "").toLowerCase().replace(/\s+/g, " ").trim();
+}
+function tokenizeRetrievalText(value) {
+    return normalizeRetrievalText(value)
+        .split(/[^a-z0-9]+/g)
+        .filter((token) => token.length >= 3);
+}
+function listNeighborhoodCandidates(repository, nodeId, options) {
     const canonicalItems = repository.listRelatedNodes(nodeId, 1, options?.relationTypes).map(({ node, relation }) => ({
         node,
         edge: {
@@ -94,6 +103,13 @@ function buildNeighborhoodResult(repository, nodeId, options) {
             });
         })()
         : [];
+    return {
+        canonicalItems,
+        inferredItems
+    };
+}
+function buildNeighborhoodResult(repository, nodeId, options) {
+    const { canonicalItems, inferredItems } = listNeighborhoodCandidates(repository, nodeId, options);
     const usageSummaries = repository.getRelationUsageSummaries([...canonicalItems, ...inferredItems].map((item) => item.edge.relationId));
     const rankedCanonical = rankNeighborhoodItems(canonicalItems, usageSummaries, neighborhoodRetrievalRankWeights);
     const rankedInferred = options?.includeInferred && options.maxInferred
@@ -104,6 +120,27 @@ function buildNeighborhoodResult(repository, nodeId, options) {
         usageSummaries
     };
 }
+export function buildNeighborhoodItemsBatch(repository, nodeIds, options) {
+    const uniqueNodeIds = Array.from(new Set(nodeIds.filter(Boolean)));
+    const rawByNodeId = new Map();
+    const relationIds = [];
+    for (const nodeId of uniqueNodeIds) {
+        const raw = listNeighborhoodCandidates(repository, nodeId, options);
+        rawByNodeId.set(nodeId, raw);
+        relationIds.push(...raw.canonicalItems.map((item) => item.edge.relationId));
+        relationIds.push(...raw.inferredItems.map((item) => item.edge.relationId));
+    }
+    const usageSummaries = repository.getRelationUsageSummaries(relationIds);
+    const results = new Map();
+    for (const [nodeId, raw] of rawByNodeId.entries()) {
+        const rankedCanonical = rankNeighborhoodItems(raw.canonicalItems, usageSummaries, neighborhoodRetrievalRankWeights);
+        const rankedInferred = options?.includeInferred && options.maxInferred
+            ? rankNeighborhoodItems(raw.inferredItems, usageSummaries, neighborhoodRetrievalRankWeights, options.maxInferred)
+            : [];
+        results.set(nodeId, [...rankedCanonical, ...rankedInferred]);
+    }
+    return results;
+}
 function matchesSearchResultFilters(item, filters) {
     const typeMatches = !filters.types?.length || filters.types.includes(item.type);
     const statusMatches = !filters.status?.length || filters.status.includes(item.status);
@@ -214,6 +251,45 @@ export function buildSemanticCandidateBonusMap(semanticMatches, settings) {
         ];
     }));
 }
+function scoreSemanticPrefilterCandidate(normalizedQuery, queryTokens, candidate) {
+    const normalizedTitle = normalizeRetrievalText(candidate.title);
+    const normalizedSummary = normalizeRetrievalText(candidate.summary);
+    let score = 0;
+    if (normalizedTitle.includes(normalizedQuery)) {
+        score += 12;
+    }
+    if (normalizedSummary.includes(normalizedQuery)) {
+        score += 6;
+    }
+    for (const token of queryTokens) {
+        if (normalizedTitle.includes(token)) {
+            score += 2;
+        }
+        if (normalizedSummary.includes(token)) {
+            score += 1;
+        }
+    }
+    return score;
+}
+export function selectSemanticCandidateIds(query, candidates, maxCandidates = maxSemanticPrefilterCandidates) {
+    if (candidates.length <= maxCandidates) {
+        return candidates.map((candidate) => candidate.id);
+    }
+    const normalizedQuery = normalizeRetrievalText(query);
+    if (!normalizedQuery) {
+        return candidates.slice(0, maxCandidates).map((candidate) => candidate.id);
+    }
+    const queryTokens = Array.from(new Set(tokenizeRetrievalText(query)));
+    return candidates
+        .map((candidate) => ({
+        id: candidate.id,
+        score: scoreSemanticPrefilterCandidate(normalizedQuery, queryTokens, candidate),
+        updatedAt: candidate.updatedAt
+    }))
+        .sort((left, right) => right.score - left.score || right.updatedAt.localeCompare(left.updatedAt))
+        .slice(0, maxCandidates)
+        .map((candidate) => candidate.id);
+}
 function computeBundleRelationBoost(item, summary) {
     return computeRelationRetrievalRank(item.edge, summary, {
         canonicalBase: 120,
@@ -397,15 +473,19 @@ export async function buildContextBundle(repository, input) {
     const candidateItems = [targetItem, ...relatedItems, ...decisions, ...openQuestions];
     const dedupedItems = Array.from(new Map(candidateItems.map((item) => [item.id, item])).values());
     const semanticQuery = [target.title, target.summary ?? target.body].filter(Boolean).join("\n");
-    const semanticBonuses = shouldUseSemanticCandidateAugmentation(semanticQuery, dedupedItems.filter((item) => item.id !== target.id))
-        ? buildSemanticCandidateBonusMap(await repository.rankSemanticCandidates(semanticQuery, dedupedItems.filter((item) => item.id !== target.id).map((item) => item.id)), repository.getSemanticAugmentationSettings())
+    const semanticCandidates = dedupedItems.filter((item) => item.id !== target.id);
+    const semanticCandidateIds = selectSemanticCandidateIds(semanticQuery, semanticCandidates);
+    const semanticBonuses = shouldUseSemanticCandidateAugmentation(semanticQuery, semanticCandidates)
+        ? buildSemanticCandidateBonusMap(await repository.rankSemanticCandidates(semanticQuery, semanticCandidateIds), repository.getSemanticAugmentationSettings())
         : new Map();
     appendCurrentTelemetryDetails({
         neighborhoodCount: neighborhood.length,
         relatedCandidateCount: relatedItems.length,
         decisionCount: decisions.length,
         openQuestionCount: openQuestions.length,
-        semanticUsed: semanticBonuses.size > 0
+        semanticUsed: semanticBonuses.size > 0,
+        semanticCandidateCount: semanticCandidates.length,
+        semanticRankedCandidateCount: semanticCandidateIds.length
     });
     const combinedBonuses = new Map();
     for (const item of dedupedItems) {

package/app/server/workspace-session.js

@@ -98,7 +98,7 @@ export class WorkspaceSessionManager {
             "relations.autoRecompute.lastRunAt": null,
             "observability.enabled": false,
             "observability.retentionDays": 14,
-            "observability.slowRequestMs": 250,
+            "observability.slowRequestMs": 50,
             "observability.capturePayloadShape": true,
             "export.defaultFormat": "markdown",
         });

package/app/shared/version.js

@@ -1 +1 @@
-export const RECALLX_VERSION = "1.0.7";
+export const RECALLX_VERSION = "1.1.0";