recallx 1.0.8 → 1.2.0

package/app/server/repositories.js

@@ -59,12 +59,14 @@ function collectSearchFieldSignals(matcher, candidates) {
             matchedFields: [],
             exactFields: [],
             matchedTermCount: 0,
+            matchedTermCounts: {},
             totalTermCount: 0
         };
     }
     const matchedFields = new Set();
     const exactFields = new Set();
     const matchedTerms = new Set();
+    const matchedTermCounts = {};
     for (const candidate of candidates) {
         const haystack = normalizeSearchText(candidate.value);
         if (!haystack) {
@@ -76,6 +78,7 @@ function collectSearchFieldSignals(matcher, candidates) {
             continue;
         }
         matchedFields.add(candidate.field);
+        matchedTermCounts[candidate.field] = termMatches.length;
         if (exactMatch) {
             exactFields.add(candidate.field);
         }
@@ -87,6 +90,7 @@ function collectSearchFieldSignals(matcher, candidates) {
         matchedFields: [...matchedFields],
         exactFields: [...exactFields],
         matchedTermCount: matchedTerms.size,
+        matchedTermCounts,
         totalTermCount: matcher.matchTerms.length
     };
 }
@@ -94,14 +98,18 @@ function classifyNodeLexicalQuality(strategy, signals) {
     if (strategy === "browse" || strategy === "semantic" || !signals.matchedFields.length) {
         return "none";
     }
-    if (strategy === "fallback_token") {
-        return "weak";
-    }
-    const strongExactFields = new Set(["title", "summary", "tags"]);
+    const strongExactFields = new Set(["title", "summary", "tags", "body"]);
     if (signals.exactFields.some((field) => strongExactFields.has(field))) {
         return "strong";
     }
     const termCoverage = signals.totalTermCount > 0 ? signals.matchedTermCount / signals.totalTermCount : 0;
+    const titleCoverage = signals.totalTermCount > 0 ? (signals.matchedTermCounts.title ?? 0) / signals.totalTermCount : 0;
+    if (strategy === "fallback_token") {
+        return titleCoverage >= 0.5 ? "strong" : "weak";
+    }
+    if (strategy === "fts" && titleCoverage >= 0.5) {
+        return "strong";
+    }
     if (strategy === "fts" && termCoverage >= 0.6 && signals.matchedFields.some((field) => strongExactFields.has(field))) {
         return "strong";
     }
@@ -1282,6 +1290,12 @@ export class RecallXRepository {
             tags: parseJson(row.tags_json, [])
         }));
     }
+    listAllNodes() {
+        const rows = this.db
+            .prepare(`SELECT * FROM nodes ORDER BY updated_at DESC, id DESC`)
+            .all();
+        return rows.map(mapNode);
+    }
     listActiveNodesByType(type, limit = 20) {
         const rows = this.db
             .prepare(`SELECT *
@@ -1586,37 +1600,60 @@ export class RecallXRepository {
         const queryPresent = Boolean(input.query.trim());
         const searchSort = input.sort === "smart" ? (queryPresent ? "relevance" : "updated_at") : input.sort;
         const normalizedQuery = input.query.trim();
+        const runSearchStage = async (operation, details, callback) => {
+            if (!options.runSearchStageSpan) {
+                return await callback();
+            }
+            return await options.runSearchStageSpan(operation, details, callback);
+        };
         const nodeResults = includeNodes
-            ? this.searchNodes({
+            ? await runSearchStage("workspace.search.nodes.deterministic", {
+                queryPresent,
+                limit: requestedWindow,
+                sort: searchSort
+            }, () => this.searchNodes({
                 query: input.query,
                 filters: input.nodeFilters ?? {},
                 limit: requestedWindow,
                 offset: 0,
                 sort: searchSort
-            })
+            }))
             : { items: [], total: 0 };
         const activityResults = includeActivities
-            ? this.searchActivities({
+            ? await runSearchStage("workspace.search.activities.deterministic", {
+                queryPresent,
+                limit: requestedWindow,
+                sort: searchSort
+            }, () => this.searchActivities({
                 query: input.query,
                 filters: input.activityFilters ?? {},
                 limit: requestedWindow,
                 offset: 0,
                 sort: searchSort
-            })
+            }))
             : { items: [], total: 0 };
         const fallbackTriggered = queryPresent && nodeResults.total + activityResults.total === 0;
         const fallbackTokens = fallbackTriggered ? tokenizeSearchQuery(input.query, SEARCH_FALLBACK_TOKEN_LIMIT) : [];
         const resolvedNodeResults = fallbackTokens.length >= 2 && includeNodes
-            ? this.searchWorkspaceNodeFallback(fallbackTokens, input.nodeFilters ?? {}, requestedWindow)
+            ? await runSearchStage("workspace.search.nodes.fallback_token", {
+                fallbackTokenCount: fallbackTokens.length,
+                limit: requestedWindow,
+                queryPresent
+            }, () => this.searchWorkspaceNodeFallback(fallbackTokens, input.nodeFilters ?? {}, requestedWindow))
             : nodeResults;
         const resolvedActivityResults = fallbackTokens.length >= 2 && includeActivities
-            ? this.searchWorkspaceActivityFallback(fallbackTokens, input.activityFilters ?? {}, requestedWindow)
+            ? await runSearchStage("workspace.search.activities.fallback_token", {
+                fallbackTokenCount: fallbackTokens.length,
+                limit: requestedWindow,
+                queryPresent
+            }, () => this.searchWorkspaceActivityFallback(fallbackTokens, input.activityFilters ?? {}, requestedWindow))
             : activityResults;
         const bestNodeLexicalQuality = summarizeLexicalQuality(resolvedNodeResults.items);
         const bestActivityLexicalQuality = summarizeLexicalQuality(resolvedActivityResults.items);
         const merged = this.mergeWorkspaceSearchResults(resolvedNodeResults.items, resolvedActivityResults.items, input.sort);
+        const fallbackTokenUsed = fallbackTokens.length >= 2;
         const deterministicResult = {
-            total: fallbackTokens.length >= 2
+            total: fallbackTokenUsed
                 ? merged.length
                 : resolvedNodeResults.total + resolvedActivityResults.total,
             items: merged.slice(input.offset, input.offset + input.limit)
@@ -1639,15 +1676,23 @@ export class RecallXRepository {
             const activityItems = result.items.flatMap((item) => item.resultType === "activity" && item.activity ? [item.activity] : []);
             appendCurrentTelemetryDetails({
                 searchHit: result.items.length > 0,
+                requestedWindow,
                 candidateCount: requestedWindow,
+                rawNodeTotalCount: nodeResults.total,
+                rawActivityTotalCount: activityResults.total,
+                resolvedNodeTotalCount: resolvedNodeResults.total,
+                resolvedActivityTotalCount: resolvedActivityResults.total,
                 nodeCandidateCount: resolvedNodeResults.items.length,
                 activityCandidateCount: resolvedActivityResults.items.length,
+                mergedCandidateCount: merged.length,
                 nodeResultCount: nodeItems.length,
                 activityResultCount: activityItems.length,
                 bestNodeLexicalQuality,
                 bestActivityLexicalQuality,
                 lexicalNodeHit: bestNodeLexicalQuality !== "none",
                 strongNodeLexicalHit: bestNodeLexicalQuality === "strong",
+                queryFallbackTriggered: fallbackTriggered,
+                queryFallbackUsed: fallbackTokenUsed,
                 resultComposition: computeWorkspaceResultComposition({
                     nodeCount: nodeItems.length,
                     activityCount: activityItems.length,
@@ -1655,7 +1700,8 @@ export class RecallXRepository {
                 }),
                 resultCount: result.items.length,
                 totalCount: result.total,
-                fallbackTokenCount: fallbackTokens.length,
+                queryFallbackTermCount: fallbackTokens.length,
+                totalCountStrategy: fallbackTokenUsed ? "merged_length" : "source_total_sum",
                 semanticFallbackEligible: telemetry.semanticFallbackEligible,
                 semanticFallbackAttempted: telemetry.semanticFallbackAttempted,
                 semanticFallbackUsed: telemetry.semanticFallbackUsed,
@@ -2039,6 +2085,7 @@ export class RecallXRepository {
            a.body,
            a.source_label,
            a.created_at,
+           a.metadata_json,
            n.title AS target_title,
            n.type AS target_type,
            n.status AS target_status
@@ -2066,6 +2113,7 @@ export class RecallXRepository {
                 body: row.body ? String(row.body) : null,
                 sourceLabel: row.source_label ? String(row.source_label) : null,
                 createdAt: String(row.created_at),
+                metadata: parseJson(row.metadata_json, {}),
                 lexicalQuality,
                 matchReason: buildSearchMatchReason(params.strategy, signals.matchedFields, {
                     strength: lexicalQuality === "none" ? undefined : lexicalQuality,
@@ -2312,25 +2360,59 @@ export class RecallXRepository {
             : "";
         const rows = this.db
             .prepare(`SELECT
-           r.*,
-           CASE WHEN r.from_node_id = ? THEN r.to_node_id ELSE r.from_node_id END AS related_id
+           r.id,
+           r.from_node_id,
+           r.to_node_id,
+           r.relation_type,
+           r.status,
+           r.created_by,
+           r.source_type,
+           r.source_label,
+           r.created_at,
+           r.metadata_json,
+           n.id AS node_id,
+           n.type AS node_type,
+           n.status AS node_status,
+           n.canonicality AS node_canonicality,
+           n.visibility AS node_visibility,
+           n.title AS node_title,
+           n.body AS node_body,
+           n.summary AS node_summary,
+           n.created_by AS node_created_by,
+           n.source_type AS node_source_type,
+           n.source_label AS node_source_label,
+           n.created_at AS node_created_at,
+           n.updated_at AS node_updated_at,
+           n.tags_json AS node_tags_json,
+           n.metadata_json AS node_metadata_json
          FROM relations r
+         JOIN nodes n
+           ON n.id = CASE WHEN r.from_node_id = ? THEN r.to_node_id ELSE r.from_node_id END
          WHERE (r.from_node_id = ? OR r.to_node_id = ?)
            AND r.status != 'archived'
            ${relationWhere}
          ORDER BY r.created_at DESC`)
             .all(nodeId, nodeId, nodeId, ...(relationFilter ?? []));
-        const relatedNodes = this.getNodesByIds(rows.map((row) => String(row.related_id)));
-        return rows.flatMap((row) => {
-            const node = relatedNodes.get(String(row.related_id));
-            if (!node) {
-                return [];
+        return rows.map((row) => ({
+            relation: mapRelation(row),
+            node: {
+                id: String(row.node_id),
+                type: row.node_type,
+                status: row.node_status,
+                canonicality: row.node_canonicality,
+                visibility: String(row.node_visibility),
+                title: row.node_title ? String(row.node_title) : null,
+                body: row.node_body ? String(row.node_body) : null,
+                summary: row.node_summary ? String(row.node_summary) : null,
+                createdBy: row.node_created_by ? String(row.node_created_by) : null,
+                sourceType: row.node_source_type ? String(row.node_source_type) : null,
+                sourceLabel: row.node_source_label ? String(row.node_source_label) : null,
+                createdAt: String(row.node_created_at),
+                updatedAt: String(row.node_updated_at),
+                tags: parseJson(row.node_tags_json, []),
+                metadata: parseJson(row.node_metadata_json, {})
             }
-            return [{
-                    relation: mapRelation(row),
-                    node
-                }];
-        });
+        }));
     }
     listProjectMemberNodes(projectId, limit) {
         const rows = this.db
@@ -2375,6 +2457,12 @@ export class RecallXRepository {
             .all(...uniqueIds, ...uniqueIds);
         return rows.map(mapRelation);
     }
+    listAllRelations() {
+        const rows = this.db
+            .prepare(`SELECT * FROM relations ORDER BY created_at ASC, id ASC`)
+            .all();
+        return rows.map(mapRelation);
+    }
     createRelation(input) {
         const now = nowIso();
         const id = createId("rel");
@@ -2678,6 +2766,60 @@ export class RecallXRepository {
             .all(entityType, entityId, limit);
         return rows.map(mapGovernanceEvent);
     }
+    listRecentGovernanceEvents(options) {
+        const limit = options?.limit ?? 12;
+        const entityTypes = options?.entityTypes?.length ? options.entityTypes : undefined;
+        const actions = options?.actions?.length ? options.actions : undefined;
+        const where = [`json_extract(ge.metadata_json, '$.manualAction') IS NOT NULL`];
+        const params = [];
+        if (entityTypes?.length) {
+            where.push(`ge.entity_type IN (${entityTypes.map(() => "?").join(", ")})`);
+            params.push(...entityTypes);
+        }
+        if (actions?.length) {
+            where.push(`json_extract(ge.metadata_json, '$.manualAction') IN (${actions.map(() => "?").join(", ")})`);
+            params.push(...actions);
+        }
+        const rows = this.db
+            .prepare(`SELECT
+           ge.*,
+           json_extract(ge.metadata_json, '$.manualAction') AS manual_action,
+           CASE
+             WHEN ge.entity_type = 'node' THEN n.title
+             ELSE COALESCE(fn.title, r.from_node_id) || ' ' || r.relation_type || ' ' || COALESCE(tn.title, r.to_node_id)
+           END AS display_title,
+           CASE
+             WHEN ge.entity_type = 'node' THEN n.type
+             ELSE r.status
+           END AS display_subtitle,
+           CASE WHEN ge.entity_type = 'node' THEN ge.entity_id ELSE NULL END AS node_id,
+           CASE WHEN ge.entity_type = 'relation' THEN r.from_node_id ELSE NULL END AS from_node_id,
+           CASE WHEN ge.entity_type = 'relation' THEN r.to_node_id ELSE NULL END AS to_node_id,
+           CASE WHEN ge.entity_type = 'relation' THEN r.relation_type ELSE NULL END AS relation_type
+         FROM governance_events ge
+         LEFT JOIN nodes n
+           ON ge.entity_type = 'node'
+          AND ge.entity_id = n.id
+         LEFT JOIN relations r
+           ON ge.entity_type = 'relation'
+          AND ge.entity_id = r.id
+         LEFT JOIN nodes fn ON fn.id = r.from_node_id
+         LEFT JOIN nodes tn ON tn.id = r.to_node_id
+         WHERE ${where.join(" AND ")}
+         ORDER BY ge.created_at DESC
+         LIMIT ?`)
+            .all(...params, limit);
+        return rows.map((row) => ({
+            ...mapGovernanceEvent(row),
+            action: row.manual_action ? String(row.manual_action) : null,
+            title: row.display_title ? String(row.display_title) : null,
+            subtitle: row.display_subtitle ? String(row.display_subtitle) : null,
+            nodeId: row.node_id ? String(row.node_id) : null,
+            fromNodeId: row.from_node_id ? String(row.from_node_id) : null,
+            toNodeId: row.to_node_id ? String(row.to_node_id) : null,
+            relationType: row.relation_type ? String(row.relation_type) : null
+        }));
+    }
     upsertGovernanceState(params) {
         const now = params.lastEvaluatedAt ?? nowIso();
         const existing = params.previousState === undefined
@@ -2986,6 +3128,12 @@ export class RecallXRepository {
             .all(...uniqueIds, limit);
         return rows.map(mapActivity);
     }
+    listAllActivities() {
+        const rows = this.db
+            .prepare(`SELECT * FROM activities ORDER BY created_at ASC, id ASC`)
+            .all();
+        return rows.map(mapActivity);
+    }
     appendActivity(input) {
         const id = createId("act");
         const now = nowIso();
@@ -3068,6 +3216,12 @@ export class RecallXRepository {
             .all(nodeId);
         return rows.map(mapArtifact);
     }
+    listAllArtifacts() {
+        const rows = this.db
+            .prepare(`SELECT * FROM artifacts ORDER BY created_at DESC, id DESC`)
+            .all();
+        return rows.map(mapArtifact);
+    }
     getArtifact(id) {
         const row = this.db.prepare(`SELECT * FROM artifacts WHERE id = ?`).get(id);
         return mapArtifact(assertPresent(row, `Artifact ${id} not found`));

package/app/server/sqlite-errors.js

@@ -0,0 +1,10 @@
+export function isReadonlySqliteWriteError(error) {
+    if (!error || typeof error !== "object") {
+        return false;
+    }
+    const candidate = error;
+    const code = typeof candidate.code === "string" ? candidate.code : "";
+    const errstr = typeof candidate.errstr === "string" ? candidate.errstr.toLowerCase() : "";
+    const message = typeof candidate.message === "string" ? candidate.message.toLowerCase() : "";
+    return code === "ERR_SQLITE_ERROR" && (errstr.includes("readonly database") || message.includes("readonly database"));
+}

package/app/server/workspace-import-helpers.js

@@ -0,0 +1,161 @@
+export const DEFAULT_IMPORT_OPTIONS = {
+    normalizeTitleWhitespace: true,
+    trimBodyWhitespace: false,
+    duplicateMode: "warn",
+};
+export function resolveImportOptions(options) {
+    return {
+        normalizeTitleWhitespace: typeof options?.normalizeTitleWhitespace === "boolean"
+            ? options.normalizeTitleWhitespace
+            : DEFAULT_IMPORT_OPTIONS.normalizeTitleWhitespace,
+        trimBodyWhitespace: typeof options?.trimBodyWhitespace === "boolean"
+            ? options.trimBodyWhitespace
+            : DEFAULT_IMPORT_OPTIONS.trimBodyWhitespace,
+        duplicateMode: options?.duplicateMode === "skip_exact" || options?.duplicateMode === "warn"
+            ? options.duplicateMode
+            : DEFAULT_IMPORT_OPTIONS.duplicateMode,
+    };
+}
+export function normalizeTitle(value, options) {
+    const trimmed = value.trim();
+    if (!options.normalizeTitleWhitespace) {
+        return trimmed || "Imported node";
+    }
+    return trimmed.replace(/\s+/g, " ").trim() || "Imported node";
+}
+export function normalizeBody(value, options) {
+    const unix = value.replace(/\r\n/g, "\n");
+    if (!options.trimBodyWhitespace) {
+        return unix;
+    }
+    return unix.replace(/[ \t]+$/gm, "").replace(/\s+$/u, "");
+}
+export function buildTitleKey(title, options) {
+    return normalizeTitle(title, options).toLowerCase();
+}
+export function buildExactKey(type, title, body, options) {
+    return `${type}::${buildTitleKey(title, options)}::${normalizeBody(body, options)}`;
+}
+export function buildDuplicateIndex(existingNodes, options) {
+    const exact = new Map();
+    const title = new Map();
+    for (const node of existingNodes) {
+        const nodeType = node.type;
+        const nodeTitle = node.title ?? node.id;
+        const nodeBody = node.body ?? "";
+        const exactKey = buildExactKey(nodeType, nodeTitle, nodeBody, options);
+        const titleKey = buildTitleKey(nodeTitle, options);
+        exact.set(exactKey, [...(exact.get(exactKey) ?? []), node]);
+        title.set(titleKey, [...(title.get(titleKey) ?? []), node]);
+    }
+    return { exact, title };
+}
+export function detectDuplicateMatch(params) {
+    const exactKey = buildExactKey(params.node.type, params.node.title, params.node.body, params.options);
+    const titleKey = buildTitleKey(params.node.title, params.options);
+    const workspaceExact = params.existing.exact.get(exactKey)?.[0];
+    if (workspaceExact) {
+        return {
+            title: params.node.title,
+            sourcePath: params.node.sourcePath,
+            matchType: "exact",
+            existingNodeId: workspaceExact.id,
+            existingNodeTitle: workspaceExact.title,
+            existingSource: "workspace",
+        };
+    }
+    const batchExact = params.seen.exact.get(exactKey);
+    if (batchExact) {
+        return {
+            title: params.node.title,
+            sourcePath: params.node.sourcePath,
+            matchType: "exact",
+            existingNodeId: null,
+            existingNodeTitle: batchExact.title,
+            existingSource: "batch",
+        };
+    }
+    const workspaceTitle = params.existing.title.get(titleKey)?.[0];
+    if (workspaceTitle) {
+        return {
+            title: params.node.title,
+            sourcePath: params.node.sourcePath,
+            matchType: "title",
+            existingNodeId: workspaceTitle.id,
+            existingNodeTitle: workspaceTitle.title,
+            existingSource: "workspace",
+        };
+    }
+    const batchTitle = params.seen.title.get(titleKey);
+    if (batchTitle) {
+        return {
+            title: params.node.title,
+            sourcePath: params.node.sourcePath,
+            matchType: "title",
+            existingNodeId: null,
+            existingNodeTitle: batchTitle.title,
+            existingSource: "batch",
+        };
+    }
+    return null;
+}
+export function rememberSeenNode(node, options, seen) {
+    const exactKey = buildExactKey(node.type, node.title, node.body, options);
+    const titleKey = buildTitleKey(node.title, options);
+    if (!seen.exact.has(exactKey)) {
+        seen.exact.set(exactKey, node);
+    }
+    if (!seen.title.has(titleKey)) {
+        seen.title.set(titleKey, node);
+    }
+}
+export function buildPreviewFromPlan(plan) {
+    const duplicateItems = plan.nodes
+        .filter((node) => node.duplicate !== null)
+        .map((node) => node.duplicate);
+    const exactDuplicateCandidates = duplicateItems.filter((item) => item.matchType === "exact").length;
+    const skippedOriginalIds = new Set(plan.options.duplicateMode === "skip_exact"
+        ? plan.nodes.filter((node) => node.duplicate?.matchType === "exact" && node.originalId).map((node) => node.originalId)
+        : []);
+    const skippedNodes = plan.options.duplicateMode === "skip_exact"
+        ? plan.nodes.filter((node) => node.duplicate?.matchType === "exact").length
+        : 0;
+    const skippedRelations = plan.options.duplicateMode === "skip_exact"
+        ? plan.relations.filter((relation) => (relation.fromOriginalId && skippedOriginalIds.has(relation.fromOriginalId)) ||
+            (relation.toOriginalId && skippedOriginalIds.has(relation.toOriginalId))).length
+        : 0;
+    const skippedActivities = plan.options.duplicateMode === "skip_exact"
+        ? plan.activities.filter((activity) => activity.targetOriginalId && skippedOriginalIds.has(activity.targetOriginalId)).length
+        : 0;
+    return {
+        format: plan.format,
+        label: plan.label,
+        sourcePath: plan.sourcePath,
+        createdAt: plan.createdAt,
+        options: plan.options,
+        nodesDetected: plan.nodes.length,
+        relationsDetected: plan.relations.length,
+        activitiesDetected: plan.activities.length,
+        duplicateCandidates: duplicateItems.length,
+        exactDuplicateCandidates,
+        nodesReady: plan.nodes.length - skippedNodes,
+        relationsReady: plan.relations.length - skippedRelations,
+        activitiesReady: plan.activities.length - skippedActivities,
+        skippedNodes,
+        skippedRelations,
+        skippedActivities,
+        warnings: [
+            ...plan.warnings,
+            ...(duplicateItems.length
+                ? [`Detected ${duplicateItems.length} likely duplicate node(s) in the current import preview.`]
+                : []),
+        ],
+        sampleItems: plan.nodes.slice(0, 5).map((node) => ({
+            title: node.title,
+            type: node.type,
+            sourcePath: node.sourcePath,
+            duplicateKind: node.duplicate?.matchType ?? null,
+        })),
+        duplicateItems,
+    };
+}