roam-research-mcp 0.36.0 → 1.3.2

package/build/tools/operations/outline.js

@@ -3,6 +3,9 @@ import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js';
 import { formatRoamDate } from '../../utils/helpers.js';
 import { capitalizeWords } from '../helpers/text.js';
 import { parseMarkdown, convertToRoamActions, convertToRoamMarkdown } from '../../markdown-utils.js';
+import { pageUidCache } from '../../cache/page-uid-cache.js';
+// Threshold for skipping child fetch during verification
+const VERIFICATION_THRESHOLD = 5;
 export class OutlineOperations {
     constructor(graph) {
         this.graph = graph;
@@ -14,49 +17,31 @@ export class OutlineOperations {
         };
     }
     /**
-     * Helper function to find block with improved relationship checks
+     * Helper function to find block with reduced retries for rate limit efficiency.
+     * Uses only the most reliable query strategy with 2 retries max.
      */
-    async findBlockWithRetry(pageUid, blockString, maxRetries = 5, initialDelay = 1000) {
-        // Try multiple query strategies
-        const queries = [
-            // Strategy 1: Direct page and string match
-            `[:find ?b-uid ?order
-          :where [?p :block/uid "${pageUid}"]
-                 [?b :block/page ?p]
-                 [?b :block/string "${blockString}"]
-                 [?b :block/order ?order]
-                 [?b :block/uid ?b-uid]]`,
-            // Strategy 2: Parent-child relationship
-            `[:find ?b-uid ?order
-          :where [?p :block/uid "${pageUid}"]
-                 [?b :block/parents ?p]
-                 [?b :block/string "${blockString}"]
-                 [?b :block/order ?order]
-                 [?b :block/uid ?b-uid]]`,
-            // Strategy 3: Broader page relationship
-            `[:find ?b-uid ?order
-          :where [?p :block/uid "${pageUid}"]
-                 [?b :block/page ?page]
-                 [?p :block/page ?page]
-                 [?b :block/string "${blockString}"]
-                 [?b :block/order ?order]
-                 [?b :block/uid ?b-uid]]`
-        ];
+    async findBlockWithRetry(pageUid, blockString, maxRetries = 2, initialDelay = 1000) {
+        // Use only the most reliable query strategy (direct page and string match)
+        const query = `[:find ?b-uid ?order
+        :where [?p :block/uid "${pageUid}"]
+               [?b :block/page ?p]
+               [?b :block/string "${blockString}"]
+               [?b :block/order ?order]
+               [?b :block/uid ?b-uid]]`;
         for (let retry = 0; retry < maxRetries; retry++) {
-            // Try each query strategy
-            for (const queryStr of queries) {
-                const blockResults = await q(this.graph, queryStr, []);
-                if (blockResults && blockResults.length > 0) {
-                    // Use the most recently created block
-                    const sorted = blockResults.sort((a, b) => b[1] - a[1]);
-                    return sorted[0][0];
-                }
+            const blockResults = await q(this.graph, query, []);
+            if (blockResults && blockResults.length > 0) {
+                // Use the most recently created block (highest order)
+                const sorted = blockResults.sort((a, b) => b[1] - a[1]);
+                return sorted[0][0];
+            }
+            // Exponential backoff between retries
+            if (retry < maxRetries - 1) {
+                const delay = initialDelay * Math.pow(2, retry);
+                await new Promise(resolve => setTimeout(resolve, delay));
             }
-            // Exponential backoff
-            const delay = initialDelay * Math.pow(2, retry);
-            await new Promise(resolve => setTimeout(resolve, delay));
         }
-        throw new McpError(ErrorCode.InternalError, `Failed to find block "${blockString}" under page "${pageUid}" after trying multiple strategies`);
+        throw new McpError(ErrorCode.InternalError, `Failed to find block "${blockString}" under page "${pageUid}" after ${maxRetries} attempts`);
     }
     ;
     /**
@@ -224,21 +209,30 @@ export class OutlineOperations {
         if (invalidItems.length > 0) {
             throw new McpError(ErrorCode.InvalidRequest, 'outline contains invalid items - each item must have a level (1-10) and non-empty text');
         }
-        // Helper function to find or create page with retries
+        // Helper function to find or create page with retries and caching
         const findOrCreatePage = async (titleOrUid, maxRetries = 3, delayMs = 500) => {
-            // First try to find by title
-            const titleQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
             const variations = [
                 titleOrUid, // Original
                 capitalizeWords(titleOrUid), // Each word capitalized
                 titleOrUid.toLowerCase() // All lowercase
             ];
+            // Check cache first for any variation
+            for (const variation of variations) {
+                const cachedUid = pageUidCache.get(variation);
+                if (cachedUid) {
+                    return cachedUid;
+                }
+            }
+            const titleQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
             for (let retry = 0; retry < maxRetries; retry++) {
                 // Try each case variation
                 for (const variation of variations) {
                     const findResults = await q(this.graph, titleQuery, [variation]);
                     if (findResults && findResults.length > 0) {
-                        return findResults[0][0];
+                        const uid = findResults[0][0];
+                        // Cache the result
+                        pageUidCache.set(titleOrUid, uid);
+                        return uid;
                     }
                 }
                 // If not found as title, try as UID
@@ -264,6 +258,15 @@ export class OutlineOperations {
                     await new Promise(resolve => setTimeout(resolve, delayMs));
                 }
             }
+            // One more attempt to find and cache after creation attempts
+            for (const variation of variations) {
+                const findResults = await q(this.graph, titleQuery, [variation]);
+                if (findResults && findResults.length > 0) {
+                    const uid = findResults[0][0];
+                    pageUidCache.onPageCreated(titleOrUid, uid);
+                    return uid;
+                }
+            }
             throw new McpError(ErrorCode.InvalidRequest, `Failed to find or create page "${titleOrUid}" after multiple attempts`);
         };
         // Get or create the target page
@@ -359,18 +362,27 @@ export class OutlineOperations {
                 throw error;
             throw new McpError(ErrorCode.InternalError, `Failed to create outline: ${error.message}`);
         }
-        // Post-creation verification to get actual UIDs for top-level blocks and their children
+        // Post-creation verification to get actual UIDs for top-level blocks
         const createdBlocks = [];
         // Only query for top-level blocks (level 1) based on the original outline input
         const topLevelOutlineItems = validOutline.filter(item => item.level === 1);
+        // Skip recursive child fetching for large batches to reduce API calls
+        const skipChildFetch = topLevelOutlineItems.length > VERIFICATION_THRESHOLD;
         for (const item of topLevelOutlineItems) {
             try {
                 // Assert item.text is a string as it's filtered earlier to be non-undefined and non-empty
                 const foundUid = await this.findBlockWithRetry(targetParentUid, item.text);
                 if (foundUid) {
-                    const nestedBlock = await this.fetchBlockWithChildren(foundUid);
-                    if (nestedBlock) {
-                        createdBlocks.push(nestedBlock);
+                    if (skipChildFetch) {
+                        // Large batch: just return parent UID, skip recursive child queries
+                        createdBlocks.push({ uid: foundUid, text: item.text, level: 1, order: 0 });
+                    }
+                    else {
+                        // Small batch: full verification with children (current behavior)
+                        const nestedBlock = await this.fetchBlockWithChildren(foundUid);
+                        if (nestedBlock) {
+                            createdBlocks.push(nestedBlock);
+                        }
                     }
                 }
             }
@@ -391,39 +403,56 @@ export class OutlineOperations {
         // First get the page UID
         let targetPageUid = page_uid;
         if (!targetPageUid && page_title) {
-            const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
-            const findResults = await q(this.graph, findQuery, [page_title]);
-            if (findResults && findResults.length > 0) {
-                targetPageUid = findResults[0][0];
+            // Check cache first
+            const cachedUid = pageUidCache.get(page_title);
+            if (cachedUid) {
+                targetPageUid = cachedUid;
             }
             else {
-                throw new McpError(ErrorCode.InvalidRequest, `Page with title "${page_title}" not found`);
+                const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
+                const findResults = await q(this.graph, findQuery, [page_title]);
+                if (findResults && findResults.length > 0) {
+                    targetPageUid = findResults[0][0];
+                    pageUidCache.set(page_title, targetPageUid);
+                }
+                else {
+                    throw new McpError(ErrorCode.InvalidRequest, `Page with title "${page_title}" not found`);
+                }
             }
         }
         // If no page specified, use today's date page
         if (!targetPageUid) {
             const today = new Date();
             const dateStr = formatRoamDate(today);
-            const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
-            const findResults = await q(this.graph, findQuery, [dateStr]);
-            if (findResults && findResults.length > 0) {
-                targetPageUid = findResults[0][0];
+            // Check cache for today's page
+            const cachedDailyUid = pageUidCache.get(dateStr);
+            if (cachedDailyUid) {
+                targetPageUid = cachedDailyUid;
             }
             else {
-                // Create today's page
-                try {
-                    await createPage(this.graph, {
-                        action: 'create-page',
-                        page: { title: dateStr }
-                    });
-                    const results = await q(this.graph, findQuery, [dateStr]);
-                    if (!results || results.length === 0) {
-                        throw new McpError(ErrorCode.InternalError, 'Could not find created today\'s page');
-                    }
-                    targetPageUid = results[0][0];
+                const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
+                const findResults = await q(this.graph, findQuery, [dateStr]);
+                if (findResults && findResults.length > 0) {
+                    targetPageUid = findResults[0][0];
+                    pageUidCache.set(dateStr, targetPageUid);
                 }
-                catch (error) {
-                    throw new McpError(ErrorCode.InternalError, `Failed to create today's page: ${error instanceof Error ? error.message : String(error)}`);
+                else {
+                    // Create today's page
+                    try {
+                        await createPage(this.graph, {
+                            action: 'create-page',
+                            page: { title: dateStr }
+                        });
+                        const results = await q(this.graph, findQuery, [dateStr]);
+                        if (!results || results.length === 0) {
+                            throw new McpError(ErrorCode.InternalError, 'Could not find created today\'s page');
+                        }
+                        targetPageUid = results[0][0];
+                        pageUidCache.onPageCreated(dateStr, targetPageUid);
+                    }
+                    catch (error) {
+                        throw new McpError(ErrorCode.InternalError, `Failed to create today's page: ${error instanceof Error ? error.message : String(error)}`);
+                    }
                 }
             }
         }
@@ -469,7 +498,18 @@ export class OutlineOperations {
             if (!result) {
                 throw new McpError(ErrorCode.InternalError, 'Failed to import nested markdown content');
             }
-            // After successful batch action, get all nested UIDs under the parent
+            // Skip nested structure fetch for large imports to reduce API calls
+            const skipNestedFetch = actions.length > VERIFICATION_THRESHOLD;
+            if (skipNestedFetch) {
+                // Large import: return success with block count, skip recursive queries
+                return {
+                    success: true,
+                    page_uid: targetPageUid,
+                    parent_uid: targetParentUid,
+                    created_uids: []
+                };
+            }
+            // Small import: get all nested UIDs under the parent (current behavior)
             const createdUids = await this.fetchNestedStructure(targetParentUid);
             return {
                 success: true,

package/build/tools/operations/pages.js

@@ -3,6 +3,9 @@ import { McpError, ErrorCode } from '@modelcontextprotocol/sdk/types.js';
 import { capitalizeWords } from '../helpers/text.js';
 import { resolveRefs } from '../helpers/refs.js';
 import { convertToRoamActions, convertToRoamMarkdown } from '../../markdown-utils.js';
+import { pageUidCache } from '../../cache/page-uid-cache.js';
+import { buildTableActions } from './table.js';
+import { BatchOperations } from './batch.js';
 // Helper to get ordinal suffix for dates
 function getOrdinalSuffix(day) {
     if (day > 3 && day < 21)
@@ -17,8 +20,9 @@ function getOrdinalSuffix(day) {
 export class PageOperations {
     constructor(graph) {
         this.graph = graph;
+        this.batchOps = new BatchOperations(graph);
     }
-    async findPagesModifiedToday(max_num_pages = 50) {
+    async findPagesModifiedToday(limit = 50, offset = 0, sort_order = 'desc') {
         // Define ancestor rule for traversing block hierarchy
         const ancestorRule = `[
       [ (ancestor ?b ?a)
@@ -31,15 +35,21 @@ export class PageOperations {
         const startOfDay = new Date();
         startOfDay.setHours(0, 0, 0, 0);
         try {
-            // Query for pages modified today
-            const results = await q(this.graph, `[:find ?title
+            // Query for pages modified today, including modification time for sorting
+            let query = `[:find ?title ?time
           :in $ ?start_of_day %
           :where
           [?page :node/title ?title]
           (ancestor ?block ?page)
           [?block :edit/time ?time]
-          [(> ?time ?start_of_day)]]
-          :limit ${max_num_pages}`, [startOfDay.getTime(), ancestorRule]);
+          [(> ?time ?start_of_day)]]`;
+            if (limit !== -1) {
+                query += ` :limit ${limit}`;
+            }
+            if (offset > 0) {
+                query += ` :offset ${offset}`;
+            }
+            const results = await q(this.graph, query, [startOfDay.getTime(), ancestorRule]);
             if (!results || results.length === 0) {
                 return {
                     success: true,
@@ -47,7 +57,16 @@ export class PageOperations {
                     message: 'No pages have been modified today'
                 };
             }
-            // Extract unique page titles
+            // Sort results by modification time
+            results.sort((a, b) => {
+                if (sort_order === 'desc') {
+                    return b[1] - a[1]; // Newest first
+                }
+                else {
+                    return a[1] - b[1]; // Oldest first
+                }
+            });
+            // Extract unique page titles from sorted results
             const uniquePages = Array.from(new Set(results.map(([title]) => title)));
             return {
                 success: true,
@@ -62,72 +81,146 @@ export class PageOperations {
     async createPage(title, content) {
         // Ensure title is properly formatted
         const pageTitle = String(title).trim();
-        // First try to find if the page exists
-        const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
-        const findResults = await q(this.graph, findQuery, [pageTitle]);
         let pageUid;
-        if (findResults && findResults.length > 0) {
-            // Page exists, use its UID
-            pageUid = findResults[0][0];
+        // Check cache first to avoid unnecessary query
+        const cachedUid = pageUidCache.get(pageTitle);
+        if (cachedUid) {
+            pageUid = cachedUid;
         }
         else {
-            // Create new page
-            try {
-                await createRoamPage(this.graph, {
-                    action: 'create-page',
-                    page: {
-                        title: pageTitle
+            // First try to find if the page exists
+            const findQuery = `[:find ?uid :in $ ?title :where [?e :node/title ?title] [?e :block/uid ?uid]]`;
+            const findResults = await q(this.graph, findQuery, [pageTitle]);
+            if (findResults && findResults.length > 0) {
+                // Page exists, use its UID and cache it
+                pageUid = findResults[0][0];
+                pageUidCache.set(pageTitle, pageUid);
+            }
+            else {
+                // Create new page
+                try {
+                    await createRoamPage(this.graph, {
+                        action: 'create-page',
+                        page: {
+                            title: pageTitle
+                        }
+                    });
+                    // Get the new page's UID
+                    const results = await q(this.graph, findQuery, [pageTitle]);
+                    if (!results || results.length === 0) {
+                        throw new Error('Could not find created page');
                     }
-                });
-                // Get the new page's UID
-                const results = await q(this.graph, findQuery, [pageTitle]);
-                if (!results || results.length === 0) {
-                    throw new Error('Could not find created page');
+                    pageUid = results[0][0];
+                    // Cache the newly created page
+                    pageUidCache.onPageCreated(pageTitle, pageUid);
+                }
+                catch (error) {
+                    throw new McpError(ErrorCode.InternalError, `Failed to create page: ${error instanceof Error ? error.message : String(error)}`);
                 }
-                pageUid = results[0][0];
-            }
-            catch (error) {
-                throw new McpError(ErrorCode.InternalError, `Failed to create page: ${error instanceof Error ? error.message : String(error)}`);
             }
         }
         // If content is provided, create blocks using batch operations
         if (content && content.length > 0) {
             try {
-                // Convert content array to MarkdownNode format expected by convertToRoamActions
-                const nodes = content.map(block => ({
-                    content: convertToRoamMarkdown(block.text.replace(/^#+\s*/, '')),
-                    level: block.level,
-                    ...(block.heading && { heading_level: block.heading }),
-                    children: []
-                }));
-                // Create hierarchical structure based on levels
-                const rootNodes = [];
-                const levelMap = {};
-                for (const node of nodes) {
-                    if (node.level === 1) {
-                        rootNodes.push(node);
-                        levelMap[1] = node;
+                // Idempotency check: If page already has content, skip adding more
+                // This prevents duplicate content when the tool is called twice
+                const existingBlocksQuery = `[:find (count ?b) .
+                                      :where [?p :block/uid "${pageUid}"]
+                                             [?p :block/children ?b]]`;
+                const existingBlockCountResult = await q(this.graph, existingBlocksQuery, []);
+                const existingBlockCount = typeof existingBlockCountResult === 'number' ? existingBlockCountResult : 0;
+                if (existingBlockCount && existingBlockCount > 0) {
+                    // Page already has content - this might be a duplicate call
+                    // Return success without adding duplicate content
+                    return { success: true, uid: pageUid };
+                }
+                // Separate text content from table content, maintaining order
+                const textItems = [];
+                const tableItems = [];
+                for (let i = 0; i < content.length; i++) {
+                    const item = content[i];
+                    if (item.type === 'table') {
+                        tableItems.push({ index: i, item: item });
                     }
                     else {
-                        const parentLevel = node.level - 1;
-                        const parent = levelMap[parentLevel];
-                        if (!parent) {
-                            throw new Error(`Invalid block hierarchy: level ${node.level} block has no parent`);
+                        // Default to text type
+                        textItems.push(item);
+                    }
+                }
+                // Process text blocks
+                const allActions = [];
+                if (textItems.length > 0) {
+                    // Filter out empty blocks (empty or whitespace-only text) to prevent creating visual linebreaks
+                    const nonEmptyContent = textItems.filter(block => block.text && block.text.trim().length > 0);
+                    if (nonEmptyContent.length > 0) {
+                        // Normalize levels to prevent gaps after filtering
+                        const normalizedContent = [];
+                        for (let i = 0; i < nonEmptyContent.length; i++) {
+                            const block = nonEmptyContent[i];
+                            if (i === 0) {
+                                normalizedContent.push({ ...block, level: 1 });
+                            }
+                            else {
+                                const prevLevel = normalizedContent[i - 1].level;
+                                const maxAllowedLevel = prevLevel + 1;
+                                normalizedContent.push({
+                                    ...block,
+                                    level: Math.min(block.level, maxAllowedLevel)
+                                });
+                            }
                         }
-                        parent.children.push(node);
-                        levelMap[node.level] = node;
+                        // Convert content array to MarkdownNode format expected by convertToRoamActions
+                        const nodes = normalizedContent.map(block => ({
+                            content: convertToRoamMarkdown(block.text.replace(/^#+\s*/, '')),
+                            level: block.level,
+                            ...(block.heading && { heading_level: block.heading }),
+                            children: []
+                        }));
+                        // Create hierarchical structure based on levels
+                        const rootNodes = [];
+                        const levelMap = {};
+                        for (const node of nodes) {
+                            if (node.level === 1) {
+                                rootNodes.push(node);
+                                levelMap[1] = node;
+                            }
+                            else {
+                                const parentLevel = node.level - 1;
+                                const parent = levelMap[parentLevel];
+                                if (!parent) {
+                                    throw new Error(`Invalid block hierarchy: level ${node.level} block has no parent`);
+                                }
+                                parent.children.push(node);
+                                levelMap[node.level] = node;
+                            }
+                        }
+                        // Generate batch actions for text blocks
+                        const textActions = convertToRoamActions(rootNodes, pageUid, 'last');
+                        allActions.push(...textActions);
                     }
                 }
-                // Generate batch actions for all blocks
-                const actions = convertToRoamActions(rootNodes, pageUid, 'last');
-                // Execute batch operation
-                if (actions.length > 0) {
+                // Execute text block actions first (no placeholders, use SDK directly)
+                if (allActions.length > 0) {
                     const batchResult = await batchActions(this.graph, {
                         action: 'batch-actions',
-                        actions
+                        actions: allActions
                     });
                     if (!batchResult) {
-                        throw new Error('Failed to create blocks');
+                        throw new Error('Failed to create text blocks');
+                    }
+                }
+                // Process table items separately (use BatchOperations to handle UID placeholders)
+                for (const { item } of tableItems) {
+                    const tableActions = buildTableActions({
+                        parent_uid: pageUid,
+                        headers: item.headers,
+                        rows: item.rows,
+                        order: 'last'
+                    });
+                    // Use BatchOperations.processBatch to handle {{uid:*}} placeholders
+                    const tableResult = await this.batchOps.processBatch(tableActions);
+                    if (!tableResult.success) {
+                        throw new Error(`Failed to create table: ${typeof tableResult.error === 'string' ? tableResult.error : tableResult.error?.message}`);
                     }
                 }
             }
@@ -142,11 +235,18 @@ export class PageOperations {
             const month = today.toLocaleString('en-US', { month: 'long' });
             const year = today.getFullYear();
             const formattedTodayTitle = `${month} ${day}${getOrdinalSuffix(day)}, ${year}`;
-            const dailyPageQuery = `[:find ?uid .
-                              :where [?e :node/title "${formattedTodayTitle}"]
-                                     [?e :block/uid ?uid]]`;
-            const dailyPageResult = await q(this.graph, dailyPageQuery, []);
-            const dailyPageUid = dailyPageResult ? String(dailyPageResult) : null;
+            // Check cache for daily page
+            let dailyPageUid = pageUidCache.get(formattedTodayTitle);
+            if (!dailyPageUid) {
+                const dailyPageQuery = `[:find ?uid .
+                                :where [?e :node/title "${formattedTodayTitle}"]
+                                       [?e :block/uid ?uid]]`;
+                const dailyPageResult = await q(this.graph, dailyPageQuery, []);
+                dailyPageUid = dailyPageResult ? String(dailyPageResult) : undefined;
+                if (dailyPageUid) {
+                    pageUidCache.set(formattedTodayTitle, dailyPageUid);
+                }
+            }
             if (dailyPageUid) {
                 await createBlock(this.graph, {
                     action: 'create-block',
@@ -178,15 +278,27 @@ export class PageOperations {
             capitalizeWords(title), // Each word capitalized
             title.toLowerCase() // All lowercase
         ];
+        // Check cache first for any variation
         let uid = null;
         for (const variation of variations) {
+            const cachedUid = pageUidCache.get(variation);
+            if (cachedUid) {
+                uid = cachedUid;
+                break;
+            }
+        }
+        // If not cached, query the database
+        if (!uid) {
+            const orClause = variations.map(v => `[?e :node/title "${v}"]`).join(' ');
             const searchQuery = `[:find ?uid .
-                          :where [?e :node/title "${variation}"]
-                                 [?e :block/uid ?uid]]`;
+                          :where [?e :block/uid ?uid]
+                                 (or ${orClause})]`;
             const result = await q(this.graph, searchQuery, []);
             uid = (result === null || result === undefined) ? null : String(result);
-            if (uid)
-                break;
+            // Cache the result for the original title
+            if (uid) {
+                pageUidCache.set(title, uid);
+            }
         }
         if (!uid) {
             throw new McpError(ErrorCode.InvalidRequest, `Page with title "${title}" not found (tried original, capitalized words, and lowercase)`);