unified-sf-docs-mcp 1.0.0

package/README.md

@@ -0,0 +1,82 @@
+# Unified Salesforce Documentation MCP Server
+
+A powerful Model Context Protocol (MCP) server that empowers LLMs to scrape, digest, and search through modern and legacy Salesforce documentation. It elegantly handles deeply nested Shadow DOMs, typical of Lightning Web Components (LWC), and legacy iframe-based documentation structures.
+
+## Features
+
+-   **Deep Shadow DOM Piercing:** Bypasses 400KB+ of SPA boilerplate on `help.salesforce.com` and `developer.salesforce.com` to extract only the pure article Markdown.
+-   **Hierarchical Spidering:** Automatically queues and scrapes all related pages linked from a central guide using `mass_extract_guide`.
+-   **Offline RAG Capabilities:** Chunks and indexes scraped Markdown into a local SQLite database (`docs.db`) allowing for instantaneous local search using `search_local_docs`.
+
+## Available Tools
+
+1.  **`scrape_single_page`**: Provide a Salesforce documentation URL. The server will use a headless browser (Puppeteer) to load the page, wait for dynamic content, pierce all shadow DOMs, and return clean Markdown.
+2.  **`mass_extract_guide`**: Provide a "Table of Contents" or central guide URL. The server will extract the parent page, find all hierarchical child links, scrape them concurrently, chunk their content, and save them to a local SQLite database for offline querying.
+3.  **`search_local_docs`**: Provide a natural language query (e.g., `LWC lifecycle hooks`). The server queries the SQLite database using fuzzy SQL search to instantly return the best matching pre-scraped chunks of documentation.
+
+## Quick Start Installation
+
+For anyone downloading this project for the first time:
+
+1.  **Clone the Repository:**
+    ```bash
+    git clone https://github.com/ttrevisan-ilmn/unified-sf-docs-mcp.git
+    cd unified-sf-docs-mcp
+    ```
+2.  **Install Dependencies:**
+    ```bash
+    npm install
+    ```
+3.  **Build the Project:**
+    ```bash
+    npm run build
+    ```
+    *(Note: The server runs from the compiled `/dist` directory, so building is required).*
+*(Note: To use the tools interactively, integrate this MCP server with an MCP client like Claude Desktop or Cursor.)*
+
+## Testing
+
+You can use the provided test scripts to verify the core functionality or the scraper against different Salesforce URL layouts:
+
+```bash
+# Test the database, chunking, and search functionality
+npx tsx tests/test-core.js
+
+# Test the robust Shadow DOM scraper against 4 different URL permutations
+npx tsx tests/test-all.js
+```
+
+## Integrating with AI Assistants
+
+MCP servers act as a bridge between an LLM and local tools. To actually use this server, you need to plug it into an AI coding assistant like **Cursor** or **Claude Desktop**. 
+
+### 1. Claude Desktop
+
+1. Open the Claude Desktop configuration file:
+   - macOS: `~/Library/Application Support/Claude/claude_desktop_config.json`
+   - Windows: `%APPDATA%\Claude\claude_desktop_config.json`
+2. Add the following entry to your `mcpServers` object, replacing `/PATH/TO` with the absolute path to where you cloned this repository:
+
+```json
+{
+  "mcpServers": {
+    "unified-sf-docs": {
+      "command": "node",
+      "args": [
+        "/PATH/TO/unified-sf-docs-mcp/dist/index.js"
+      ]
+    }
+  }
+}
+```
+3. Restart Claude Desktop. The tools will now be available when talking to Claude!
+
+### 2. Cursor
+
+1. Open Cursor Settings -> Features -> MCP
+2. Click **+ Add new MCP server**
+3. Configure the settings:
+   - **Type**: `command`
+   - **Name**: `unified-sf-docs`
+   - **Command**: `node /PATH/TO/unified-sf-docs-mcp/dist/index.js` (Be sure to use the absolute path)
+4. Click Save. Cursor will connect to the server and surface the 3 new tools to Cursor Agent.

package/dist/db.js

@@ -0,0 +1,173 @@
+// @ts-ignore
+import initSqlJs from "sql.js";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const DATA_DIR = join(__dirname, "..", "..", "db");
+const DB_PATH = join(DATA_DIR, "salesforce-docs.db");
+let db = null;
+let SQL = null;
+async function initSql() {
+    if (!SQL) {
+        SQL = await initSqlJs();
+    }
+    return SQL;
+}
+export async function getDatabase() {
+    if (!db) {
+        const sqljs = await initSql();
+        if (!existsSync(DATA_DIR))
+            mkdirSync(DATA_DIR, { recursive: true });
+        if (existsSync(DB_PATH)) {
+            const buffer = readFileSync(DB_PATH);
+            db = new sqljs.Database(buffer);
+        }
+        else {
+            db = new sqljs.Database();
+            initializeDatabaseSchema(db);
+        }
+    }
+    return db;
+}
+function initializeDatabaseSchema(dbInstance) {
+    dbInstance.run(`
+        CREATE TABLE IF NOT EXISTS documents (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            url TEXT NOT NULL UNIQUE,
+            title TEXT NOT NULL,
+            hash TEXT NOT NULL,
+            category TEXT,
+            last_scraped DATETIME DEFAULT CURRENT_TIMESTAMP
+        )
+    `);
+    dbInstance.run(`
+        CREATE TABLE IF NOT EXISTS chunks (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            document_id INTEGER NOT NULL,
+            chunk_index INTEGER NOT NULL,
+            content TEXT NOT NULL,
+            content_lower TEXT NOT NULL,
+            FOREIGN KEY (document_id) REFERENCES documents(id) ON DELETE CASCADE
+        )
+    `);
+    dbInstance.run(`CREATE INDEX IF NOT EXISTS idx_documents_url ON documents(url)`);
+    dbInstance.run(`CREATE INDEX IF NOT EXISTS idx_chunks_document_id ON chunks(document_id)`);
+    saveDatabase(dbInstance);
+}
+export function saveDatabase(dbInstance = db) {
+    if (dbInstance) {
+        const data = dbInstance.export();
+        const buffer = Buffer.from(data);
+        writeFileSync(DB_PATH, buffer);
+    }
+}
+export async function saveDocument(url, title, markdown, hash, category) {
+    const database = await getDatabase();
+    // Check if it already exists and hasn't changed
+    const checkStmt = database.prepare('SELECT id, hash FROM documents WHERE url = ?');
+    checkStmt.bind([url]);
+    let existingId = null;
+    let existingHash = null;
+    if (checkStmt.step()) {
+        const row = checkStmt.get();
+        existingId = row[0];
+        existingHash = row[1];
+    }
+    checkStmt.free();
+    if (existingId && existingHash === hash) {
+        // Unchanged, skip
+        return { action: 'skipped', id: existingId };
+    }
+    if (existingId) {
+        // Delete old chunks
+        database.run('DELETE FROM chunks WHERE document_id = ?', [existingId]);
+        database.run('UPDATE documents SET title = ?, hash = ?, last_scraped = CURRENT_TIMESTAMP WHERE id = ?', [title, hash, existingId]);
+    }
+    else {
+        database.run('INSERT INTO documents (url, title, hash, category) VALUES (?, ?, ?, ?)', [url, title, hash, category]);
+        const res = database.exec('SELECT last_insert_rowid()');
+        existingId = res[0].values[0][0];
+    }
+    // Split markdown into chunks (approx 1000 chars)
+    const chunks = splitIntoChunks(markdown, 1000);
+    for (let i = 0; i < chunks.length; i++) {
+        database.run('INSERT INTO chunks (document_id, chunk_index, content, content_lower) VALUES (?, ?, ?, ?)', [existingId, i, chunks[i], chunks[i].toLowerCase()]);
+    }
+    saveDatabase();
+    return { action: 'saved', id: existingId };
+}
+function splitIntoChunks(text, maxLen) {
+    const chunks = [];
+    const paragraphs = text.split('\\n\\n');
+    let currentChunk = '';
+    for (const p of paragraphs) {
+        if ((currentChunk.length + p.length) > maxLen && currentChunk.length > 0) {
+            chunks.push(currentChunk.trim());
+            currentChunk = '';
+        }
+        currentChunk += p + '\\n\\n';
+    }
+    if (currentChunk.trim().length > 0)
+        chunks.push(currentChunk.trim());
+    return chunks;
+}
+export async function searchDocuments(query, maxResults = 5) {
+    const database = await getDatabase();
+    const queryLower = query.toLowerCase();
+    const searchTerms = queryLower.split(/\\s+/).filter(w => w.length > 2);
+    if (searchTerms.length === 0)
+        return [];
+    const likeConditions = searchTerms.map(t => 'c.content_lower LIKE ?').join(' OR ');
+    const params = searchTerms.map(t => `%${t}%`);
+    const sql = `
+        SELECT 
+            d.id, d.url, d.title, d.category,
+            c.content, c.content_lower
+        FROM chunks c
+        JOIN documents d ON c.document_id = d.id
+        WHERE (${likeConditions})
+        LIMIT 200
+    `;
+    const stmt = database.prepare(sql);
+    stmt.bind(params);
+    const rows = [];
+    const columns = stmt.getColumnNames();
+    while (stmt.step()) {
+        const rowData = stmt.get();
+        const row = {};
+        columns.forEach((col, idx) => row[col] = rowData[idx]);
+        rows.push(row);
+    }
+    stmt.free();
+    // Score based on word hits
+    const scoredRows = rows.map(row => {
+        let hits = 0;
+        for (const term of searchTerms) {
+            if (row.content_lower.includes(term))
+                hits++;
+        }
+        const density = hits / searchTerms.length;
+        return { ...row, score: density, hits };
+    });
+    scoredRows.sort((a, b) => b.score - a.score);
+    // Deduplicate by URL
+    const seenUrls = new Set();
+    const finalResults = [];
+    for (const row of scoredRows) {
+        if (!seenUrls.has(row.url)) {
+            seenUrls.add(row.url);
+            finalResults.push({
+                url: row.url,
+                title: row.title,
+                category: row.category,
+                matchContent: row.content,
+                score: row.score
+            });
+            if (finalResults.length >= maxResults)
+                break;
+        }
+    }
+    return finalResults;
+}

package/dist/index.js

@@ -0,0 +1,149 @@
+#!/usr/bin/env node
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { CallToolRequestSchema, ListToolsRequestSchema, } from "@modelcontextprotocol/sdk/types.js";
+import { scrapePage, closeBrowser } from "./scraper.js";
+import { saveDocument, searchDocuments } from "./db.js";
+import { z } from "zod";
+const server = new Server({ name: "unified-sf-docs-mcp", version: "1.0.0" }, { capabilities: { tools: {} } });
+const ScrapePageSchema = z.object({
+    url: z.string().url(),
+    category: z.string().optional().default("general")
+});
+const MassExtractSchema = z.object({
+    rootUrl: z.string().url(),
+    maxPages: z.number().int().min(1).max(100).optional().default(20),
+    category: z.string().optional().default("general")
+});
+const SearchDocsSchema = z.object({
+    query: z.string().min(1).max(500),
+    maxResults: z.number().int().min(1).max(20).optional().default(5)
+});
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+    return {
+        tools: [
+            {
+                name: "scrape_single_page",
+                description: "Scrape a single Salesforce documentation page (handles both developer.salesforce and help.salesforce iframes/structures). Returns markdown.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        url: { type: "string" },
+                        category: { type: "string" }
+                    },
+                    required: ["url"]
+                }
+            },
+            {
+                name: "mass_extract_guide",
+                description: "Spiders a root Salesforce documentation page, extracts hierarchical links, and scrapes them in bulk. Stores contents in a local SQLite database for later searching.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        rootUrl: { type: "string", description: "The Table of Contents or landing page." },
+                        maxPages: { type: "number", description: "Maximum number of pages to extract (default 20, max 100)." },
+                        category: { type: "string" }
+                    },
+                    required: ["rootUrl"]
+                }
+            },
+            {
+                name: "search_local_docs",
+                description: "Search locally extracted Salesforce documentation in the SQLite database.",
+                inputSchema: {
+                    type: "object",
+                    properties: {
+                        query: { type: "string" },
+                        maxResults: { type: "number" }
+                    },
+                    required: ["query"]
+                }
+            }
+        ]
+    };
+});
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+    const { name, arguments: args } = request.params;
+    try {
+        if (name === "scrape_single_page") {
+            const { url, category } = ScrapePageSchema.parse(args);
+            console.error(`Scraping ${url}...`);
+            const result = await scrapePage(url);
+            if (result.error) {
+                return { content: [{ type: "text", text: `Failed to scrape: ${result.error}` }], isError: true };
+            }
+            // Save automatically to local DB
+            await saveDocument(url, result.title, result.markdown, result.hash, category);
+            return {
+                content: [{ type: "text", text: `# ${result.title}\n\n${result.markdown}` }]
+            };
+        }
+        if (name === "mass_extract_guide") {
+            const { rootUrl, maxPages, category } = MassExtractSchema.parse(args);
+            console.error(`Starting mass extraction at ${rootUrl}`);
+            // Scrape root to get links
+            const rootResult = await scrapePage(rootUrl, new URL(rootUrl).origin);
+            if (rootResult.error) {
+                return { content: [{ type: "text", text: `Root scrape failed: ${rootResult.error}` }], isError: true };
+            }
+            await saveDocument(rootUrl, rootResult.title, rootResult.markdown, rootResult.hash, category);
+            const queue = [...new Set(rootResult.childLinks)].filter(l => l !== rootUrl).slice(0, maxPages);
+            let successRaw = 1;
+            let failureCount = 0;
+            for (const link of queue) {
+                console.error(`Scraping queued link: ${link}`);
+                const pg = await scrapePage(link, new URL(rootUrl).origin);
+                if (!pg.error) {
+                    await saveDocument(pg.url, pg.title, pg.markdown, pg.hash, category);
+                    successRaw++;
+                }
+                else {
+                    console.error(`Failed on ${link}: ${pg.error}`);
+                    failureCount++;
+                }
+            }
+            return {
+                content: [{ type: "text", text: `Mass extraction complete.\nSuccessfully extracted and saved ${successRaw} pages.\nFailed: ${failureCount} pages.\nDatabase updated.` }]
+            };
+        }
+        if (name === "search_local_docs") {
+            const { query, maxResults } = SearchDocsSchema.parse(args);
+            const results = await searchDocuments(query, maxResults);
+            if (results.length === 0) {
+                return { content: [{ type: "text", text: "No results found in the local database." }] };
+            }
+            let output = `# Search Results for "${query}"\n\n`;
+            for (const r of results) {
+                output += `## [${r.title}](${r.url})\n*Category: ${r.category}* | *Score: ${(r.score * 100).toFixed(1)}%*\n\n`;
+                output += `> ${r.matchContent.substring(0, 500)}...\n\n---\n`;
+            }
+            return { content: [{ type: "text", text: output }] };
+        }
+        return {
+            content: [{ type: "text", text: `Unknown tool: ${name}` }],
+            isError: true
+        };
+    }
+    catch (e) {
+        return {
+            content: [{ type: "text", text: `Error: ${e.message}` }],
+            isError: true
+        };
+    }
+});
+// Clean up Puppeteer on exit
+process.on('SIGINT', async () => {
+    await closeBrowser();
+    process.exit(0);
+});
+process.on('SIGTERM', async () => {
+    await closeBrowser();
+    process.exit(0);
+});
+async function main() {
+    console.error("Starting Unified Salesforce Docs MCP Server...");
+    const transport = new StdioServerTransport();
+    await server.connect(transport);
+    console.error("Server running on stdio transport.");
+}
+main().catch(console.error);

package/dist/scraper.js

@@ -0,0 +1,275 @@
+import puppeteer from 'puppeteer';
+import TurndownService from 'turndown';
+// @ts-ignore
+import { gfm } from 'turndown-plugin-gfm';
+import crypto from 'crypto';
+// Configure Turndown for better markdown output
+const turndownService = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced',
+    bulletListMarker: '-',
+});
+turndownService.use(gfm);
+turndownService.addRule('codeBlocks', {
+    filter: ['pre'],
+    replacement: function (_content, node) {
+        const element = node;
+        const code = element.querySelector('code');
+        const language = code?.className?.match(/language-(\w+)/)?.[1] || '';
+        const codeContent = code?.textContent || element.textContent || '';
+        return `\n\`\`\`${language}\n${codeContent}\n\`\`\`\n`;
+    },
+});
+let browser = null;
+async function getBrowser() {
+    if (!browser) {
+        browser = await puppeteer.launch({
+            headless: true,
+            args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
+        });
+    }
+    return browser;
+}
+/**
+ * Extracts content from a single URL, handling shadow DOMs, iframes, and various SFDC template structures.
+ */
+export async function scrapePage(url, baseDomain) {
+    const browserInstance = await getBrowser();
+    const page = await browserInstance.newPage();
+    try {
+        await page.setViewport({ width: 1280, height: 800 });
+        // User agent to look normal
+        await page.setUserAgent('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+        // Wait until network is idle specifically to handle SPA renders and iframe loads
+        await page.goto(url, { waitUntil: 'networkidle0', timeout: 60000 });
+        // Wait for specific Salesforce content locators to appear to avoid grabbing 'Loading...' pages
+        try {
+            if (url.includes('help.salesforce.com')) {
+                await page.waitForSelector('.slds-text-longform', { timeout: 15000 });
+            }
+            else if (url.includes('developer.salesforce.com')) {
+                await page.waitForFunction(() => {
+                    return document.querySelector('doc-content-layout') ||
+                        document.querySelector('doc-xml-content') ||
+                        document.querySelector('iframe');
+                }, { timeout: 10000 });
+            }
+        }
+        catch (e) {
+            console.warn(`Timeout waiting for specific content selectors on ${url}`);
+        }
+        // Additional wait just in case visual components are still sliding in
+        await new Promise(r => setTimeout(r, 2000));
+        // Take an opportunistic screenshot if development debugging layout issues
+        if (url.includes('help.salesforce.com')) {
+            await page.screenshot({ path: 'help_debug.png' }).catch(() => { });
+        }
+        // In-page extraction script
+        const extraction = await page.evaluate(() => {
+            let title = 'Untitled';
+            let html = '';
+            const childLinks = new Set();
+            // Collect all same-site hierarchical links (help to spider)
+            document.querySelectorAll('a').forEach(a => {
+                if (a.href && !a.href.startsWith('java') && !a.href.startsWith('mailto')) {
+                    childLinks.add(a.href);
+                }
+            });
+            // Helper function to extract readable HTML piercing shadow DOMs (legacy sf-doc-scraper behavior)
+            function extractReadableHTML(element) {
+                if (!element)
+                    return '';
+                const tagName = element.tagName?.toLowerCase();
+                if (tagName === 'doc-heading') {
+                    const headingEl = element.shadowRoot?.querySelector('h2, h3, h4');
+                    const headingContent = element.shadowRoot?.querySelector('doc-heading-content');
+                    const titleSpan = headingContent?.shadowRoot?.querySelector('.title');
+                    const headingText = titleSpan?.textContent?.trim() || element.getAttribute('header') || '';
+                    const level = headingEl?.tagName?.toLowerCase() || 'h2';
+                    return `<${level}>${headingText}</${level}>`;
+                }
+                if (tagName === 'doc-content-callout') {
+                    const shadowDiv = element.shadowRoot?.querySelector('.dx-callout');
+                    const isTip = shadowDiv?.classList?.contains('dx-callout-tip');
+                    const isWarning = shadowDiv?.classList?.contains('dx-callout-warning');
+                    let calloutType = 'Note';
+                    if (isTip)
+                        calloutType = 'Tip';
+                    if (isWarning)
+                        calloutType = 'Warning';
+                    const slottedContent = element.innerHTML;
+                    return `<blockquote><strong>${calloutType}:</strong> ${slottedContent}</blockquote>`;
+                }
+                if (tagName === 'dx-code-block') {
+                    const language = element.getAttribute('language') || '';
+                    const code = element.getAttribute('code-block') || element.textContent || '';
+                    const decodedCode = code
+                        .replace(/&quot;/g, '"')
+                        .replace(/&lt;/g, '<')
+                        .replace(/&gt;/g, '>')
+                        .replace(/&amp;/g, '&');
+                    return `<pre><code class="language-${language}">${decodedCode}</code></pre>`;
+                }
+                if (tagName === 'div' && element.classList?.contains('custom-code-block')) {
+                    const codeBlock = element.querySelector('dx-code-block');
+                    if (codeBlock)
+                        return extractReadableHTML(codeBlock);
+                }
+                return element.outerHTML;
+            }
+            // Helper function to find element deep in shadow DOMs
+            function deepQuerySelector(root, selector) {
+                const found = root.querySelector(selector);
+                if (found)
+                    return found;
+                const allElements = root.querySelectorAll('*');
+                for (const el of Array.from(allElements)) {
+                    if (el.shadowRoot) {
+                        const deepFound = deepQuerySelector(el.shadowRoot, selector);
+                        if (deepFound)
+                            return deepFound;
+                    }
+                }
+                return null;
+            }
+            // 1. Try to extract from an iframe (Older Developer Guides like life_sciences_dev_guide)
+            const iframe = document.querySelector('iframe');
+            if (iframe && iframe.contentDocument && iframe.contentDocument.body) {
+                // Find main content inside iframe
+                const docHtml = iframe.contentDocument.querySelector('#doc')?.innerHTML ||
+                    iframe.contentDocument.querySelector('body')?.innerHTML || '';
+                const docTitle = iframe.contentDocument.querySelector('title')?.innerText ||
+                    iframe.contentDocument.querySelector('h1')?.innerText || 'Untitled';
+                // Get links inside iframe
+                iframe.contentDocument.querySelectorAll('a').forEach(a => {
+                    if (a.href && !a.href.startsWith('java') && !a.href.startsWith('mailto')) {
+                        childLinks.add(a.href);
+                    }
+                });
+                if (docHtml.length > 500) {
+                    return { html: docHtml, title: docTitle, childLinks: Array.from(childLinks) };
+                }
+            }
+            // 2. Try help.salesforce.com specific structure (often inside shadow DOMs)
+            const sldsText = deepQuerySelector(document, '.slds-text-longform');
+            if (sldsText) {
+                const rawTitle = document.querySelector('title')?.innerText || 'Untitled';
+                const cleanTitle = rawTitle.replace(' | Salesforce', '').trim();
+                return { html: sldsText.innerHTML, title: cleanTitle, childLinks: Array.from(childLinks) };
+            }
+            // 2.5 Try doc-xml-content (Legacy Developer Guides, like Health Cloud / Life Sciences)
+            const docXmlContent = document.querySelector('doc-xml-content');
+            if (docXmlContent?.shadowRoot) {
+                const docContent = docXmlContent.shadowRoot.querySelector('doc-content');
+                if (docContent?.shadowRoot) {
+                    const innerHtml = docContent.shadowRoot.innerHTML;
+                    // Extract title from h1
+                    const h1Match = innerHtml.match(/<h1[^>]*>(.*?)<\/h1>/);
+                    if (h1Match)
+                        title = h1Match[1].replace(/<[^>]*>?/gm, '');
+                    // Find child links within the shadow DOM
+                    const shadowLinks = docContent.shadowRoot.querySelectorAll('a');
+                    shadowLinks.forEach(a => {
+                        if (a.href && !a.href.startsWith('java') && !a.href.startsWith('mailto')) {
+                            childLinks.add(a.href);
+                        }
+                    });
+                    return { html: innerHtml, title, childLinks: Array.from(childLinks) };
+                }
+            }
+            // 3. Try doc-content-layout / doc-amf-reference (New Developer Guides)
+            const docRef = document.querySelector('doc-amf-reference');
+            if (docRef) {
+                const markdownContent = docRef.querySelector('.markdown-content');
+                if (markdownContent) {
+                    let refHtml = '';
+                    for (const el of Array.from(markdownContent.children)) {
+                        if (el.tagName?.toLowerCase() === 'h1')
+                            title = el.textContent?.trim() || title;
+                        refHtml += extractReadableHTML(el);
+                    }
+                    if (!title || title === 'Untitled')
+                        title = document.querySelector('title')?.innerText || 'Untitled';
+                    return { html: refHtml, title, childLinks: Array.from(childLinks) };
+                }
+            }
+            const docLayout = document.querySelector('doc-content-layout');
+            if (docLayout?.shadowRoot) {
+                const slot = docLayout.shadowRoot.querySelector('.content-body slot');
+                if (slot) {
+                    const assignedElements = slot.assignedElements();
+                    if (assignedElements.length > 0) {
+                        let guideHtml = '';
+                        for (const el of assignedElements) {
+                            if (el.tagName?.toLowerCase() === 'h1')
+                                title = el.textContent?.trim() || title;
+                            guideHtml += extractReadableHTML(el);
+                        }
+                        if (!title || title === 'Untitled')
+                            title = document.querySelector('title')?.innerText || 'Untitled';
+                        return { html: guideHtml, title, childLinks: Array.from(childLinks) };
+                    }
+                }
+            }
+            // 4. Fallback: <article> or <main>
+            const container = document.querySelector('article') || document.querySelector('main');
+            if (container) {
+                title = document.querySelector('h1')?.innerText || document.querySelector('title')?.innerText || 'Untitled';
+                return { html: container.innerHTML, title, childLinks: Array.from(childLinks) };
+            }
+            // Complete fallback
+            return {
+                html: document.body.innerHTML,
+                title: document.querySelector('title')?.innerText || 'Untitled',
+                childLinks: Array.from(childLinks)
+            };
+        });
+        if (!extraction.html || extraction.html.trim() === '') {
+            return {
+                url,
+                title: 'Untitled',
+                markdown: '',
+                hash: '',
+                error: 'No content found on page',
+                childLinks: []
+            };
+        }
+        // Convert to markdown
+        let markdown = turndownService.turndown(extraction.html);
+        // Filter child links to stay within the domain/base if provided, to avoid massive spidering
+        let validLinks = extraction.childLinks;
+        if (baseDomain) {
+            validLinks = validLinks.filter(l => l.startsWith(baseDomain));
+        }
+        const hash = crypto.createHash('sha256').update(markdown).digest('hex');
+        return {
+            url,
+            title: extraction.title,
+            markdown,
+            hash,
+            childLinks: validLinks,
+        };
+    }
+    catch (error) {
+        return {
+            url,
+            title: 'Error',
+            markdown: '',
+            hash: '',
+            error: error.message,
+            childLinks: [],
+        };
+    }
+    finally {
+        await page.close();
+    }
+}
+/**
+ * Ensures the browser is closed when application shuts down
+ */
+export async function closeBrowser() {
+    if (browser) {
+        await browser.close();
+        browser = null;
+    }
+}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "unified-sf-docs-mcp",
+  "version": "1.0.0",
+  "description": "Unified MCP server for scraping Salesforce developer documentation and searching a local SQLite vector-like database.",
+  "type": "module",
+  "main": "dist/index.js",
+  "bin": {
+    "unified-sf-docs-mcp": "dist/index.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "tsx tests/test-all.js",
+    "test:core": "tsx tests/test-core.js"
+  },
+  "keywords": [
+    "salesforce",
+    "mcp",
+    "model-context-protocol"
+  ],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.0",
+    "puppeteer": "^24.1.0",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2",
+    "sql.js": "^1.10.0",
+    "lru-cache": "^10.0.0",
+    "zod": "^3.22.0"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "@types/turndown": "^5.0.4",
+    "tsx": "^4.0.0",
+    "typescript": "^5.3.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  }
+}