sfcc-dev-mcp 1.0.15 → 1.0.19

package/.github/skills/salesforce-developer-site-scraper/scripts/scrape-to-markdown.js

@@ -0,0 +1,205 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('fs/promises');
+const path = require('path');
+const { chromium } = require('playwright');
+const { Readability } = require('@mozilla/readability');
+const { JSDOM } = require('jsdom');
+const TurndownService = require('turndown');
+
+const DEFAULT_TIMEOUT_MS = 45000;
+const DEFAULT_REMOVE_SELECTORS = [
+  '#onetrust-consent-sdk',
+  '#onetrust-banner-sdk',
+  '.onetrust-pc-sdk',
+  '.ot-sdk-container',
+  '.ot-sdk-row',
+  '#ot-sdk-btn',
+  '.ot-sdk-show-settings',
+  'iframe[id*="onetrust"]',
+  'iframe[src*="onetrust"]',
+  '[id*="onetrust"]',
+  '[class*="onetrust"]',
+];
+
+function getArg(flag) {
+  const index = process.argv.indexOf(flag);
+  if (index === -1) return null;
+  return process.argv[index + 1] || null;
+}
+
+function hasFlag(flag) {
+  return process.argv.includes(flag);
+}
+
+function parseCookie(cookieStr) {
+  if (!cookieStr) return null;
+  const parts = cookieStr.split(';').map((part) => part.trim());
+  const [nameValue, ...attrs] = parts;
+  const eqIndex = nameValue.indexOf('=');
+  if (eqIndex === -1) return null;
+  const cookie = {
+    name: nameValue.slice(0, eqIndex),
+    value: nameValue.slice(eqIndex + 1),
+  };
+  for (const attr of attrs) {
+    const [key, value] = attr.split('=');
+    if (!key) continue;
+    const normalized = key.toLowerCase();
+    if (normalized === 'domain') cookie.domain = value;
+    if (normalized === 'path') cookie.path = value;
+    if (normalized === 'secure') cookie.secure = true;
+    if (normalized === 'samesite') cookie.sameSite = value;
+  }
+  return cookie;
+}
+
+function parseList(listStr) {
+  if (!listStr) return [];
+  return listStr
+    .split(',')
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+function requireArg(flag, value) {
+  if (!value) {
+    console.error(`Missing required argument: ${flag}`);
+    process.exit(1);
+  }
+}
+
+async function main() {
+  const url = getArg('--url');
+  const outPath = getArg('--out');
+  const consentSelector = getArg('--consent-selector');
+  const waitMs = Number(getArg('--wait') || 0);
+  const contentSelector = getArg('--content-selector');
+  const removeSelectorsArg = getArg('--remove-selectors');
+  const cookieStr = getArg('--cookie');
+  const storageState = getArg('--storage-state');
+  const timeoutMs = Number(getArg('--timeout') || DEFAULT_TIMEOUT_MS);
+  const disableDefaultRemovals = hasFlag('--no-default-removals');
+
+  requireArg('--url', url);
+  requireArg('--out', outPath);
+
+  const browser = await chromium.launch({ headless: !hasFlag('--headed') });
+  const contextOptions = storageState ? { storageState } : {};
+  const context = await browser.newContext(contextOptions);
+
+  let docsApiContent = null;
+  let docsApiTitle = null;
+
+  const cookie = parseCookie(cookieStr);
+  if (cookie && cookie.domain) {
+    await context.addCookies([cookie]);
+  }
+
+  const page = await context.newPage();
+  page.on('response', async (response) => {
+    const responseUrl = response.url();
+    if (!responseUrl.includes('/docs/get_document_content/')) return;
+    try {
+      const payload = await response.json();
+      if (payload && payload.content) {
+        docsApiContent = payload.content;
+        docsApiTitle = payload.title || payload.id || docsApiTitle;
+      }
+    } catch (error) {
+      // Ignore non-JSON responses.
+    }
+  });
+  await page.goto(url, { waitUntil: 'networkidle', timeout: timeoutMs });
+
+  if (consentSelector) {
+    try {
+      await page.locator(consentSelector).first().click({ timeout: 3000 });
+      await page.waitForLoadState('networkidle', { timeout: timeoutMs });
+    } catch (error) {
+      console.warn('Consent selector not found or not clickable. Continuing.');
+    }
+  }
+
+  if (waitMs > 0) {
+    await page.waitForTimeout(waitMs);
+  }
+
+  const removeSelectors = [
+    ...(disableDefaultRemovals ? [] : DEFAULT_REMOVE_SELECTORS),
+    ...parseList(removeSelectorsArg),
+  ];
+  if (removeSelectors.length > 0) {
+    await page.evaluate((selectors) => {
+      selectors.forEach((selector) => {
+        document.querySelectorAll(selector).forEach((node) => node.remove());
+      });
+    }, removeSelectors);
+  }
+
+  let html = null;
+  let title = null;
+
+  if (contentSelector) {
+    try {
+      html = await page.locator(contentSelector).first().evaluate((el) => el.outerHTML);
+      title = await page.title();
+    } catch (error) {
+      console.warn('Content selector not found. Falling back to Readability.');
+    }
+  }
+
+  if (!html && !contentSelector) {
+    try {
+      html = await page.locator('main, article').first().evaluate((el) => el.outerHTML);
+      title = await page.title();
+    } catch (error) {
+      // Fall through to Readability.
+    }
+  }
+
+  if (!html) {
+    const pageContent = await page.content();
+    const dom = new JSDOM(pageContent, { url });
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+    if (article && article.content) {
+      html = article.content;
+      title = article.title || await page.title();
+    } else {
+      html = dom.window.document.body ? dom.window.document.body.innerHTML : pageContent;
+      title = await page.title();
+    }
+  }
+
+  const turndown = new TurndownService({ headingStyle: 'atx' });
+  let markdownBody = turndown.turndown(html || '');
+
+  if (markdownBody.trim().length < 80 && docsApiContent) {
+    markdownBody = turndown.turndown(docsApiContent);
+    title = docsApiTitle || title;
+  }
+  const isoStamp = new Date().toISOString();
+  const markdown = [
+    `# ${title || 'Documentation Snapshot'}`,
+    '',
+    `Source: ${url}`,
+    `Fetched: ${isoStamp}`,
+    '',
+    markdownBody.trim(),
+    '',
+  ].join('\n');
+
+  await fs.mkdir(path.dirname(outPath), { recursive: true });
+  await fs.writeFile(outPath, markdown, 'utf8');
+
+  await browser.close();
+  console.log(`Saved Markdown to ${outPath}`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});

package/.github/skills/salesforce-help-site-scraper/SKILL.md

@@ -0,0 +1,84 @@
+---
+name: salesforce-help-site-scraper
+description: 'Scrape Salesforce Help articles into clean Markdown with consent handling and content cleanup. Use when you need an internal, readable snapshot of Help content for research or documentation support.'
+license: Forward Proprietary
+compatibility: VS Code 1.x+, Node.js 18+
+---
+
+# Salesforce Help Site Scraper
+
+Use this skill to extract Salesforce Help article content into clean Markdown when pages render dynamically or are blocked by consent banners.
+
+## When to Use This Skill
+
+- You need a readable Markdown snapshot of a Help article for internal research.
+- OneTrust cookie banners block access to the main content.
+- You want to remove headers, footers, or navigation chrome before extraction.
+- NOT for: high-volume crawling, bypassing access controls, or republishing Salesforce content.
+
+## Prerequisites
+
+- Node.js 18+
+- Scraper script at `skills/salesforce-help-site-scraper/scripts/scrape-help-to-markdown.js`
+
+## How to Use
+
+### Basic Usage
+
+```bash
+node skills/salesforce-help-site-scraper/scripts/scrape-help-to-markdown.js \
+  --url "https://help.salesforce.com/s/articleView?id=sf.flow.htm&type=5" \
+  --out "./artifacts/online-research/help_flow_overview.md" \
+  --consent-selector "#onetrust-accept-btn-handler" \
+  --remove-selectors "header,footer,nav,aside" \
+  --wait 2500
+```
+
+### Script Options
+
+| Option | Required | Description |
+| --- | --- | --- |
+| `--url` | Yes | Target Help article URL. |
+| `--out` | Yes | Output Markdown file path. |
+| `--consent-selector` | No | Selector for cookie/consent accept button (OneTrust). |
+| `--remove-selectors` | No | Comma-separated selectors to remove before extraction. |
+| `--wait` | No | Milliseconds to wait after navigation or consent click. |
+
+## Compliance Notes
+
+- Prefer the Salesforce Knowledge APIs for structured, supported access where possible.
+- Check and respect `robots.txt` before scraping.
+- Do not republish or redistribute Salesforce Help content.
+- Attribute content to Salesforce when used internally.
+
+## Examples
+
+### Example: Capture a Flow Help article
+
+```bash
+node skills/salesforce-help-site-scraper/scripts/scrape-help-to-markdown.js \
+  --url "https://help.salesforce.com/s/articleView?id=sf.flow_build.htm&type=5" \
+  --out "./artifacts/online-research/help_flow_build.md" \
+  --consent-selector "#onetrust-accept-btn-handler" \
+  --remove-selectors "header,footer,nav,aside" \
+  --wait 2500
+```
+
+## Troubleshooting
+
+### Issue: Output is empty or too short
+
+**Solution**: Increase `--wait` or refine `--remove-selectors` to avoid removing the main content container.
+
+### Issue: Consent banner blocks content
+
+**Solution**: Provide `--consent-selector` for the OneTrust accept button.
+
+## References
+
+- Salesforce Help: https://help.salesforce.com/
+- Salesforce Knowledge Developer Guide: https://developer.salesforce.com/docs/atlas.en-us.knowledge_dev.meta/knowledge_dev/
+- Salesforce REST API - Knowledge Resources: https://developer.salesforce.com/docs/atlas.en-us.api_rest.meta/api_rest/resources_knowledge.htm
+- Salesforce Master Subscription Agreement: https://www.salesforce.com/company/legal/agreements/
+- Salesforce Intellectual Property and Trademarks: https://www.salesforce.com/company/legal/intellectual/
+- Robots Exclusion Protocol (RFC 9309): https://www.rfc-editor.org/rfc/rfc9309

package/.github/skills/salesforce-help-site-scraper/scripts/scrape-help-to-markdown.js

@@ -0,0 +1,212 @@
+#!/usr/bin/env node
+
+'use strict';
+
+const fs = require('fs/promises');
+const path = require('path');
+const { chromium } = require('playwright');
+const { Readability } = require('@mozilla/readability');
+const { JSDOM } = require('jsdom');
+const TurndownService = require('turndown');
+
+const DEFAULT_TIMEOUT_MS = 45000;
+const DEFAULT_REMOVE_SELECTORS = [
+  '#onetrust-consent-sdk',
+  '#onetrust-banner-sdk',
+  '.onetrust-pc-sdk',
+  '.ot-sdk-container',
+  '.ot-sdk-row',
+  '#ot-sdk-btn',
+  '.ot-sdk-show-settings',
+  'iframe[id*="onetrust"]',
+  'iframe[src*="onetrust"]',
+  '[id*="onetrust"]',
+  '[class*="onetrust"]',
+];
+const DEFAULT_CONTENT_SELECTORS = [
+  'main article',
+  'article',
+  'main',
+  '.slds-rich-text-editor__output',
+  '.article-body',
+  '.help-article',
+  '[data-aura-class*="article"]',
+];
+
+function getArg(flag) {
+  const index = process.argv.indexOf(flag);
+  if (index === -1) return null;
+  return process.argv[index + 1] || null;
+}
+
+function hasFlag(flag) {
+  return process.argv.includes(flag);
+}
+
+function parseCookie(cookieStr) {
+  if (!cookieStr) return null;
+  const parts = cookieStr.split(';').map((part) => part.trim());
+  const [nameValue, ...attrs] = parts;
+  const eqIndex = nameValue.indexOf('=');
+  if (eqIndex === -1) return null;
+  const cookie = {
+    name: nameValue.slice(0, eqIndex),
+    value: nameValue.slice(eqIndex + 1),
+  };
+  for (const attr of attrs) {
+    const [key, value] = attr.split('=');
+    if (!key) continue;
+    const normalized = key.toLowerCase();
+    if (normalized === 'domain') cookie.domain = value;
+    if (normalized === 'path') cookie.path = value;
+    if (normalized === 'secure') cookie.secure = true;
+    if (normalized === 'samesite') cookie.sameSite = value;
+  }
+  return cookie;
+}
+
+function parseList(listStr) {
+  if (!listStr) return [];
+  return listStr
+    .split(',')
+    .map((item) => item.trim())
+    .filter(Boolean);
+}
+
+function requireArg(flag, value) {
+  if (!value) {
+    console.error(`Missing required argument: ${flag}`);
+    process.exit(1);
+  }
+}
+
+async function pickBestContentHtml(page, selectors, minLength) {
+  return page.evaluate(
+    ({ selectors, minLength }) => {
+      let best = null;
+      let bestScore = 0;
+      selectors.forEach((selector) => {
+        document.querySelectorAll(selector).forEach((el) => {
+          const text = (el.innerText || '').trim();
+          const score = text.length;
+          if (score >= minLength && score > bestScore) {
+            bestScore = score;
+            best = el;
+          }
+        });
+      });
+      return best ? best.outerHTML : null;
+    },
+    { selectors, minLength }
+  );
+}
+
+async function main() {
+  const url = getArg('--url');
+  const outPath = getArg('--out');
+  const consentSelector = getArg('--consent-selector');
+  const waitMs = Number(getArg('--wait') || 0);
+  const contentSelector = getArg('--content-selector');
+  const removeSelectorsArg = getArg('--remove-selectors');
+  const cookieStr = getArg('--cookie');
+  const storageState = getArg('--storage-state');
+  const timeoutMs = Number(getArg('--timeout') || DEFAULT_TIMEOUT_MS);
+  const disableDefaultRemovals = hasFlag('--no-default-removals');
+
+  requireArg('--url', url);
+  requireArg('--out', outPath);
+
+  const browser = await chromium.launch({ headless: !hasFlag('--headed') });
+  const contextOptions = storageState ? { storageState } : {};
+  const context = await browser.newContext(contextOptions);
+
+  const cookie = parseCookie(cookieStr);
+  if (cookie && cookie.domain) {
+    await context.addCookies([cookie]);
+  }
+
+  const page = await context.newPage();
+  await page.goto(url, { waitUntil: 'networkidle', timeout: timeoutMs });
+
+  if (consentSelector) {
+    try {
+      await page.locator(consentSelector).first().click({ timeout: 3000 });
+      await page.waitForLoadState('networkidle', { timeout: timeoutMs });
+    } catch (error) {
+      console.warn('Consent selector not found or not clickable. Continuing.');
+    }
+  }
+
+  if (waitMs > 0) {
+    await page.waitForTimeout(waitMs);
+  }
+
+  const removeSelectors = [
+    ...(disableDefaultRemovals ? [] : DEFAULT_REMOVE_SELECTORS),
+    ...parseList(removeSelectorsArg),
+  ];
+  if (removeSelectors.length > 0) {
+    await page.evaluate((selectors) => {
+      selectors.forEach((selector) => {
+        document.querySelectorAll(selector).forEach((node) => node.remove());
+      });
+    }, removeSelectors);
+  }
+
+  let html = null;
+  let title = null;
+
+  if (contentSelector) {
+    try {
+      html = await page.locator(contentSelector).first().evaluate((el) => el.outerHTML);
+      title = await page.title();
+    } catch (error) {
+      console.warn('Content selector not found. Falling back to automatic selection.');
+    }
+  }
+
+  if (!html) {
+    html = await pickBestContentHtml(page, DEFAULT_CONTENT_SELECTORS, 200);
+    if (html) {
+      title = await page.title();
+    }
+  }
+
+  if (!html) {
+    const pageContent = await page.content();
+    const dom = new JSDOM(pageContent, { url });
+    const reader = new Readability(dom.window.document);
+    const article = reader.parse();
+    if (article && article.content) {
+      html = article.content;
+      title = article.title || (await page.title());
+    } else {
+      html = dom.window.document.body ? dom.window.document.body.innerHTML : pageContent;
+      title = await page.title();
+    }
+  }
+
+  const turndown = new TurndownService({ headingStyle: 'atx' });
+  const markdownBody = turndown.turndown(html || '');
+  const isoStamp = new Date().toISOString();
+  const markdown = [
+    `# ${title || 'Help Article Snapshot'}`,
+    '',
+    `Source: ${url}`,
+    `Fetched: ${isoStamp}`,
+    '',
+    markdownBody.trim(),
+    '',
+  ].join('\n');
+
+  await fs.mkdir(path.dirname(outPath), { recursive: true });
+  await fs.writeFile(outPath, markdown, 'utf8');
+
+  await browser.close();
+  console.log(`Saved Markdown to ${outPath}`);
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});