wp-devdocs-mcp 1.1.1

package/src/indexer/index-manager.js

@@ -0,0 +1,187 @@
+import fg from 'fast-glob';
+import { readFileSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import { createHash } from 'node:crypto';
+import { fetchSource } from './sources/index.js';
+import { parsePhpFile } from './php-parser.js';
+import { parseJsFile } from './js-parser.js';
+import {
+  listSources,
+  getSource,
+  upsertHook,
+  markHooksRemoved,
+  upsertBlockRegistration,
+  upsertApiUsage,
+  getIndexedFile,
+  upsertIndexedFile,
+  updateSourceLastIndexed,
+} from '../db/sqlite.js';
+import { indexDocsSource } from '../docs/doc-index-manager.js';
+
+const IGNORE_PATTERNS = [
+  '**/node_modules/**',
+  '**/vendor/**',
+  '**/dist/**',
+  '**/build/**',
+  '**/.git/**',
+  '**/tests/**',
+  '**/test/**',
+  '**/__tests__/**',
+  '**/spec/**',
+];
+
+const PHP_PATTERNS = ['**/*.php'];
+const JS_PATTERNS = ['**/*.js', '**/*.jsx', '**/*.ts', '**/*.tsx'];
+
+/**
+ * Index all enabled sources or a specific source.
+ * @param {object} opts - { sourceName, force }
+ * @returns {object} Stats about the indexing run
+ */
+export async function indexSources(opts = {}) {
+  const { sourceName, force = false } = opts;
+
+  let sources;
+  if (sourceName) {
+    const source = getSource(sourceName);
+    if (!source) throw new Error(`Source not found: ${sourceName}`);
+    if (!source.enabled) throw new Error(`Source "${sourceName}" is disabled`);
+    sources = [source];
+  } else {
+    sources = listSources().filter(s => s.enabled);
+  }
+
+  if (sources.length === 0) {
+    return { message: 'No enabled sources found. Add a source first.' };
+  }
+
+  const stats = {
+    sources_processed: 0,
+    files_processed: 0,
+    files_skipped: 0,
+    hooks_inserted: 0,
+    hooks_updated: 0,
+    hooks_skipped: 0,
+    hooks_removed: 0,
+    blocks_indexed: 0,
+    apis_indexed: 0,
+    docs_inserted: 0,
+    docs_updated: 0,
+    docs_skipped: 0,
+    docs_removed: 0,
+    errors: [],
+  };
+
+  for (const source of sources) {
+    try {
+      console.error(`Fetching source: ${source.name} (${source.type})...`);
+      const localPath = await fetchSource(source);
+      console.error(`Indexing source: ${source.name} from ${localPath} (${source.content_type || 'source'})`);
+
+      if (source.content_type === 'docs') {
+        await indexDocsSource(source, localPath, force, stats);
+      } else {
+        await indexSource(source, localPath, force, stats);
+      }
+      stats.sources_processed++;
+      updateSourceLastIndexed(source.id);
+    } catch (err) {
+      const msg = `Error processing source "${source.name}": ${err.message}`;
+      console.error(msg);
+      stats.errors.push(msg);
+    }
+  }
+
+  return stats;
+}
+
+/**
+ * Index a single source — scans files, parses hooks/blocks/APIs, and upserts into the database.
+ * @param {object} source - Source row from the database
+ * @param {string} localPath - Absolute path to the source on disk
+ * @param {boolean} force - Skip mtime/hash caching when true
+ * @param {object} stats - Mutable stats object to accumulate counts
+ */
+async function indexSource(source, localPath, force, stats) {
+  // Scan for PHP and JS/TS files
+  const files = await fg([...PHP_PATTERNS, ...JS_PATTERNS], {
+    cwd: localPath,
+    ignore: IGNORE_PATTERNS,
+    absolute: false,
+    onlyFiles: true,
+  });
+
+  console.error(`Found ${files.length} files to check in ${source.name}`);
+
+  for (const file of files) {
+    const fullPath = join(localPath, file);
+
+    try {
+      const fileStat = statSync(fullPath);
+      const mtimeMs = fileStat.mtimeMs;
+
+      // Check mtime for incremental skip
+      const indexed = !force ? getIndexedFile(source.id, file) : null;
+      if (indexed && indexed.mtime_ms === mtimeMs) {
+        stats.files_skipped++;
+        continue;
+      }
+
+      const content = readFileSync(fullPath, 'utf-8');
+      const contentHash = createHash('sha256').update(content).digest('hex').slice(0, 16);
+
+      // Skip if content hash matches (handles moved files / touched timestamps)
+      if (indexed && indexed.content_hash === contentHash) {
+        upsertIndexedFile(source.id, file, mtimeMs, contentHash);
+        stats.files_skipped++;
+        continue;
+      }
+
+      const isPhp = file.endsWith('.php');
+      const activeHookIds = [];
+
+      if (isPhp) {
+        const hooks = parsePhpFile(content, file, source.id);
+        for (const hook of hooks) {
+          const result = upsertHook(hook);
+          activeHookIds.push(result.id);
+          if (result.action === 'inserted') stats.hooks_inserted++;
+          else if (result.action === 'updated') stats.hooks_updated++;
+          else stats.hooks_skipped++;
+        }
+      } else {
+        const { hooks, blocks, apis } = parseJsFile(content, file, source.id);
+
+        for (const hook of hooks) {
+          const result = upsertHook(hook);
+          activeHookIds.push(result.id);
+          if (result.action === 'inserted') stats.hooks_inserted++;
+          else if (result.action === 'updated') stats.hooks_updated++;
+          else stats.hooks_skipped++;
+        }
+
+        for (const block of blocks) {
+          upsertBlockRegistration(block);
+          stats.blocks_indexed++;
+        }
+
+        for (const api of apis) {
+          upsertApiUsage(api);
+          stats.apis_indexed++;
+        }
+      }
+
+      // Soft-delete hooks that were in this file but no longer found
+      const removed = markHooksRemoved(source.id, file, activeHookIds);
+      stats.hooks_removed += removed;
+
+      // Track this file as indexed
+      upsertIndexedFile(source.id, file, mtimeMs, contentHash);
+      stats.files_processed++;
+    } catch (err) {
+      const msg = `Error indexing file ${file}: ${err.message}`;
+      console.error(msg);
+      stats.errors.push(msg);
+    }
+  }
+}

package/src/indexer/js-parser.js

@@ -0,0 +1,306 @@
+import {
+  getLineNumber,
+  extractCodeWindow,
+  generateContentHash,
+  inferDescription,
+  extractDocblock,
+  findEnclosingFunction,
+} from './parser-utils.js';
+
+// JS hook patterns — addAction/addFilter from @wordpress/hooks or wp.hooks
+const JS_HOOK_REGEX = /\b(?:addAction|addFilter|applyFilters|doAction)\s*\(\s*/g;
+
+const JS_TYPE_MAP = {
+  addAction: 'js_action',
+  addFilter: 'js_filter',
+  applyFilters: 'js_filter',
+  doAction: 'js_action',
+};
+
+// Block registration patterns
+const BLOCK_REG_REGEX = /\b(registerBlockType|registerBlockVariation|registerBlockStyle|registerBlockCollection)\s*\(\s*/g;
+
+// WP API usage patterns
+const API_USAGE_REGEX = /\bwp\.(blocks|editor|blockEditor|data|element|components|plugins|editPost|editSite|hooks|i18n|richText)\s*\.\s*(\w+)/g;
+
+/**
+ * Parse a JS/TS file and extract hooks, block registrations, and API usages.
+ */
+export function parseJsFile(content, filePath, sourceId) {
+  const hooks = [];
+  const blocks = [];
+  const apis = [];
+  const lines = content.split('\n');
+
+  // --- JS Hooks ---
+  let match;
+  JS_HOOK_REGEX.lastIndex = 0;
+  while ((match = JS_HOOK_REGEX.exec(content)) !== null) {
+    const funcName = match[0].match(/\b(addAction|addFilter|applyFilters|doAction)/)[1];
+    const type = JS_TYPE_MAP[funcName];
+    const startOffset = match.index + match[0].length;
+
+    const argsStr = extractJsArguments(content, startOffset);
+    if (!argsStr) continue;
+
+    const args = splitJsArguments(argsStr);
+    if (args.length === 0) continue;
+
+    const rawName = args[0].trim();
+    const hookName = cleanJsHookName(rawName);
+    if (!hookName) continue;
+
+    const isDynamic = rawName.includes('`') || rawName.includes('${') || rawName.includes('+');
+
+    const lineNumber = getLineNumber(content, match.index);
+    const lineIndex = lineNumber - 1;
+
+    const params = args.slice(1).map(p => p.trim()).filter(Boolean);
+    const docblock = extractDocblock(lines, lineIndex);
+    const { codeBefore, hookLine, codeAfter } = extractCodeWindow(lines, lineIndex);
+    const jsFunction = findEnclosingFunction(lines, lineIndex);
+
+    const hookData = {
+      source_id: sourceId,
+      file_path: filePath,
+      line_number: lineNumber,
+      name: hookName,
+      type,
+      php_function: jsFunction || null,
+      params: params.join(', ') || null,
+      param_count: params.length,
+      docblock: docblock || null,
+      inferred_description: null,
+      function_context: jsFunction || null,
+      class_name: null,
+      code_before: codeBefore || null,
+      code_after: codeAfter || null,
+      hook_line: hookLine || null,
+      is_dynamic: isDynamic ? 1 : 0,
+      content_hash: null,
+    };
+
+    hookData.inferred_description = inferDescription(hookData);
+    hookData.content_hash = generateContentHash(hookData);
+    hooks.push(hookData);
+  }
+
+  // --- Block Registrations ---
+  BLOCK_REG_REGEX.lastIndex = 0;
+  while ((match = BLOCK_REG_REGEX.exec(content)) !== null) {
+    const regFunc = match[1];
+    const startOffset = match.index + match[0].length;
+
+    const argsStr = extractJsArguments(content, startOffset);
+    if (!argsStr) continue;
+
+    const args = splitJsArguments(argsStr);
+    if (args.length === 0) continue;
+
+    const blockName = cleanJsHookName(args[0].trim());
+    if (!blockName) continue;
+
+    const lineNumber = getLineNumber(content, match.index);
+    const lineIndex = lineNumber - 1;
+    const { codeBefore, hookLine, codeAfter } = extractCodeWindow(lines, lineIndex, 4, 20);
+
+    // Try to extract settings object
+    const settingsStr = args.length > 1 ? args.slice(1).join(', ') : '';
+    const blockTitle = extractProperty(settingsStr, 'title');
+    const blockCategory = extractProperty(settingsStr, 'category');
+
+    const codeContext = [codeBefore, hookLine, codeAfter].filter(Boolean).join('\n');
+
+    const blockData = {
+      source_id: sourceId,
+      file_path: filePath,
+      line_number: lineNumber,
+      block_name: blockName,
+      block_title: blockTitle || null,
+      block_category: blockCategory || null,
+      block_attributes: null,
+      supports: null,
+      code_context: codeContext.slice(0, 2000) || null,
+      content_hash: null,
+    };
+
+    blockData.content_hash = generateContentHash({
+      name: blockName,
+      type: regFunc,
+      params: settingsStr,
+      docblock: '',
+      hookLine: hookLine,
+    });
+
+    blocks.push(blockData);
+  }
+
+  // --- API Usages ---
+  API_USAGE_REGEX.lastIndex = 0;
+  while ((match = API_USAGE_REGEX.exec(content)) !== null) {
+    const namespace = match[1];
+    const method = match[2];
+    const apiCall = `wp.${namespace}.${method}`;
+
+    const lineNumber = getLineNumber(content, match.index);
+    const lineIndex = lineNumber - 1;
+    const { codeBefore, hookLine, codeAfter } = extractCodeWindow(lines, lineIndex, 3, 3);
+    const codeContext = [codeBefore, hookLine, codeAfter].filter(Boolean).join('\n');
+
+    const apiData = {
+      source_id: sourceId,
+      file_path: filePath,
+      line_number: lineNumber,
+      api_call: apiCall,
+      namespace,
+      method,
+      code_context: codeContext.slice(0, 2000) || null,
+      content_hash: null,
+    };
+
+    apiData.content_hash = generateContentHash({
+      name: apiCall,
+      type: 'api_usage',
+      params: '',
+      docblock: '',
+      hookLine: hookLine,
+    });
+
+    apis.push(apiData);
+  }
+
+  return { hooks, blocks, apis };
+}
+
+/**
+ * Extract arguments string (handles nested parens, brackets, braces, strings, template literals).
+ */
+function extractJsArguments(content, start) {
+  let depth = 1;
+  let i = start;
+  const max = Math.min(start + 5000, content.length);
+
+  while (i < max && depth > 0) {
+    const ch = content[i];
+    if (ch === '(') depth++;
+    else if (ch === ')') depth--;
+    else if (ch === "'" || ch === '"' || ch === '`') {
+      i = skipJsString(content, i);
+    } else if (ch === '/' && i + 1 < max && content[i + 1] === '/') {
+      // Line comment — skip to end of line
+      while (i < max && content[i] !== '\n') i++;
+    } else if (ch === '/' && i + 1 < max && content[i + 1] === '*') {
+      // Block comment
+      i += 2;
+      while (i < max && !(content[i] === '*' && content[i + 1] === '/')) i++;
+      i++; // skip past /
+    }
+    if (depth > 0) i++;
+  }
+
+  if (depth !== 0) return null;
+  return content.slice(start, i);
+}
+
+function skipJsString(content, i) {
+  const quote = content[i];
+  i++;
+  while (i < content.length) {
+    if (content[i] === '\\') {
+      i += 2;
+      continue;
+    }
+    if (quote === '`' && content[i] === '$' && content[i + 1] === '{') {
+      // Template literal expression — skip nested
+      i += 2;
+      let depth = 1;
+      while (i < content.length && depth > 0) {
+        if (content[i] === '{') depth++;
+        else if (content[i] === '}') depth--;
+        if (depth > 0) i++;
+      }
+    }
+    if (content[i] === quote) return i;
+    i++;
+  }
+  return i;
+}
+
+function splitJsArguments(argsStr) {
+  const args = [];
+  let current = '';
+  let depth = 0;
+  let inString = false;
+  let stringChar = '';
+
+  for (let i = 0; i < argsStr.length; i++) {
+    const ch = argsStr[i];
+
+    if (inString) {
+      current += ch;
+      if (ch === '\\') {
+        i++;
+        if (i < argsStr.length) current += argsStr[i];
+        continue;
+      }
+      if (ch === stringChar) inString = false;
+      continue;
+    }
+
+    if (ch === "'" || ch === '"' || ch === '`') {
+      inString = true;
+      stringChar = ch;
+      current += ch;
+    } else if (ch === '(' || ch === '[' || ch === '{') {
+      depth++;
+      current += ch;
+    } else if (ch === ')' || ch === ']' || ch === '}') {
+      depth--;
+      current += ch;
+    } else if (ch === ',' && depth === 0) {
+      args.push(current);
+      current = '';
+    } else {
+      current += ch;
+    }
+  }
+
+  if (current.trim()) args.push(current);
+  return args;
+}
+
+function cleanJsHookName(raw) {
+  const trimmed = raw.trim();
+
+  // Simple quoted string
+  const simpleMatch = trimmed.match(/^['"`]([^'"`]+)['"`]$/);
+  if (simpleMatch) return simpleMatch[1];
+
+  // Template literal without expressions
+  const templateMatch = trimmed.match(/^`([^$`]+)`$/);
+  if (templateMatch) return templateMatch[1];
+
+  // Template literal with expressions
+  if (trimmed.startsWith('`')) {
+    return trimmed.replace(/`/g, '').replace(/\$\{[^}]+\}/g, '{dynamic}') || null;
+  }
+
+  // String concatenation
+  if (trimmed.includes('+')) {
+    const parts = trimmed.split(/\s*\+\s*/);
+    const cleaned = parts.map(part => {
+      const qm = part.trim().match(/^['"`]([^'"`]*)['"`]$/);
+      if (qm) return qm[1];
+      return '{dynamic}';
+    }).join('');
+    return cleaned || null;
+  }
+
+  return null;
+}
+
+function extractProperty(objStr, prop) {
+  const regex = new RegExp(`(?:['"]?${prop}['"]?)\\s*:\\s*['"\`]([^'"\`]+)['"\`]`);
+  const match = objStr.match(regex);
+  return match ? match[1] : null;
+}

package/src/indexer/parser-utils.js

@@ -0,0 +1,158 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Get 1-based line number for a character offset in content.
+ */
+export function getLineNumber(content, offset) {
+  let line = 1;
+  for (let i = 0; i < offset && i < content.length; i++) {
+    if (content[i] === '\n') line++;
+  }
+  return line;
+}
+
+/**
+ * Extract a code window around a given line.
+ * Returns { codeBefore, codeAfter, hookLine }.
+ */
+export function extractCodeWindow(lines, lineIndex, before = 8, after = 4) {
+  const start = Math.max(0, lineIndex - before);
+  const end = Math.min(lines.length - 1, lineIndex + after);
+
+  return {
+    codeBefore: lines.slice(start, lineIndex).join('\n'),
+    hookLine: lines[lineIndex] || '',
+    codeAfter: lines.slice(lineIndex + 1, end + 1).join('\n'),
+  };
+}
+
+/**
+ * Generate a content hash from the relevant parts of a hook for change detection.
+ */
+export function generateContentHash(data) {
+  const hash = createHash('sha256');
+  hash.update(JSON.stringify({
+    name: data.name,
+    type: data.type,
+    params: data.params,
+    docblock: data.docblock,
+    hookLine: data.hookLine,
+  }));
+  return hash.digest('hex').slice(0, 16);
+}
+
+/**
+ * Infer a human-readable description from hook parts.
+ */
+export function inferDescription(data) {
+  const parts = [];
+
+  const typeLabel = {
+    action: 'Action hook',
+    filter: 'Filter hook',
+    action_ref_array: 'Action hook (ref array)',
+    filter_ref_array: 'Filter hook (ref array)',
+    js_action: 'JavaScript action hook',
+    js_filter: 'JavaScript filter hook',
+  }[data.type] || 'Hook';
+
+  parts.push(typeLabel);
+
+  if (data.is_dynamic) {
+    parts.push('(dynamic name)');
+  }
+
+  parts.push(`"${data.name}"`);
+
+  if (data.php_function) {
+    parts.push(`in ${data.php_function}()`);
+  }
+  if (data.class_name) {
+    parts.push(`of class ${data.class_name}`);
+  }
+
+  if (data.param_count > 0) {
+    parts.push(`with ${data.param_count} parameter${data.param_count > 1 ? 's' : ''}`);
+  }
+
+  return parts.join(' ');
+}
+
+/**
+ * Extract docblock from lines above a given line index.
+ * Looks up to `maxLines` lines above for a closing doc comment.
+ */
+export function extractDocblock(lines, lineIndex, maxLines = 5) {
+  const docLines = [];
+  let foundEnd = false;
+
+  for (let i = lineIndex - 1; i >= Math.max(0, lineIndex - maxLines); i--) {
+    const line = lines[i].trim();
+
+    if (!foundEnd) {
+      // Look for closing */ or a @-annotated line or * continuation
+      if (line.endsWith('*/') || line.startsWith('*') || line.startsWith('/**')) {
+        foundEnd = true;
+        docLines.unshift(lines[i]);
+      } else if (line === '' || line.startsWith('//')) {
+        continue;
+      } else {
+        break;
+      }
+    } else {
+      docLines.unshift(lines[i]);
+      if (line.startsWith('/**') || line.startsWith('/*')) {
+        break;
+      }
+    }
+  }
+
+  if (docLines.length === 0) return '';
+  return docLines.join('\n').trim();
+}
+
+/**
+ * Find the enclosing function/method name for a given line index by scanning upward.
+ */
+export function findEnclosingFunction(lines, lineIndex) {
+  // PHP function pattern
+  const phpFuncRe = /(?:public|protected|private|static|\s)*function\s+(\w+)\s*\(/;
+  // JS function patterns
+  const jsFuncRe = /(?:(?:async\s+)?function\s+(\w+)|(?:const|let|var)\s+(\w+)\s*=\s*(?:async\s+)?(?:function|\(|=>))/;
+
+  let braceDepth = 0;
+  for (let i = lineIndex; i >= 0; i--) {
+    const line = lines[i];
+
+    // Count braces to track nesting
+    for (let c = line.length - 1; c >= 0; c--) {
+      if (line[c] === '}') braceDepth++;
+      if (line[c] === '{') braceDepth--;
+    }
+
+    const phpMatch = line.match(phpFuncRe);
+    if (phpMatch && braceDepth <= 0) return phpMatch[1];
+
+    const jsMatch = line.match(jsFuncRe);
+    if (jsMatch && braceDepth <= 0) return jsMatch[1] || jsMatch[2];
+  }
+  return null;
+}
+
+/**
+ * Find the enclosing class name for a given line index by scanning upward.
+ */
+export function findEnclosingClass(lines, lineIndex) {
+  const classRe = /(?:abstract\s+)?class\s+(\w+)/;
+  let braceDepth = 0;
+  for (let i = lineIndex; i >= 0; i--) {
+    const line = lines[i];
+    for (let c = line.length - 1; c >= 0; c--) {
+      if (line[c] === '}') braceDepth++;
+      if (line[c] === '{') braceDepth--;
+    }
+    const match = line.match(classRe);
+    if (match && braceDepth <= 0) return match[1];
+  }
+  return null;
+}