wp-devdocs-mcp 1.1.1

package/src/docs/doc-index-manager.js

@@ -0,0 +1,128 @@
+import fg from 'fast-glob';
+import { readFileSync, statSync } from 'node:fs';
+import { createHash } from 'node:crypto';
+import {
+  upsertDoc,
+  markDocsRemoved,
+  getIndexedFile,
+  upsertIndexedFile,
+  getActiveDocId,
+} from '../db/sqlite.js';
+import { DocParserRegistry, extractFrontmatter } from './parsers/base-doc-parser.js';
+import { BlockEditorParser } from './parsers/block-editor-parser.js';
+import { PluginHandbookParser } from './parsers/plugin-handbook-parser.js';
+import { RestApiParser } from './parsers/rest-api-parser.js';
+import { WpCliParser } from './parsers/wp-cli-parser.js';
+import { AdminHandbookParser } from './parsers/admin-handbook-parser.js';
+import { GeneralDocParser } from './parsers/general-doc-parser.js';
+
+const MD_PATTERNS = ['**/*.md'];
+
+const IGNORE_PATTERNS = [
+  '**/node_modules/**',
+  '**/.git/**',
+  '**/vendor/**',
+  '**/images/**',
+  '**/img/**',
+  '**/assets/**',
+  '**/static/**',
+  '**/CHANGELOG.md',
+  '**/changelog.md',
+  '**/CODE_OF_CONDUCT.md',
+  '**/CONTRIBUTING.md',
+  '**/LICENSE.md',
+];
+
+// Build the parser registry — specific parsers first, general fallback last
+const registry = new DocParserRegistry();
+registry.register(new BlockEditorParser());
+registry.register(new PluginHandbookParser());
+registry.register(new RestApiParser());
+registry.register(new WpCliParser());
+registry.register(new AdminHandbookParser());
+registry.register(new GeneralDocParser());
+
+/**
+ * Index markdown documentation from a docs-type source.
+ * Mirrors the pattern of indexSource() in index-manager.js.
+ */
+export async function indexDocsSource(source, localPath, force, stats) {
+  const files = await fg(MD_PATTERNS, {
+    cwd: localPath,
+    ignore: IGNORE_PATTERNS,
+    absolute: false,
+    onlyFiles: true,
+  });
+
+  console.error(`Found ${files.length} markdown files to check in ${source.name}`);
+
+  const activeDocIds = [];
+
+  for (const file of files) {
+    const fullPath = `${localPath}/${file}`;
+
+    try {
+      const fileStat = statSync(fullPath);
+      const mtimeMs = fileStat.mtimeMs;
+
+      // Cache indexed file lookup (used for both mtime and content hash checks)
+      const indexed = force ? null : getIndexedFile(source.id, file);
+
+      // Check mtime for incremental skip
+      if (indexed && indexed.mtime_ms === mtimeMs) {
+        const docId = getActiveDocId(source.id, file);
+        if (docId) activeDocIds.push(docId);
+        stats.files_skipped++;
+        continue;
+      }
+
+      const content = readFileSync(fullPath, 'utf-8');
+      const contentHash = createHash('sha256').update(content).digest('hex').slice(0, 16);
+
+      // Skip if content hash matches
+      if (indexed && indexed.content_hash === contentHash) {
+        const docId = getActiveDocId(source.id, file);
+        if (docId) activeDocIds.push(docId);
+        upsertIndexedFile(source.id, file, mtimeMs, contentHash);
+        stats.files_skipped++;
+        continue;
+      }
+
+      // Extract frontmatter for parser selection
+      const { frontmatter } = extractFrontmatter(content);
+
+      // Select parser
+      const parser = registry.getParser(file, frontmatter, source.name);
+      if (!parser) {
+        // Should never happen with GeneralDocParser as fallback
+        continue;
+      }
+
+      // Parse
+      const docData = parser.parse(content, file, source.id);
+      if (!docData || !docData.title) {
+        continue;
+      }
+
+      // Upsert
+      const result = upsertDoc(docData);
+      activeDocIds.push(result.id);
+
+      if (result.action === 'inserted') stats.docs_inserted++;
+      else if (result.action === 'updated') stats.docs_updated++;
+      else stats.docs_skipped++;
+
+      // Track file
+      upsertIndexedFile(source.id, file, mtimeMs, contentHash);
+      stats.files_processed++;
+    } catch (err) {
+      const msg = `Error indexing doc ${file}: ${err.message}`;
+      console.error(msg);
+      stats.errors.push(msg);
+    }
+  }
+
+  // Soft-delete docs no longer found in this source
+  const removed = markDocsRemoved(source.id, activeDocIds);
+  stats.docs_removed += removed;
+}

package/src/docs/parsers/admin-handbook-parser.js

@@ -0,0 +1,48 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+export class AdminHandbookParser extends BaseDocParser {
+  canParse(filePath, frontmatter, sourceName) {
+    return sourceName.includes('admin-handbook') || sourceName === 'admin';
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 1 ? pathParts[0] : null;
+
+    // Extract config snippets (wp-config.php defines, .htaccess, nginx)
+    const configSnippets = [];
+    const defineRegex = /define\s*\(\s*['"](\w+)['"]/g;
+    let match;
+    while ((match = defineRegex.exec(body)) !== null) {
+      if (!configSnippets.includes(match[1])) {
+        configSnippets.push(match[1]);
+      }
+    }
+
+    const metadata = {};
+    if (configSnippets.length > 0) metadata.config_defines = configSnippets;
+    if (Object.keys(frontmatter).length > 0) metadata.frontmatter = frontmatter;
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: this.inferDocType(body, frontmatter),
+      category: 'admin',
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata: Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}

package/src/docs/parsers/base-doc-parser.js

@@ -0,0 +1,175 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * Base class for document parsers. Each parser handles a specific type of
+ * WordPress documentation (handbook, REST API reference, WP-CLI, etc.).
+ */
+export class BaseDocParser {
+  /**
+   * Check if this parser can handle the given file.
+   * @param {string} filePath - Relative file path
+   * @param {object} frontmatter - Parsed frontmatter data
+   * @param {string} sourceName - The source name from DB
+   * @returns {boolean}
+   */
+  canParse(filePath, frontmatter, sourceName) {
+    throw new Error('canParse() must be implemented by subclass');
+  }
+
+  /**
+   * Parse a markdown file into a structured doc object.
+   * @param {string} content - Raw file content
+   * @param {string} filePath - Relative file path
+   * @param {number} sourceId - Source ID from DB
+   * @returns {object} Parsed doc data ready for upsertDoc()
+   */
+  parse(content, filePath, sourceId) {
+    throw new Error('parse() must be implemented by subclass');
+  }
+
+  // --- Shared utilities ---
+
+  extractFrontmatter(content) {
+    const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+    if (!match) return { frontmatter: {}, body: content };
+
+    const raw = match[1];
+    const frontmatter = {};
+    for (const line of raw.split('\n')) {
+      const sep = line.indexOf(':');
+      if (sep === -1) continue;
+      const key = line.slice(0, sep).trim();
+      const val = line.slice(sep + 1).trim().replace(/^["']|["']$/g, '');
+      if (key) frontmatter[key] = val;
+    }
+
+    const body = content.slice(match[0].length).trim();
+    return { frontmatter, body };
+  }
+
+  extractTitle(body, frontmatter) {
+    if (frontmatter.title) return frontmatter.title;
+    const h1 = body.match(/^#\s+(.+)/m);
+    if (h1) return h1[1].trim();
+    return null;
+  }
+
+  extractDescription(body) {
+    // First non-heading, non-empty paragraph
+    const lines = body.split('\n');
+    let collecting = false;
+    const desc = [];
+
+    for (const line of lines) {
+      const trimmed = line.trim();
+      if (!trimmed) {
+        if (collecting) break;
+        continue;
+      }
+      if (trimmed.startsWith('#') || trimmed.startsWith('---') || trimmed.startsWith('```')) {
+        if (collecting) break;
+        continue;
+      }
+      collecting = true;
+      desc.push(trimmed);
+    }
+
+    const result = desc.join(' ').slice(0, 500);
+    return result || null;
+  }
+
+  extractCodeExamples(body) {
+    const examples = [];
+    const regex = /```(\w*)\n([\s\S]*?)```/g;
+    let match;
+    while ((match = regex.exec(body)) !== null) {
+      examples.push({
+        language: match[1] || 'text',
+        code: match[2].trim(),
+      });
+    }
+    return examples.length > 0 ? JSON.stringify(examples) : null;
+  }
+
+  generateSlug(filePath) {
+    return filePath
+      .toLowerCase()
+      .replace(/\.md$/i, '')
+      .replace(/\\/g, '/')
+      .replace(/[^a-z0-9/_-]/g, '-')
+      .replace(/-+/g, '-')
+      .replace(/\//g, '--')
+      .replace(/^-|-$/g, '');
+  }
+
+  generateContentHash(data) {
+    const hash = createHash('sha256');
+    hash.update(JSON.stringify({
+      title: data.title,
+      content: data.content,
+      doc_type: data.doc_type,
+      category: data.category,
+    }));
+    return hash.digest('hex').slice(0, 16);
+  }
+
+  inferDocType(body, frontmatter) {
+    const text = (body + ' ' + (frontmatter.title || '')).toLowerCase();
+
+    if (frontmatter.doc_type) return frontmatter.doc_type;
+
+    if (/\breference\b/.test(text) || /\bapi\b/.test(text)) return 'reference';
+    if (/\btutorial\b/.test(text) || /\bstep[- ]by[- ]step\b/.test(text)) return 'tutorial';
+    if (/\bhow[- ]to\b/.test(text)) return 'howto';
+    if (/\bfaq\b/i.test(text) || /\bfrequently asked\b/.test(text)) return 'faq';
+    if (/\bguide\b/.test(text)) return 'guide';
+
+    return 'general';
+  }
+}
+
+/**
+ * Standalone frontmatter extraction — usable without instantiating a parser.
+ * @param {string} content - Raw file content
+ * @returns {{ frontmatter: object, body: string }}
+ */
+export function extractFrontmatter(content) {
+  const match = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  if (!match) return { frontmatter: {}, body: content };
+
+  const raw = match[1];
+  const frontmatter = {};
+  for (const line of raw.split('\n')) {
+    const sep = line.indexOf(':');
+    if (sep === -1) continue;
+    const key = line.slice(0, sep).trim();
+    const val = line.slice(sep + 1).trim().replace(/^["']|["']$/g, '');
+    if (key) frontmatter[key] = val;
+  }
+
+  const body = content.slice(match[0].length).trim();
+  return { frontmatter, body };
+}
+
+/**
+ * Registry that selects the appropriate parser for a given file.
+ * Parsers are checked in order — first match wins.
+ */
+export class DocParserRegistry {
+  constructor() {
+    this.parsers = [];
+  }
+
+  register(parser) {
+    this.parsers.push(parser);
+  }
+
+  getParser(filePath, frontmatter, sourceName) {
+    for (const parser of this.parsers) {
+      if (parser.canParse(filePath, frontmatter, sourceName)) {
+        return parser;
+      }
+    }
+    return null;
+  }
+}

package/src/docs/parsers/block-editor-parser.js

@@ -0,0 +1,49 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+export class BlockEditorParser extends BaseDocParser {
+  canParse(filePath, frontmatter, sourceName) {
+    return sourceName.includes('gutenberg') && filePath.startsWith('docs/');
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    // Infer subcategory from path: docs/getting-started/..., docs/reference-guides/..., etc.
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 2 ? pathParts[1] : null;
+
+    // Extract @wordpress/ package references
+    const packageRefs = [];
+    const pkgRegex = /@wordpress\/[\w-]+/g;
+    let match;
+    while ((match = pkgRegex.exec(body)) !== null) {
+      if (!packageRefs.includes(match[0])) {
+        packageRefs.push(match[0]);
+      }
+    }
+
+    const metadata = {};
+    if (packageRefs.length > 0) metadata.package_refs = packageRefs;
+    if (Object.keys(frontmatter).length > 0) metadata.frontmatter = frontmatter;
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: this.inferDocType(body, frontmatter),
+      category: 'block-editor',
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata: Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}

package/src/docs/parsers/general-doc-parser.js

@@ -0,0 +1,53 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+/**
+ * Fallback parser for any markdown doc that doesn't match a specialized parser.
+ * Always returns true from canParse() — must be registered last in the registry.
+ */
+export class GeneralDocParser extends BaseDocParser {
+  canParse() {
+    return true;
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 1 ? pathParts[0] : null;
+
+    // Try to infer category from path or frontmatter
+    let category = frontmatter.category || null;
+    if (!category) {
+      const pathLower = filePath.toLowerCase();
+      if (pathLower.includes('block') || pathLower.includes('gutenberg')) category = 'block-editor';
+      else if (pathLower.includes('plugin')) category = 'plugins';
+      else if (pathLower.includes('rest') || pathLower.includes('api')) category = 'rest-api';
+      else if (pathLower.includes('cli')) category = 'wp-cli';
+      else if (pathLower.includes('admin')) category = 'admin';
+    }
+
+    const metadata = Object.keys(frontmatter).length > 0
+      ? JSON.stringify({ frontmatter })
+      : null;
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: this.inferDocType(body, frontmatter),
+      category,
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}

package/src/docs/parsers/plugin-handbook-parser.js

@@ -0,0 +1,60 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+export class PluginHandbookParser extends BaseDocParser {
+  canParse(filePath, frontmatter, sourceName) {
+    return sourceName.includes('plugin-handbook') || sourceName === 'plugin';
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    // Infer subcategory from directory structure
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 1 ? pathParts[0] : null;
+
+    // Extract WP function references
+    const funcRefs = [];
+    const funcRegex = /\b(wp_\w+|get_\w+|add_\w+|remove_\w+|do_action|apply_filters|register_\w+)\s*\(/g;
+    let match;
+    while ((match = funcRegex.exec(body)) !== null) {
+      if (!funcRefs.includes(match[1])) {
+        funcRefs.push(match[1]);
+      }
+    }
+
+    // Extract hook names mentioned in content
+    const hookRefs = [];
+    const hookRegex = /['"`]([a-z_]+(?:\{[^}]*\})?[a-z_]*)['"`]\s*(?:,|\))/g;
+    while ((match = hookRegex.exec(body)) !== null) {
+      const name = match[1];
+      if (name.includes('_') && name.length > 3 && !hookRefs.includes(name)) {
+        hookRefs.push(name);
+      }
+    }
+
+    const metadata = {};
+    if (funcRefs.length > 0) metadata.function_refs = funcRefs;
+    if (hookRefs.length > 0) metadata.hook_refs = hookRefs;
+    if (Object.keys(frontmatter).length > 0) metadata.frontmatter = frontmatter;
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: this.inferDocType(body, frontmatter),
+      category: 'plugins',
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata: Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}

package/src/docs/parsers/rest-api-parser.js

@@ -0,0 +1,57 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+export class RestApiParser extends BaseDocParser {
+  canParse(filePath, frontmatter, sourceName) {
+    return sourceName.includes('rest-api') || sourceName.includes('wp-api');
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 1 ? pathParts[0] : null;
+
+    // Extract endpoint definitions (method + route)
+    const endpoints = [];
+    const endpointRegex = /\b(GET|POST|PUT|PATCH|DELETE)\s+(`?\/wp\/v2\/[\w\/-{}]+`?|`?\/wp-json\/[\w\/-{}]+`?)/g;
+    let match;
+    while ((match = endpointRegex.exec(body)) !== null) {
+      endpoints.push({ method: match[1], route: match[2].replace(/`/g, '') });
+    }
+
+    // Also look for route definitions in code blocks
+    const routeRegex = /['"]\/wp\/v2\/[\w\/-{}]+['"]/g;
+    while ((match = routeRegex.exec(body)) !== null) {
+      const route = match[0].replace(/['"]/g, '');
+      if (!endpoints.find(e => e.route === route)) {
+        endpoints.push({ method: 'ANY', route });
+      }
+    }
+
+    const metadata = {};
+    if (endpoints.length > 0) metadata.endpoints = endpoints;
+    if (Object.keys(frontmatter).length > 0) metadata.frontmatter = frontmatter;
+
+    const docType = endpoints.length > 0 ? 'api' : this.inferDocType(body, frontmatter);
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: docType,
+      category: 'rest-api',
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata: Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}

package/src/docs/parsers/wp-cli-parser.js

@@ -0,0 +1,61 @@
+import { BaseDocParser } from './base-doc-parser.js';
+
+export class WpCliParser extends BaseDocParser {
+  canParse(filePath, frontmatter, sourceName) {
+    return sourceName.includes('wp-cli');
+  }
+
+  parse(content, filePath, sourceId) {
+    const { frontmatter, body } = this.extractFrontmatter(content);
+    const title = this.extractTitle(body, frontmatter) || filePath;
+    const description = this.extractDescription(body);
+    const codeExamples = this.extractCodeExamples(body);
+
+    const pathParts = filePath.split('/');
+    const subcategory = pathParts.length > 1 ? pathParts[0] : null;
+
+    // Extract command signatures like "wp <command> <subcommand> [--flag]"
+    const commands = [];
+    const cmdRegex = /(?:^|\n)\s*(?:#+\s*)?`?(wp\s+[\w-]+(?:\s+[\w-]+)?)`?/g;
+    let match;
+    while ((match = cmdRegex.exec(body)) !== null) {
+      const cmd = match[1].trim();
+      if (!commands.includes(cmd)) {
+        commands.push(cmd);
+      }
+    }
+
+    // Extract options/flags like --option=<value>
+    const options = [];
+    const optRegex = /--[\w-]+(?:=<[^>]+>)?/g;
+    while ((match = optRegex.exec(body)) !== null) {
+      if (!options.includes(match[0])) {
+        options.push(match[0]);
+      }
+    }
+
+    const metadata = {};
+    if (commands.length > 0) metadata.commands = commands;
+    if (options.length > 0) metadata.options = options;
+    if (Object.keys(frontmatter).length > 0) metadata.frontmatter = frontmatter;
+
+    const docType = commands.length > 0 ? 'reference' : this.inferDocType(body, frontmatter);
+
+    const doc = {
+      source_id: sourceId,
+      file_path: filePath,
+      slug: this.generateSlug(filePath),
+      title,
+      doc_type: docType,
+      category: 'wp-cli',
+      subcategory,
+      description,
+      content: body,
+      code_examples: codeExamples,
+      metadata: Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null,
+    };
+
+    doc.content_hash = this.generateContentHash(doc);
+    return doc;
+  }
+}