strapi-content-embeddings 0.1.7 → 0.1.8

package/dist/server/index.js

@@ -10,6 +10,10 @@ const types_js = require("@modelcontextprotocol/sdk/types.js");
 const zod = require("zod");
 const node_crypto = require("node:crypto");
 const streamableHttp_js = require("@modelcontextprotocol/sdk/server/streamableHttp.js");
+const htmlToText = require("html-to-text");
+const removeMarkdown = require("remove-markdown");
+const _interopDefault = (e) => e && e.__esModule ? e : { default: e };
+const removeMarkdown__default = /* @__PURE__ */ _interopDefault(removeMarkdown);
 const EMBEDDING_MODELS = {
   "text-embedding-3-small": { dimensions: 1536 },
   "text-embedding-3-large": { dimensions: 3072 },
@@ -22,7 +26,8 @@ const config = {
     embeddingModel: "text-embedding-3-small",
     chunkSize: 4e3,
     chunkOverlap: 200,
-    autoChunk: false
+    autoChunk: false,
+    preprocessContent: true
   },
   validator(config2) {
     if (!config2.openAIApiKey) {
@@ -1512,6 +1517,32 @@ const admin = [
         }
       ]
     }
+  },
+  {
+    method: "GET",
+    path: "/sync/status",
+    handler: "controller.getSyncStatus",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.read"] }
+        }
+      ]
+    }
+  },
+  {
+    method: "POST",
+    path: "/sync",
+    handler: "controller.syncFromNeon",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.update"] }
+        }
+      ]
+    }
   }
 ];
 const routes = {
@@ -1671,6 +1702,103 @@ function formatChunkTitle(baseTitle, chunkIndex, totalChunks) {
   }
   return `${baseTitle} [Part ${chunkIndex + 1}/${totalChunks}]`;
 }
+const DEFAULT_OPTIONS = {
+  stripHtml: true,
+  stripMarkdown: true,
+  normalizeWhitespace: true
+};
+function containsHtml(content) {
+  return /<[a-z][\s\S]*>/i.test(content);
+}
+function containsMarkdown(content) {
+  const markdownPatterns = [
+    /^#{1,6}\s/m,
+    // Headers: # ## ### etc
+    /\*\*[^*]+\*\*/,
+    // Bold: **text**
+    /\*[^*]+\*/,
+    // Italic: *text*
+    /__[^_]+__/,
+    // Bold: __text__
+    /_[^_]+_/,
+    // Italic: _text_
+    /\[.+\]\(.+\)/,
+    // Links: [text](url)
+    /^[-*+]\s/m,
+    // Unordered lists: - * +
+    /^\d+\.\s/m,
+    // Ordered lists: 1. 2. etc
+    /^>\s/m,
+    // Blockquotes: >
+    /`[^`]+`/,
+    // Inline code: `code`
+    /```[\s\S]*?```/,
+    // Code blocks: ```code```
+    /^\|.+\|$/m
+    // Tables: |col|col|
+  ];
+  return markdownPatterns.some((pattern) => pattern.test(content));
+}
+function stripHtml(content) {
+  if (!containsHtml(content)) {
+    return content;
+  }
+  return htmlToText.convert(content, {
+    wordwrap: false,
+    preserveNewlines: true,
+    selectors: [
+      // Convert headings to plain text with colon
+      { selector: "h1", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h2", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h3", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h4", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h5", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h6", options: { uppercase: false, trailingLineBreaks: 1 } },
+      // Remove images but keep alt text
+      { selector: "img", format: "skip" },
+      // Remove scripts and styles completely
+      { selector: "script", format: "skip" },
+      { selector: "style", format: "skip" },
+      // Keep link text, remove URLs
+      { selector: "a", options: { ignoreHref: true } }
+    ]
+  });
+}
+function stripMarkdownSyntax(content) {
+  if (!containsMarkdown(content)) {
+    return content;
+  }
+  let result = removeMarkdown__default.default(content, {
+    stripListLeaders: true,
+    listUnicodeChar: "",
+    gfm: true,
+    useImgAltText: true
+  });
+  result = result.replace(/```[\s\S]*?```/g, (match) => {
+    return match.replace(/```\w*\n?/g, "").replace(/```/g, "");
+  }).replace(/`([^`]+)`/g, "$1").replace(/^[-*_]{3,}$/gm, "");
+  return result;
+}
+function normalizeWhitespace(content) {
+  return content.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+/g, " ").split("\n").map((line) => line.trim()).join("\n").trim();
+}
+function preprocessContent(content, options2 = {}) {
+  const opts = { ...DEFAULT_OPTIONS, ...options2 };
+  if (!content || typeof content !== "string") {
+    return "";
+  }
+  let result = content;
+  if (opts.stripHtml) {
+    result = stripHtml(result);
+  }
+  if (opts.stripMarkdown) {
+    result = stripMarkdownSyntax(result);
+  }
+  if (opts.normalizeWhitespace) {
+    result = normalizeWhitespace(result);
+  }
+  return result;
+}
 const PLUGIN_ID$1 = "strapi-content-embeddings";
 const CONTENT_TYPE_UID$1 = `plugin::${PLUGIN_ID$1}.embedding`;
 const embeddings = ({ strapi }) => ({
@@ -1683,6 +1811,8 @@ const embeddings = ({ strapi }) => ({
       chunkSize: config2.chunkSize || 4e3,
       chunkOverlap: config2.chunkOverlap || 200,
       autoChunk: config2.autoChunk || false,
+      preprocessContent: config2.preprocessContent !== false,
+      // Default true
       ...config2
     };
   },
@@ -1690,8 +1820,9 @@ const embeddings = ({ strapi }) => ({
    * Create a single embedding (no chunking)
    */
   async createEmbedding(data) {
-    const { title, content, collectionType, fieldName, metadata, related, autoChunk } = data.data;
+    const { title, content: rawContent, collectionType, fieldName, metadata, related, autoChunk } = data.data;
     const config2 = this.getConfig();
+    const content = config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const shouldChunk = autoChunk ?? config2.autoChunk;
     const chunkSize = config2.chunkSize || 4e3;
     if (shouldChunk && needsChunking(content, chunkSize)) {
@@ -1741,8 +1872,9 @@ const embeddings = ({ strapi }) => ({
    * Creates multiple embedding entities, one per chunk
    */
   async createChunkedEmbedding(data) {
-    const { title, content, collectionType, fieldName, metadata, related } = data.data;
+    const { title, content: rawContent, collectionType, fieldName, metadata, related } = data.data;
     const config2 = this.getConfig();
+    const content = config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const chunkSize = config2.chunkSize || 4e3;
     const chunkOverlap = config2.chunkOverlap || 200;
     const chunks = chunkContent(content, { chunkSize, chunkOverlap });
@@ -2011,8 +2143,9 @@ const embeddings = ({ strapi }) => ({
     }
   },
   async updateEmbedding(id, data) {
-    const { title, content, metadata, autoChunk } = data.data;
+    const { title, content: rawContent, metadata, autoChunk } = data.data;
     const config2 = this.getConfig();
+    const content = rawContent !== void 0 && config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const currentEntry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
       documentId: id
     });
@@ -2028,7 +2161,9 @@ const embeddings = ({ strapi }) => ({
     const contentNeedsChunking = shouldChunk && needsChunking(newContent, chunkSize);
     const contentChanged = content !== void 0 && content !== currentEntry.content;
     if (hasRelatedChunks || contentNeedsChunking) {
-      const result = await this.updateChunkedEmbedding(id, data);
+      const result = await this.updateChunkedEmbedding(id, {
+        data: { ...data.data, content }
+      });
       return result.entity;
     }
     const updateData = {};
@@ -2121,6 +2256,7 @@ const sync = ({ strapi }) => ({
     const result = {
       success: false,
       timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      dryRun,
       neonCount: 0,
       strapiCount: 0,
       actions: {

package/dist/server/index.mjs

@@ -9,6 +9,8 @@ import { ListToolsRequestSchema, CallToolRequestSchema } from "@modelcontextprot
 import { z } from "zod";
 import { randomUUID } from "node:crypto";
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { convert } from "html-to-text";
+import removeMarkdown from "remove-markdown";
 const EMBEDDING_MODELS = {
   "text-embedding-3-small": { dimensions: 1536 },
   "text-embedding-3-large": { dimensions: 3072 },
@@ -21,7 +23,8 @@ const config = {
     embeddingModel: "text-embedding-3-small",
     chunkSize: 4e3,
     chunkOverlap: 200,
-    autoChunk: false
+    autoChunk: false,
+    preprocessContent: true
   },
   validator(config2) {
     if (!config2.openAIApiKey) {
@@ -1511,6 +1514,32 @@ const admin = [
         }
       ]
     }
+  },
+  {
+    method: "GET",
+    path: "/sync/status",
+    handler: "controller.getSyncStatus",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.read"] }
+        }
+      ]
+    }
+  },
+  {
+    method: "POST",
+    path: "/sync",
+    handler: "controller.syncFromNeon",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.update"] }
+        }
+      ]
+    }
   }
 ];
 const routes = {
@@ -1670,6 +1699,103 @@ function formatChunkTitle(baseTitle, chunkIndex, totalChunks) {
   }
   return `${baseTitle} [Part ${chunkIndex + 1}/${totalChunks}]`;
 }
+const DEFAULT_OPTIONS = {
+  stripHtml: true,
+  stripMarkdown: true,
+  normalizeWhitespace: true
+};
+function containsHtml(content) {
+  return /<[a-z][\s\S]*>/i.test(content);
+}
+function containsMarkdown(content) {
+  const markdownPatterns = [
+    /^#{1,6}\s/m,
+    // Headers: # ## ### etc
+    /\*\*[^*]+\*\*/,
+    // Bold: **text**
+    /\*[^*]+\*/,
+    // Italic: *text*
+    /__[^_]+__/,
+    // Bold: __text__
+    /_[^_]+_/,
+    // Italic: _text_
+    /\[.+\]\(.+\)/,
+    // Links: [text](url)
+    /^[-*+]\s/m,
+    // Unordered lists: - * +
+    /^\d+\.\s/m,
+    // Ordered lists: 1. 2. etc
+    /^>\s/m,
+    // Blockquotes: >
+    /`[^`]+`/,
+    // Inline code: `code`
+    /```[\s\S]*?```/,
+    // Code blocks: ```code```
+    /^\|.+\|$/m
+    // Tables: |col|col|
+  ];
+  return markdownPatterns.some((pattern) => pattern.test(content));
+}
+function stripHtml(content) {
+  if (!containsHtml(content)) {
+    return content;
+  }
+  return convert(content, {
+    wordwrap: false,
+    preserveNewlines: true,
+    selectors: [
+      // Convert headings to plain text with colon
+      { selector: "h1", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h2", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h3", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h4", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h5", options: { uppercase: false, trailingLineBreaks: 1 } },
+      { selector: "h6", options: { uppercase: false, trailingLineBreaks: 1 } },
+      // Remove images but keep alt text
+      { selector: "img", format: "skip" },
+      // Remove scripts and styles completely
+      { selector: "script", format: "skip" },
+      { selector: "style", format: "skip" },
+      // Keep link text, remove URLs
+      { selector: "a", options: { ignoreHref: true } }
+    ]
+  });
+}
+function stripMarkdownSyntax(content) {
+  if (!containsMarkdown(content)) {
+    return content;
+  }
+  let result = removeMarkdown(content, {
+    stripListLeaders: true,
+    listUnicodeChar: "",
+    gfm: true,
+    useImgAltText: true
+  });
+  result = result.replace(/```[\s\S]*?```/g, (match) => {
+    return match.replace(/```\w*\n?/g, "").replace(/```/g, "");
+  }).replace(/`([^`]+)`/g, "$1").replace(/^[-*_]{3,}$/gm, "");
+  return result;
+}
+function normalizeWhitespace(content) {
+  return content.replace(/\n{3,}/g, "\n\n").replace(/[ \t]+/g, " ").split("\n").map((line) => line.trim()).join("\n").trim();
+}
+function preprocessContent(content, options2 = {}) {
+  const opts = { ...DEFAULT_OPTIONS, ...options2 };
+  if (!content || typeof content !== "string") {
+    return "";
+  }
+  let result = content;
+  if (opts.stripHtml) {
+    result = stripHtml(result);
+  }
+  if (opts.stripMarkdown) {
+    result = stripMarkdownSyntax(result);
+  }
+  if (opts.normalizeWhitespace) {
+    result = normalizeWhitespace(result);
+  }
+  return result;
+}
 const PLUGIN_ID$1 = "strapi-content-embeddings";
 const CONTENT_TYPE_UID$1 = `plugin::${PLUGIN_ID$1}.embedding`;
 const embeddings = ({ strapi }) => ({
@@ -1682,6 +1808,8 @@ const embeddings = ({ strapi }) => ({
       chunkSize: config2.chunkSize || 4e3,
       chunkOverlap: config2.chunkOverlap || 200,
       autoChunk: config2.autoChunk || false,
+      preprocessContent: config2.preprocessContent !== false,
+      // Default true
       ...config2
     };
   },
@@ -1689,8 +1817,9 @@ const embeddings = ({ strapi }) => ({
    * Create a single embedding (no chunking)
    */
   async createEmbedding(data) {
-    const { title, content, collectionType, fieldName, metadata, related, autoChunk } = data.data;
+    const { title, content: rawContent, collectionType, fieldName, metadata, related, autoChunk } = data.data;
     const config2 = this.getConfig();
+    const content = config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const shouldChunk = autoChunk ?? config2.autoChunk;
     const chunkSize = config2.chunkSize || 4e3;
     if (shouldChunk && needsChunking(content, chunkSize)) {
@@ -1740,8 +1869,9 @@ const embeddings = ({ strapi }) => ({
    * Creates multiple embedding entities, one per chunk
    */
   async createChunkedEmbedding(data) {
-    const { title, content, collectionType, fieldName, metadata, related } = data.data;
+    const { title, content: rawContent, collectionType, fieldName, metadata, related } = data.data;
     const config2 = this.getConfig();
+    const content = config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const chunkSize = config2.chunkSize || 4e3;
     const chunkOverlap = config2.chunkOverlap || 200;
     const chunks = chunkContent(content, { chunkSize, chunkOverlap });
@@ -2010,8 +2140,9 @@ const embeddings = ({ strapi }) => ({
     }
   },
   async updateEmbedding(id, data) {
-    const { title, content, metadata, autoChunk } = data.data;
+    const { title, content: rawContent, metadata, autoChunk } = data.data;
     const config2 = this.getConfig();
+    const content = rawContent !== void 0 && config2.preprocessContent ? preprocessContent(rawContent) : rawContent;
     const currentEntry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
       documentId: id
     });
@@ -2027,7 +2158,9 @@ const embeddings = ({ strapi }) => ({
     const contentNeedsChunking = shouldChunk && needsChunking(newContent, chunkSize);
     const contentChanged = content !== void 0 && content !== currentEntry.content;
     if (hasRelatedChunks || contentNeedsChunking) {
-      const result = await this.updateChunkedEmbedding(id, data);
+      const result = await this.updateChunkedEmbedding(id, {
+        data: { ...data.data, content }
+      });
       return result.entity;
     }
     const updateData = {};
@@ -2120,6 +2253,7 @@ const sync = ({ strapi }) => ({
     const result = {
       success: false,
       timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      dryRun,
       neonCount: 0,
       strapiCount: 0,
       actions: {

package/dist/server/src/config/index.d.ts

@@ -20,6 +20,8 @@ export interface PluginConfigSchema {
     chunkOverlap?: number;
     /** Automatically chunk content that exceeds chunkSize (default: false) */
     autoChunk?: boolean;
+    /** Preprocess content before embedding - strips HTML/Markdown (default: true) */
+    preprocessContent?: boolean;
 }
 declare const _default: {
     default: {
@@ -29,6 +31,7 @@ declare const _default: {
         chunkSize: number;
         chunkOverlap: number;
         autoChunk: boolean;
+        preprocessContent: boolean;
     };
     validator(config: PluginConfigSchema): void;
 };

package/dist/server/src/index.d.ts

@@ -16,6 +16,7 @@ declare const _default: {
             chunkSize: number;
             chunkOverlap: number;
             autoChunk: boolean;
+            preprocessContent: boolean;
         };
         validator(config: import("./config").PluginConfigSchema): void;
     };

package/dist/server/src/services/sync.d.ts

@@ -2,6 +2,7 @@ import type { Core } from "@strapi/strapi";
 export interface SyncResult {
     success: boolean;
     timestamp: string;
+    dryRun: boolean;
     neonCount: number;
     strapiCount: number;
     actions: {

package/dist/server/src/utils/preprocessing.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Content preprocessing utilities for cleaning text before embedding
+ * Handles HTML stripping, markdown conversion, and text normalization
+ */
+export interface PreprocessOptions {
+    /** Strip HTML tags from content (default: true) */
+    stripHtml?: boolean;
+    /** Strip markdown syntax from content (default: true) */
+    stripMarkdown?: boolean;
+    /** Normalize whitespace (collapse multiple spaces/newlines) (default: true) */
+    normalizeWhitespace?: boolean;
+}
+/**
+ * Preprocess content for embedding
+ * Strips HTML, markdown, and normalizes whitespace
+ *
+ * @param content - The raw content to preprocess
+ * @param options - Preprocessing options
+ * @returns Cleaned plain text ready for embedding
+ */
+export declare function preprocessContent(content: string, options?: PreprocessOptions): string;
+/**
+ * Check if content needs preprocessing
+ * Returns true if content contains HTML or Markdown
+ */
+export declare function needsPreprocessing(content: string): boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strapi-content-embeddings",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Strapi v5 plugin for vector embeddings with OpenAI and Neon PostgreSQL. Enables semantic search, RAG chat, and MCP (Model Context Protocol) integration.",
   "keywords": [
     "strapi",
@@ -65,8 +65,10 @@
     "@modelcontextprotocol/sdk": "^1.12.0",
     "@strapi/design-system": "^2.0.0-rc.12",
     "@strapi/icons": "^2.0.0-rc.12",
+    "html-to-text": "^9.0.5",
     "langchain": "^1.2.4",
     "pg": "^8.13.1",
+    "remove-markdown": "^0.5.5",
     "qs": "^6.13.1",
     "react-intl": "^6.8.4",
     "react-markdown": "^10.1.0",