strapi-content-embeddings 0.1.3 → 0.1.5

package/dist/server/index.mjs

@@ -19,7 +19,10 @@ const config = {
   default: {
     openAIApiKey: "",
     neonConnectionString: "",
-    embeddingModel: "text-embedding-3-small"
+    embeddingModel: "text-embedding-3-small",
+    chunkSize: 4e3,
+    chunkOverlap: 200,
+    autoChunk: false
   },
   validator(config2) {
     if (!config2.openAIApiKey) {
@@ -37,6 +40,11 @@ const config = {
         `strapi-content-embeddings: Invalid embeddingModel "${config2.embeddingModel}". Valid options: ${Object.keys(EMBEDDING_MODELS).join(", ")}. Defaulting to "text-embedding-3-small".`
       );
     }
+    if (config2.chunkSize && (config2.chunkSize < 100 || config2.chunkSize > 8e3)) {
+      console.warn(
+        `strapi-content-embeddings: chunkSize ${config2.chunkSize} is outside recommended range (100-8000). Using default value of 4000.`
+      );
+    }
   }
 };
 class PluginManager {
@@ -281,6 +289,56 @@ Context:
   isInitialized() {
     return !!(this.embeddings && this.chat && this.pool);
   }
+  /**
+   * Get all embeddings from Neon DB
+   * Returns the metadata (including Strapi documentId) for each embedding
+   */
+  async getAllNeonEmbeddings() {
+    if (!this.pool) {
+      throw new Error("Plugin manager not initialized");
+    }
+    try {
+      const result = await this.pool.query(`
+        SELECT
+          id,
+          content,
+          metadata->>'id' as strapi_id,
+          metadata->>'title' as title,
+          metadata->>'collectionType' as collection_type,
+          metadata->>'fieldName' as field_name
+        FROM embeddings_documents
+        ORDER BY id
+      `);
+      return result.rows.map((row) => ({
+        id: row.id,
+        strapiId: row.strapi_id,
+        title: row.title || "",
+        content: row.content || "",
+        collectionType: row.collection_type || "standalone",
+        fieldName: row.field_name || "content"
+      }));
+    } catch (error) {
+      console.error(`Failed to get Neon embeddings: ${error}`);
+      throw new Error(`Failed to get Neon embeddings: ${error}`);
+    }
+  }
+  /**
+   * Delete an embedding from Neon by its Neon UUID (not Strapi ID)
+   */
+  async deleteNeonEmbeddingById(neonId) {
+    if (!this.pool) {
+      throw new Error("Plugin manager not initialized");
+    }
+    try {
+      await this.pool.query(
+        `DELETE FROM embeddings_documents WHERE id = $1`,
+        [neonId]
+      );
+    } catch (error) {
+      console.error(`Failed to delete Neon embedding: ${error}`);
+      throw new Error(`Failed to delete Neon embedding: ${error}`);
+    }
+  }
   async destroy() {
     if (this.pool) {
       await this.pool.end();
@@ -580,7 +638,7 @@ async function handleGetEmbedding(strapi, args) {
 }
 const createEmbeddingTool = {
   name: "create_embedding",
-  description: "Create a new embedding from text content. The content will be vectorized and stored for semantic search.",
+  description: "Create a new embedding from text content. The content will be vectorized and stored for semantic search. For large content (over 4000 characters), enable autoChunk to automatically split into multiple embeddings.",
   inputSchema: {
     type: "object",
     properties: {
@@ -595,21 +653,68 @@ const createEmbeddingTool = {
       metadata: {
         type: "object",
         description: "Optional metadata to associate with the embedding (tags, source, etc.)"
+      },
+      autoChunk: {
+        type: "boolean",
+        description: "Automatically split large content into chunks (default: false). When enabled, content over 4000 characters will be split into multiple embeddings with overlap for context preservation."
       }
     },
     required: ["title", "content"]
   }
 };
 async function handleCreateEmbedding(strapi, args) {
-  const { title, content, metadata } = args;
+  const { title, content, metadata, autoChunk } = args;
   try {
     const embeddingsService = strapi.plugin("strapi-content-embeddings").service("embeddings");
+    if (autoChunk) {
+      const result = await embeddingsService.createChunkedEmbedding({
+        data: {
+          title,
+          content,
+          metadata: metadata || {},
+          collectionType: "standalone",
+          fieldName: "content"
+        }
+      });
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(
+              {
+                success: true,
+                message: result.wasChunked ? `Content chunked into ${result.totalChunks} embeddings` : "Embedding created successfully (no chunking needed)",
+                wasChunked: result.wasChunked,
+                totalChunks: result.totalChunks,
+                primaryEmbedding: {
+                  id: result.entity.id,
+                  documentId: result.entity.documentId,
+                  title: result.entity.title,
+                  embeddingId: result.entity.embeddingId
+                },
+                chunks: result.chunks.map((chunk) => ({
+                  documentId: chunk.documentId,
+                  title: chunk.title,
+                  contentLength: chunk.content?.length || 0
+                })),
+                contentLength: content.length,
+                estimatedTokens: Math.ceil(content.length / 4)
+              },
+              null,
+              2
+            )
+          }
+        ]
+      };
+    }
     const embedding2 = await embeddingsService.createEmbedding({
-      title,
-      content,
-      metadata: metadata || {},
-      collectionType: "standalone",
-      fieldName: "content"
+      data: {
+        title,
+        content,
+        metadata: metadata || {},
+        collectionType: "standalone",
+        fieldName: "content"
+      }
     });
     return {
       content: [
@@ -627,7 +732,8 @@ async function handleCreateEmbedding(strapi, args) {
                 contentLength: content.length,
                 metadata: embedding2.metadata,
                 createdAt: embedding2.createdAt
-              }
+              },
+              hint: content.length > 4e3 ? "Content is large. Consider using autoChunk: true for better search results." : void 0
             },
             null,
             2
@@ -713,10 +819,10 @@ function createMcpServer(strapi) {
   });
   return server;
 }
-const PLUGIN_ID$4 = "strapi-content-embeddings";
+const PLUGIN_ID$5 = "strapi-content-embeddings";
 const OAUTH_PLUGIN_ID = "strapi-oauth-mcp-manager";
 function createFallbackAuthMiddleware(strapi) {
-  const mcpPath = `/api/${PLUGIN_ID$4}/mcp`;
+  const mcpPath = `/api/${PLUGIN_ID$5}/mcp`;
   return async (ctx, next) => {
     if (!ctx.path.startsWith(mcpPath)) {
       return next();
@@ -742,35 +848,35 @@ const bootstrap = async ({ strapi }) => {
       section: "plugins",
       displayName: "Read",
       uid: "read",
-      pluginName: PLUGIN_ID$4
+      pluginName: PLUGIN_ID$5
     },
     {
       section: "plugins",
       displayName: "Update",
       uid: "update",
-      pluginName: PLUGIN_ID$4
+      pluginName: PLUGIN_ID$5
     },
     {
       section: "plugins",
       displayName: "Create",
       uid: "create",
-      pluginName: PLUGIN_ID$4
+      pluginName: PLUGIN_ID$5
     },
     {
       section: "plugins",
       displayName: "Delete",
       uid: "delete",
-      pluginName: PLUGIN_ID$4
+      pluginName: PLUGIN_ID$5
     },
     {
       section: "plugins",
       displayName: "Chat",
       uid: "chat",
-      pluginName: PLUGIN_ID$4
+      pluginName: PLUGIN_ID$5
     }
   ];
   await strapi.admin.services.permission.actionProvider.registerMany(actions);
-  const pluginConfig = strapi.config.get(`plugin::${PLUGIN_ID$4}`);
+  const pluginConfig = strapi.config.get(`plugin::${PLUGIN_ID$5}`);
   if (pluginConfig?.openAIApiKey && pluginConfig?.neonConnectionString) {
     try {
       await pluginManager.initialize({
@@ -779,42 +885,42 @@ const bootstrap = async ({ strapi }) => {
         embeddingModel: pluginConfig.embeddingModel
       });
       strapi.contentEmbeddingsManager = pluginManager;
-      strapi.log.info(`[${PLUGIN_ID$4}] Plugin initialized successfully`);
+      strapi.log.info(`[${PLUGIN_ID$5}] Plugin initialized successfully`);
     } catch (error) {
-      strapi.log.error(`[${PLUGIN_ID$4}] Failed to initialize:`, error);
+      strapi.log.error(`[${PLUGIN_ID$5}] Failed to initialize:`, error);
     }
   } else {
     strapi.log.warn(
-      `[${PLUGIN_ID$4}] Missing configuration. Set openAIApiKey and neonConnectionString in plugin config.`
+      `[${PLUGIN_ID$5}] Missing configuration. Set openAIApiKey and neonConnectionString in plugin config.`
     );
   }
-  const plugin = strapi.plugin(PLUGIN_ID$4);
+  const plugin = strapi.plugin(PLUGIN_ID$5);
   plugin.createMcpServer = () => createMcpServer(strapi);
   plugin.sessions = /* @__PURE__ */ new Map();
   const oauthPlugin = strapi.plugin(OAUTH_PLUGIN_ID);
   if (oauthPlugin) {
-    strapi.log.info(`[${PLUGIN_ID$4}] OAuth manager detected - OAuth + API token auth enabled`);
+    strapi.log.info(`[${PLUGIN_ID$5}] OAuth manager detected - OAuth + API token auth enabled`);
   } else {
     const fallbackMiddleware = createFallbackAuthMiddleware();
     strapi.server.use(fallbackMiddleware);
-    strapi.log.info(`[${PLUGIN_ID$4}] Using API token authentication (OAuth manager not installed)`);
+    strapi.log.info(`[${PLUGIN_ID$5}] Using API token authentication (OAuth manager not installed)`);
   }
-  strapi.log.info(`[${PLUGIN_ID$4}] MCP endpoint available at: /api/${PLUGIN_ID$4}/mcp`);
+  strapi.log.info(`[${PLUGIN_ID$5}] MCP endpoint available at: /api/${PLUGIN_ID$5}/mcp`);
 };
 const destroy = async ({ strapi }) => {
   await pluginManager.destroy();
   console.log("Content Embeddings plugin destroyed");
 };
-const PLUGIN_ID$3 = "strapi-content-embeddings";
+const PLUGIN_ID$4 = "strapi-content-embeddings";
 const register = ({ strapi }) => {
   Object.values(strapi.contentTypes).forEach((contentType) => {
-    if (contentType.uid.startsWith("admin::") || contentType.uid.startsWith("strapi::") || contentType.uid === `plugin::${PLUGIN_ID$3}.embedding`) {
+    if (contentType.uid.startsWith("admin::") || contentType.uid.startsWith("strapi::") || contentType.uid === `plugin::${PLUGIN_ID$4}.embedding`) {
       return;
     }
     contentType.attributes.embedding = {
       type: "relation",
       relation: "morphOne",
-      target: `plugin::${PLUGIN_ID$3}.embedding`,
+      target: `plugin::${PLUGIN_ID$4}.embedding`,
       morphBy: "related",
       private: false,
       configurable: false
@@ -884,11 +990,11 @@ const embedding = {
 const contentTypes = {
   embedding
 };
-const PLUGIN_ID$2 = "strapi-content-embeddings";
+const PLUGIN_ID$3 = "strapi-content-embeddings";
 const controller = ({ strapi }) => ({
   async createEmbedding(ctx) {
     try {
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").createEmbedding(ctx.request.body);
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").createEmbedding(ctx.request.body);
       ctx.body = result;
     } catch (error) {
       ctx.throw(500, error.message || "Failed to create embedding");
@@ -897,7 +1003,7 @@ const controller = ({ strapi }) => ({
   async deleteEmbedding(ctx) {
     try {
       const { id } = ctx.params;
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").deleteEmbedding(id);
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").deleteEmbedding(id);
       ctx.body = result;
     } catch (error) {
       ctx.throw(500, error.message || "Failed to delete embedding");
@@ -906,7 +1012,7 @@ const controller = ({ strapi }) => ({
   async updateEmbedding(ctx) {
     try {
       const { id } = ctx.params;
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").updateEmbedding(id, ctx.request.body);
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").updateEmbedding(id, ctx.request.body);
       ctx.body = result;
     } catch (error) {
       ctx.throw(500, error.message || "Failed to update embedding");
@@ -915,7 +1021,7 @@ const controller = ({ strapi }) => ({
   async getEmbeddings(ctx) {
     try {
       const { page, pageSize, filters } = ctx.query;
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").getEmbeddings({
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").getEmbeddings({
         page: page ? parseInt(page, 10) : 1,
         pageSize: pageSize ? parseInt(pageSize, 10) : 10,
         filters
@@ -928,7 +1034,7 @@ const controller = ({ strapi }) => ({
   async getEmbedding(ctx) {
     try {
       const { id } = ctx.params;
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").getEmbedding(id);
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").getEmbedding(id);
       if (!result) {
         ctx.throw(404, "Embedding not found");
       }
@@ -943,14 +1049,46 @@ const controller = ({ strapi }) => ({
   async queryEmbeddings(ctx) {
     try {
       const { query } = ctx.query;
-      const result = await strapi.plugin(PLUGIN_ID$2).service("embeddings").queryEmbeddings(query);
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").queryEmbeddings(query);
       ctx.body = result;
     } catch (error) {
       ctx.throw(500, error.message || "Failed to query embeddings");
     }
+  },
+  /**
+   * Sync embeddings from Neon DB to Strapi DB
+   * GET /api/strapi-content-embeddings/sync
+   *
+   * Query params:
+   * - removeOrphans: boolean (default: false) - Remove Strapi entries that don't exist in Neon
+   * - dryRun: boolean (default: false) - Preview changes without applying them
+   */
+  async syncFromNeon(ctx) {
+    try {
+      const { removeOrphans, dryRun } = ctx.query;
+      const result = await strapi.plugin(PLUGIN_ID$3).service("sync").syncFromNeon({
+        removeOrphans: removeOrphans === "true",
+        dryRun: dryRun === "true"
+      });
+      ctx.body = result;
+    } catch (error) {
+      ctx.throw(500, error.message || "Failed to sync embeddings");
+    }
+  },
+  /**
+   * Get sync status - compare Neon and Strapi without making changes
+   * GET /api/strapi-content-embeddings/sync/status
+   */
+  async getSyncStatus(ctx) {
+    try {
+      const result = await strapi.plugin(PLUGIN_ID$3).service("sync").getSyncStatus();
+      ctx.body = result;
+    } catch (error) {
+      ctx.throw(500, error.message || "Failed to get sync status");
+    }
   }
 });
-const PLUGIN_ID$1 = "strapi-content-embeddings";
+const PLUGIN_ID$2 = "strapi-content-embeddings";
 const SESSION_TIMEOUT_MS = 4 * 60 * 60 * 1e3;
 function isSessionExpired(session) {
   return Date.now() - session.createdAt > SESSION_TIMEOUT_MS;
@@ -968,7 +1106,7 @@ function cleanupExpiredSessions(plugin, strapi) {
     }
   }
   if (cleaned > 0) {
-    strapi.log.debug(`[${PLUGIN_ID$1}] Cleaned up ${cleaned} expired MCP sessions`);
+    strapi.log.debug(`[${PLUGIN_ID$2}] Cleaned up ${cleaned} expired MCP sessions`);
   }
 }
 const mcpController = ({ strapi }) => ({
@@ -976,7 +1114,7 @@ const mcpController = ({ strapi }) => ({
    * Handle MCP requests (POST, GET, DELETE)
    */
   async handle(ctx) {
-    const plugin = strapi.plugin(PLUGIN_ID$1);
+    const plugin = strapi.plugin(PLUGIN_ID$2);
     if (!plugin.createMcpServer) {
       ctx.status = 503;
       ctx.body = {
@@ -992,7 +1130,7 @@ const mcpController = ({ strapi }) => ({
       const requestedSessionId = ctx.request.headers["mcp-session-id"];
       let session = requestedSessionId ? plugin.sessions.get(requestedSessionId) : null;
       if (session && isSessionExpired(session)) {
-        strapi.log.debug(`[${PLUGIN_ID$1}] Session expired, removing: ${requestedSessionId}`);
+        strapi.log.debug(`[${PLUGIN_ID$2}] Session expired, removing: ${requestedSessionId}`);
         try {
           session.server.close();
         } catch {
@@ -1027,13 +1165,13 @@ const mcpController = ({ strapi }) => ({
         };
         plugin.sessions.set(sessionId, session);
         strapi.log.debug(
-          `[${PLUGIN_ID$1}] New MCP session created: ${sessionId} (auth: ${ctx.state.authMethod || "unknown"})`
+          `[${PLUGIN_ID$2}] New MCP session created: ${sessionId} (auth: ${ctx.state.authMethod || "unknown"})`
         );
       }
       try {
         await session.transport.handleRequest(ctx.req, ctx.res, ctx.request.body);
       } catch (transportError) {
-        strapi.log.warn(`[${PLUGIN_ID$1}] Transport error, cleaning up session: ${requestedSessionId}`, {
+        strapi.log.warn(`[${PLUGIN_ID$2}] Transport error, cleaning up session: ${requestedSessionId}`, {
           error: transportError instanceof Error ? transportError.message : String(transportError)
         });
         try {
@@ -1056,7 +1194,7 @@ const mcpController = ({ strapi }) => ({
       }
       ctx.respond = false;
     } catch (error) {
-      strapi.log.error(`[${PLUGIN_ID$1}] Error handling MCP request`, {
+      strapi.log.error(`[${PLUGIN_ID$2}] Error handling MCP request`, {
         error: error instanceof Error ? error.message : String(error),
         method: ctx.method,
         path: ctx.path
@@ -1083,6 +1221,32 @@ const contentApi = [
     path: "/embeddings-query",
     handler: "controller.queryEmbeddings"
   },
+  // Sync routes - for cron jobs or manual triggering
+  // Use API token for authentication
+  {
+    method: "GET",
+    path: "/sync",
+    handler: "controller.syncFromNeon",
+    config: {
+      description: "Sync embeddings from Neon DB to Strapi. Query params: removeOrphans=true, dryRun=true"
+    }
+  },
+  {
+    method: "POST",
+    path: "/sync",
+    handler: "controller.syncFromNeon",
+    config: {
+      description: "Sync embeddings from Neon DB to Strapi. Query params: removeOrphans=true, dryRun=true"
+    }
+  },
+  {
+    method: "GET",
+    path: "/sync/status",
+    handler: "controller.getSyncStatus",
+    config: {
+      description: "Get sync status between Neon and Strapi without making changes"
+    }
+  },
   // MCP routes - auth handled by middleware
   {
     method: "POST",
@@ -1202,11 +1366,180 @@ const routes = {
     routes: [...admin]
   }
 };
-const PLUGIN_ID = "strapi-content-embeddings";
-const CONTENT_TYPE_UID = `plugin::${PLUGIN_ID}.embedding`;
+const DEFAULT_SEPARATORS = [
+  "\n\n",
+  // Paragraphs
+  "\n",
+  // Lines
+  ". ",
+  // Sentences
+  "! ",
+  // Exclamations
+  "? ",
+  // Questions
+  "; ",
+  // Semicolons
+  ", ",
+  // Commas
+  " ",
+  // Words
+  ""
+  // Characters (last resort)
+];
+function estimateTokens(text) {
+  return Math.ceil(text.length / 4);
+}
+function needsChunking(content, maxChars = 4e3) {
+  return content.length > maxChars;
+}
+function splitWithSeparator(text, separator) {
+  if (separator === "") {
+    return text.split("");
+  }
+  const parts = text.split(separator);
+  const result = [];
+  for (let i = 0; i < parts.length; i++) {
+    if (i < parts.length - 1) {
+      result.push(parts[i] + separator);
+    } else if (parts[i]) {
+      result.push(parts[i]);
+    }
+  }
+  return result;
+}
+function splitText(text, chunkSize, separators) {
+  if (text.length <= chunkSize) {
+    return [text];
+  }
+  let bestSeparator = separators[separators.length - 1];
+  for (const sep of separators) {
+    if (text.includes(sep)) {
+      bestSeparator = sep;
+      break;
+    }
+  }
+  const splits = splitWithSeparator(text, bestSeparator);
+  const chunks = [];
+  let currentChunk = "";
+  for (const split of splits) {
+    if ((currentChunk + split).length <= chunkSize) {
+      currentChunk += split;
+    } else {
+      if (currentChunk) {
+        chunks.push(currentChunk);
+      }
+      if (split.length > chunkSize) {
+        const remainingSeparators = separators.slice(separators.indexOf(bestSeparator) + 1);
+        if (remainingSeparators.length > 0) {
+          chunks.push(...splitText(split, chunkSize, remainingSeparators));
+        } else {
+          for (let i = 0; i < split.length; i += chunkSize) {
+            chunks.push(split.slice(i, i + chunkSize));
+          }
+        }
+        currentChunk = "";
+      } else {
+        currentChunk = split;
+      }
+    }
+  }
+  if (currentChunk) {
+    chunks.push(currentChunk);
+  }
+  return chunks;
+}
+function addOverlap(chunks, overlap) {
+  if (overlap <= 0 || chunks.length <= 1) {
+    return chunks;
+  }
+  const result = [];
+  for (let i = 0; i < chunks.length; i++) {
+    let chunk = chunks[i];
+    if (i > 0) {
+      const prevChunk = chunks[i - 1];
+      const overlapText = prevChunk.slice(-overlap);
+      chunk = overlapText + chunk;
+    }
+    result.push(chunk);
+  }
+  return result;
+}
+function chunkContent(content, options2 = {}) {
+  const {
+    chunkSize = 4e3,
+    chunkOverlap = 200,
+    separators = DEFAULT_SEPARATORS
+  } = options2;
+  const cleanContent = content.trim();
+  if (!cleanContent) {
+    return [];
+  }
+  if (cleanContent.length <= chunkSize) {
+    return [{
+      text: cleanContent,
+      chunkIndex: 0,
+      totalChunks: 1,
+      startOffset: 0,
+      endOffset: cleanContent.length
+    }];
+  }
+  const rawChunks = splitText(cleanContent, chunkSize - chunkOverlap, separators);
+  const chunksWithOverlap = addOverlap(rawChunks, chunkOverlap);
+  const result = [];
+  let currentOffset = 0;
+  for (let i = 0; i < chunksWithOverlap.length; i++) {
+    const text = chunksWithOverlap[i].trim();
+    if (text) {
+      result.push({
+        text,
+        chunkIndex: i,
+        totalChunks: chunksWithOverlap.length,
+        startOffset: currentOffset,
+        endOffset: currentOffset + rawChunks[i].length
+      });
+    }
+    currentOffset += rawChunks[i].length;
+  }
+  const totalChunks = result.length;
+  result.forEach((chunk, idx) => {
+    chunk.chunkIndex = idx;
+    chunk.totalChunks = totalChunks;
+  });
+  return result;
+}
+function formatChunkTitle(baseTitle, chunkIndex, totalChunks) {
+  if (totalChunks === 1) {
+    return baseTitle;
+  }
+  return `${baseTitle} [Part ${chunkIndex + 1}/${totalChunks}]`;
+}
+const PLUGIN_ID$1 = "strapi-content-embeddings";
+const CONTENT_TYPE_UID$1 = `plugin::${PLUGIN_ID$1}.embedding`;
 const embeddings = ({ strapi }) => ({
+  /**
+   * Get plugin config with defaults
+   */
+  getConfig() {
+    const config2 = strapi.config.get("plugin::strapi-content-embeddings") || {};
+    return {
+      chunkSize: config2.chunkSize || 4e3,
+      chunkOverlap: config2.chunkOverlap || 200,
+      autoChunk: config2.autoChunk || false,
+      ...config2
+    };
+  },
+  /**
+   * Create a single embedding (no chunking)
+   */
   async createEmbedding(data) {
-    const { title, content, collectionType, fieldName, metadata, related } = data.data;
+    const { title, content, collectionType, fieldName, metadata, related, autoChunk } = data.data;
+    const config2 = this.getConfig();
+    const shouldChunk = autoChunk ?? config2.autoChunk;
+    const chunkSize = config2.chunkSize || 4e3;
+    if (shouldChunk && needsChunking(content, chunkSize)) {
+      const result = await this.createChunkedEmbedding(data);
+      return result.entity;
+    }
     const entityData = {
       title,
       content,
@@ -1217,7 +1550,7 @@ const embeddings = ({ strapi }) => ({
     if (related && related.__type && related.id) {
       entityData.related = related;
     }
-    const entity = await strapi.documents(CONTENT_TYPE_UID).create({
+    const entity = await strapi.documents(CONTENT_TYPE_UID$1).create({
       data: entityData
     });
     if (!pluginManager.isInitialized()) {
@@ -1232,7 +1565,7 @@ const embeddings = ({ strapi }) => ({
         collectionType: collectionType || "standalone",
         fieldName: fieldName || "content"
       });
-      const updatedEntity = await strapi.documents(CONTENT_TYPE_UID).update({
+      const updatedEntity = await strapi.documents(CONTENT_TYPE_UID$1).update({
         documentId: entity.documentId,
         data: {
           embeddingId: result.embeddingId,
@@ -1245,8 +1578,110 @@ const embeddings = ({ strapi }) => ({
       return entity;
     }
   },
+  /**
+   * Create embeddings with automatic chunking for large content
+   * Creates multiple embedding entities, one per chunk
+   */
+  async createChunkedEmbedding(data) {
+    const { title, content, collectionType, fieldName, metadata, related } = data.data;
+    const config2 = this.getConfig();
+    const chunkSize = config2.chunkSize || 4e3;
+    const chunkOverlap = config2.chunkOverlap || 200;
+    const chunks = chunkContent(content, { chunkSize, chunkOverlap });
+    if (chunks.length === 0) {
+      throw new Error("Content is empty or could not be chunked");
+    }
+    if (chunks.length === 1) {
+      const entity = await this.createEmbedding({
+        data: {
+          ...data.data,
+          autoChunk: false
+          // Prevent recursive chunking
+        }
+      });
+      return {
+        entity,
+        chunks: [entity],
+        totalChunks: 1,
+        wasChunked: false
+      };
+    }
+    console.log(`Chunking content into ${chunks.length} parts (chunkSize: ${chunkSize}, overlap: ${chunkOverlap})`);
+    const createdChunks = [];
+    let parentDocumentId = null;
+    for (const chunk of chunks) {
+      const chunkTitle = formatChunkTitle(title, chunk.chunkIndex, chunk.totalChunks);
+      const chunkMetadata = {
+        ...metadata,
+        isChunk: true,
+        chunkIndex: chunk.chunkIndex,
+        totalChunks: chunk.totalChunks,
+        startOffset: chunk.startOffset,
+        endOffset: chunk.endOffset,
+        originalTitle: title,
+        parentDocumentId,
+        estimatedTokens: estimateTokens(chunk.text)
+      };
+      const entityData = {
+        title: chunkTitle,
+        content: chunk.text,
+        collectionType: collectionType || "standalone",
+        fieldName: fieldName || "content",
+        metadata: chunkMetadata
+      };
+      if (chunk.chunkIndex === 0 && related && related.__type && related.id) {
+        entityData.related = related;
+      }
+      const entity = await strapi.documents(CONTENT_TYPE_UID$1).create({
+        data: entityData
+      });
+      if (chunk.chunkIndex === 0) {
+        parentDocumentId = entity.documentId;
+      } else {
+        await strapi.documents(CONTENT_TYPE_UID$1).update({
+          documentId: entity.documentId,
+          data: {
+            metadata: {
+              ...chunkMetadata,
+              parentDocumentId
+            }
+          }
+        });
+      }
+      if (pluginManager.isInitialized()) {
+        try {
+          const result = await pluginManager.createEmbedding({
+            id: entity.documentId,
+            title: chunkTitle,
+            content: chunk.text,
+            collectionType: collectionType || "standalone",
+            fieldName: fieldName || "content"
+          });
+          const updatedEntity = await strapi.documents(CONTENT_TYPE_UID$1).update({
+            documentId: entity.documentId,
+            data: {
+              embeddingId: result.embeddingId,
+              embedding: result.embedding
+            }
+          });
+          createdChunks.push(updatedEntity);
+        } catch (error) {
+          console.error(`Failed to create embedding for chunk ${chunk.chunkIndex}:`, error);
+          createdChunks.push(entity);
+        }
+      } else {
+        createdChunks.push(entity);
+      }
+    }
+    return {
+      entity: createdChunks[0],
+      chunks: createdChunks,
+      totalChunks: createdChunks.length,
+      wasChunked: true
+    };
+  },
   async deleteEmbedding(id) {
-    const currentEntry = await strapi.documents(CONTENT_TYPE_UID).findOne({
+    const currentEntry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
       documentId: String(id)
     });
     if (!currentEntry) {
@@ -1259,25 +1694,174 @@ const embeddings = ({ strapi }) => ({
         console.error("Failed to delete from vector store:", error);
       }
     }
-    const deletedEntry = await strapi.documents(CONTENT_TYPE_UID).delete({
+    const deletedEntry = await strapi.documents(CONTENT_TYPE_UID$1).delete({
       documentId: String(id)
     });
     return deletedEntry;
   },
+  /**
+   * Find all chunks related to a parent document
+   * Returns chunks including the parent itself
+   */
+  async findRelatedChunks(documentId) {
+    const entry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
+      documentId
+    });
+    if (!entry) {
+      return [];
+    }
+    const metadata = entry.metadata;
+    const parentId = metadata?.parentDocumentId || documentId;
+    const isChunked = metadata?.isChunk === true;
+    if (!isChunked && !metadata?.parentDocumentId) {
+      const children = await strapi.documents(CONTENT_TYPE_UID$1).findMany({
+        filters: {
+          metadata: {
+            $containsi: `"parentDocumentId":"${documentId}"`
+          }
+        }
+      });
+      if (children.length === 0) {
+        return [entry];
+      }
+      return [entry, ...children];
+    }
+    const allChunks = await strapi.documents(CONTENT_TYPE_UID$1).findMany({
+      filters: {
+        $or: [
+          { documentId: parentId },
+          {
+            metadata: {
+              $containsi: `"parentDocumentId":"${parentId}"`
+            }
+          }
+        ]
+      }
+    });
+    return allChunks.sort((a, b) => {
+      const aIndex = a.metadata?.chunkIndex ?? 0;
+      const bIndex = b.metadata?.chunkIndex ?? 0;
+      return aIndex - bIndex;
+    });
+  },
+  /**
+   * Delete all chunks related to a parent document
+   */
+  async deleteRelatedChunks(documentId) {
+    const chunks = await this.findRelatedChunks(documentId);
+    for (const chunk of chunks) {
+      if (pluginManager.isInitialized()) {
+        try {
+          await pluginManager.deleteEmbedding(chunk.documentId);
+        } catch (error) {
+          console.error(`Failed to delete chunk ${chunk.documentId} from vector store:`, error);
+        }
+      }
+      await strapi.documents(CONTENT_TYPE_UID$1).delete({
+        documentId: chunk.documentId
+      });
+    }
+    return chunks.length;
+  },
+  /**
+   * Update embeddings with automatic chunking support
+   * Handles re-chunking when content changes and exceeds chunk size
+   */
+  async updateChunkedEmbedding(id, data) {
+    const { title, content, metadata, autoChunk } = data.data;
+    const config2 = this.getConfig();
+    const currentEntry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
+      documentId: id
+    });
+    if (!currentEntry) {
+      throw new Error(`Embedding with id ${id} not found`);
+    }
+    const currentMetadata = currentEntry.metadata;
+    const parentDocumentId = currentMetadata?.parentDocumentId || id;
+    const newContent = content ?? currentEntry.content;
+    const newTitle = title ?? currentMetadata?.originalTitle ?? currentEntry.title;
+    const shouldChunk = autoChunk ?? config2.autoChunk;
+    const chunkSize = config2.chunkSize || 4e3;
+    const contentNeedsChunking = shouldChunk && needsChunking(newContent, chunkSize);
+    const existingChunks = await this.findRelatedChunks(id);
+    let originalRelated;
+    const firstChunk = existingChunks.find(
+      (c) => c.metadata?.chunkIndex === 0 || c.documentId === parentDocumentId
+    );
+    if (firstChunk?.related) {
+      originalRelated = firstChunk.related;
+    }
+    const deletedCount = await this.deleteRelatedChunks(id);
+    console.log(`Deleted ${deletedCount} existing chunk(s) for update`);
+    const preservedMetadata = { ...metadata };
+    delete preservedMetadata?.isChunk;
+    delete preservedMetadata?.chunkIndex;
+    delete preservedMetadata?.totalChunks;
+    delete preservedMetadata?.startOffset;
+    delete preservedMetadata?.endOffset;
+    delete preservedMetadata?.originalTitle;
+    delete preservedMetadata?.parentDocumentId;
+    delete preservedMetadata?.estimatedTokens;
+    if (contentNeedsChunking) {
+      return await this.createChunkedEmbedding({
+        data: {
+          title: newTitle.replace(/\s*\[Part \d+\/\d+\]$/, ""),
+          // Remove old part suffix
+          content: newContent,
+          collectionType: currentEntry.collectionType || "standalone",
+          fieldName: currentEntry.fieldName || "content",
+          metadata: preservedMetadata,
+          related: originalRelated,
+          autoChunk: true
+        }
+      });
+    } else {
+      const entity = await this.createEmbedding({
+        data: {
+          title: newTitle.replace(/\s*\[Part \d+\/\d+\]$/, ""),
+          // Remove old part suffix
+          content: newContent,
+          collectionType: currentEntry.collectionType || "standalone",
+          fieldName: currentEntry.fieldName || "content",
+          metadata: preservedMetadata,
+          related: originalRelated,
+          autoChunk: false
+        }
+      });
+      return {
+        entity,
+        chunks: [entity],
+        totalChunks: 1,
+        wasChunked: false
+      };
+    }
+  },
   async updateEmbedding(id, data) {
-    const { title, content, metadata } = data.data;
-    const currentEntry = await strapi.documents(CONTENT_TYPE_UID).findOne({
+    const { title, content, metadata, autoChunk } = data.data;
+    const config2 = this.getConfig();
+    const currentEntry = await strapi.documents(CONTENT_TYPE_UID$1).findOne({
       documentId: id
     });
     if (!currentEntry) {
       throw new Error(`Embedding with id ${id} not found`);
     }
+    const currentMetadata = currentEntry.metadata;
+    const isCurrentlyChunked = currentMetadata?.isChunk === true;
+    const hasRelatedChunks = currentMetadata?.parentDocumentId || isCurrentlyChunked;
+    const shouldChunk = autoChunk ?? config2.autoChunk;
+    const chunkSize = config2.chunkSize || 4e3;
+    const newContent = content ?? currentEntry.content;
+    const contentNeedsChunking = shouldChunk && needsChunking(newContent, chunkSize);
+    const contentChanged = content !== void 0 && content !== currentEntry.content;
+    if (hasRelatedChunks || contentNeedsChunking) {
+      const result = await this.updateChunkedEmbedding(id, data);
+      return result.entity;
+    }
     const updateData = {};
     if (title !== void 0) updateData.title = title;
     if (content !== void 0) updateData.content = content;
     if (metadata !== void 0) updateData.metadata = metadata;
-    const contentChanged = content !== void 0 && content !== currentEntry.content;
-    let updatedEntity = await strapi.documents(CONTENT_TYPE_UID).update({
+    let updatedEntity = await strapi.documents(CONTENT_TYPE_UID$1).update({
       documentId: id,
       data: updateData
     });
@@ -1291,7 +1875,7 @@ const embeddings = ({ strapi }) => ({
           collectionType: currentEntry.collectionType || "standalone",
           fieldName: currentEntry.fieldName || "content"
         });
-        updatedEntity = await strapi.documents(CONTENT_TYPE_UID).update({
+        updatedEntity = await strapi.documents(CONTENT_TYPE_UID$1).update({
           documentId: id,
           data: {
             embeddingId: result.embeddingId,
@@ -1320,7 +1904,7 @@ const embeddings = ({ strapi }) => ({
     }
   },
   async getEmbedding(id) {
-    return await strapi.documents(CONTENT_TYPE_UID).findOne({
+    return await strapi.documents(CONTENT_TYPE_UID$1).findOne({
       documentId: String(id)
     });
   },
@@ -1329,12 +1913,12 @@ const embeddings = ({ strapi }) => ({
     const pageSize = params?.pageSize || 10;
     const start = (page - 1) * pageSize;
     const [data, totalCount] = await Promise.all([
-      strapi.documents(CONTENT_TYPE_UID).findMany({
+      strapi.documents(CONTENT_TYPE_UID$1).findMany({
         limit: pageSize,
         start,
         filters: params?.filters
       }),
-      strapi.documents(CONTENT_TYPE_UID).count({
+      strapi.documents(CONTENT_TYPE_UID$1).count({
         filters: params?.filters
       })
     ]);
@@ -1345,8 +1929,216 @@ const embeddings = ({ strapi }) => ({
     };
   }
 });
+const PLUGIN_ID = "strapi-content-embeddings";
+const CONTENT_TYPE_UID = `plugin::${PLUGIN_ID}.embedding`;
+const sync = ({ strapi }) => ({
+  /**
+   * Sync embeddings from Neon DB to Strapi DB
+   *
+   * This performs the following operations:
+   * 1. Fetches all embeddings from Neon DB (source of truth)
+   * 2. Fetches all embeddings from Strapi DB
+   * 3. Creates missing entries in Strapi that exist in Neon
+   * 4. Updates Strapi entries where content differs from Neon
+   * 5. Optionally removes orphaned Strapi entries (no matching Neon record)
+   */
+  async syncFromNeon(options2) {
+    const { removeOrphans = false, dryRun = false } = options2 || {};
+    const result = {
+      success: false,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      neonCount: 0,
+      strapiCount: 0,
+      actions: {
+        created: 0,
+        updated: 0,
+        orphansRemoved: 0
+      },
+      details: {
+        created: [],
+        updated: [],
+        orphansRemoved: []
+      },
+      errors: []
+    };
+    if (!pluginManager.isInitialized()) {
+      result.errors.push(
+        "Plugin manager not initialized. Check your Neon and OpenAI configuration."
+      );
+      return result;
+    }
+    try {
+      const neonEmbeddings = await pluginManager.getAllNeonEmbeddings();
+      result.neonCount = neonEmbeddings.length;
+      const strapiEmbeddings = await strapi.documents(CONTENT_TYPE_UID).findMany({
+        limit: 1e4
+        // High limit to get all
+      });
+      result.strapiCount = strapiEmbeddings.length;
+      const neonBystrapiId = /* @__PURE__ */ new Map();
+      for (const neon of neonEmbeddings) {
+        if (neon.strapiId) {
+          neonBystrapiId.set(neon.strapiId, neon);
+        }
+      }
+      const strapiByDocumentId = /* @__PURE__ */ new Map();
+      for (const strapi2 of strapiEmbeddings) {
+        strapiByDocumentId.set(strapi2.documentId, strapi2);
+      }
+      for (const neon of neonEmbeddings) {
+        if (!neon.strapiId) {
+          result.errors.push(
+            `Neon embedding ${neon.id} has no strapiId in metadata`
+          );
+          continue;
+        }
+        const existingStrapi = strapiByDocumentId.get(neon.strapiId);
+        if (!existingStrapi) {
+          if (!dryRun) {
+            try {
+              await strapi.documents(CONTENT_TYPE_UID).create({
+                data: {
+                  documentId: neon.strapiId,
+                  title: neon.title,
+                  content: neon.content,
+                  embeddingId: neon.id,
+                  collectionType: neon.collectionType,
+                  fieldName: neon.fieldName
+                }
+              });
+              result.actions.created++;
+              result.details.created.push(
+                `${neon.strapiId} (${neon.title || "untitled"})`
+              );
+            } catch (error) {
+              result.errors.push(
+                `Failed to create Strapi entry for ${neon.strapiId}: ${error}`
+              );
+            }
+          } else {
+            result.actions.created++;
+            result.details.created.push(
+              `[DRY RUN] ${neon.strapiId} (${neon.title || "untitled"})`
+            );
+          }
+        } else {
+          const contentChanged = existingStrapi.content !== neon.content;
+          const titleChanged = existingStrapi.title !== neon.title;
+          const embeddingIdMissing = !existingStrapi.embeddingId;
+          if (contentChanged || titleChanged || embeddingIdMissing) {
+            if (!dryRun) {
+              try {
+                await strapi.documents(CONTENT_TYPE_UID).update({
+                  documentId: neon.strapiId,
+                  data: {
+                    title: neon.title,
+                    content: neon.content,
+                    embeddingId: neon.id
+                  }
+                });
+                result.actions.updated++;
+                result.details.updated.push(
+                  `${neon.strapiId} (${neon.title || "untitled"})`
+                );
+              } catch (error) {
+                result.errors.push(
+                  `Failed to update Strapi entry ${neon.strapiId}: ${error}`
+                );
+              }
+            } else {
+              result.actions.updated++;
+              result.details.updated.push(
+                `[DRY RUN] ${neon.strapiId} (${neon.title || "untitled"})`
+              );
+            }
+          }
+        }
+      }
+      if (removeOrphans) {
+        for (const strapiEmbed of strapiEmbeddings) {
+          const hasNeonRecord = neonBystrapiId.has(strapiEmbed.documentId);
+          if (!hasNeonRecord) {
+            if (!dryRun) {
+              try {
+                await strapi.documents(CONTENT_TYPE_UID).delete({
+                  documentId: strapiEmbed.documentId
+                });
+                result.actions.orphansRemoved++;
+                result.details.orphansRemoved.push(
+                  `${strapiEmbed.documentId} (${strapiEmbed.title || "untitled"})`
+                );
+              } catch (error) {
+                result.errors.push(
+                  `Failed to remove orphan ${strapiEmbed.documentId}: ${error}`
+                );
+              }
+            } else {
+              result.actions.orphansRemoved++;
+              result.details.orphansRemoved.push(
+                `[DRY RUN] ${strapiEmbed.documentId} (${strapiEmbed.title || "untitled"})`
+              );
+            }
+          }
+        }
+      }
+      result.success = result.errors.length === 0;
+      return result;
+    } catch (error) {
+      result.errors.push(`Sync failed: ${error}`);
+      return result;
+    }
+  },
+  /**
+   * Get sync status - compare Neon and Strapi without making changes
+   */
+  async getSyncStatus() {
+    if (!pluginManager.isInitialized()) {
+      throw new Error("Plugin manager not initialized");
+    }
+    const neonEmbeddings = await pluginManager.getAllNeonEmbeddings();
+    const strapiEmbeddings = await strapi.documents(CONTENT_TYPE_UID).findMany({
+      limit: 1e4
+    });
+    const neonBystrapiId = /* @__PURE__ */ new Map();
+    for (const neon of neonEmbeddings) {
+      if (neon.strapiId) {
+        neonBystrapiId.set(neon.strapiId, neon);
+      }
+    }
+    const strapiByDocumentId = /* @__PURE__ */ new Map();
+    for (const s of strapiEmbeddings) {
+      strapiByDocumentId.set(s.documentId, s);
+    }
+    let missingInStrapi = 0;
+    let contentDifferences = 0;
+    for (const neon of neonEmbeddings) {
+      if (!neon.strapiId) continue;
+      const strapiRecord = strapiByDocumentId.get(neon.strapiId);
+      if (!strapiRecord) {
+        missingInStrapi++;
+      } else if (strapiRecord.content !== neon.content) {
+        contentDifferences++;
+      }
+    }
+    let missingInNeon = 0;
+    for (const s of strapiEmbeddings) {
+      if (!neonBystrapiId.has(s.documentId)) {
+        missingInNeon++;
+      }
+    }
+    return {
+      neonCount: neonEmbeddings.length,
+      strapiCount: strapiEmbeddings.length,
+      inSync: missingInStrapi === 0 && missingInNeon === 0 && contentDifferences === 0,
+      missingInStrapi,
+      missingInNeon,
+      contentDifferences
+    };
+  }
+});
 const services = {
-  embeddings
+  embeddings,
+  sync
 };
 const index = {
   register,
@@ -1363,3 +2155,4 @@ const index = {
 export {
   index as default
 };
+//# sourceMappingURL=index.mjs.map