strapi-content-embeddings 0.1.5 → 0.1.6

package/dist/server/index.mjs

@@ -1,6 +1,5 @@
 import { OpenAIEmbeddings, ChatOpenAI } from "@langchain/openai";
 import { PGVectorStore } from "@langchain/community/vectorstores/pgvector";
-import { Document } from "@langchain/core/documents";
 import { StringOutputParser } from "@langchain/core/output_parsers";
 import { ChatPromptTemplate } from "@langchain/core/prompts";
 import { RunnableSequence, RunnablePassthrough } from "@langchain/core/runnables";
@@ -170,39 +169,46 @@ class PluginManager {
     console.log("Plugin Manager Initialization Complete");
   }
   async createEmbedding(docData) {
-    if (!this.embeddings || !this.vectorStoreConfig) {
+    if (!this.embeddings || !this.vectorStoreConfig || !this.pool) {
       throw new Error("Plugin manager not initialized");
     }
-    try {
-      const embeddingVector = await this.embeddings.embedQuery(docData.content);
-      const doc = new Document({
-        pageContent: docData.content,
-        metadata: {
+    const maxRetries = 3;
+    const retryDelay = 2e3;
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        const embeddingVector = await this.embeddings.embedQuery(docData.content);
+        const metadata = {
           id: docData.id,
           title: docData.title,
           collectionType: docData.collectionType || "standalone",
           fieldName: docData.fieldName || "content"
+        };
+        const vectorString = `[${embeddingVector.join(",")}]`;
+        const result = await this.pool.query(
+          `INSERT INTO embeddings_documents (content, metadata, embedding)
+           VALUES ($1, $2::jsonb, $3::vector)
+           RETURNING id`,
+          [docData.content, JSON.stringify(metadata), vectorString]
+        );
+        return {
+          embeddingId: result.rows[0]?.id || "",
+          embedding: embeddingVector
+        };
+      } catch (error) {
+        const isRateLimit = error.message?.includes("429") || error.message?.includes("rate");
+        const isLastAttempt = attempt === maxRetries;
+        if (isRateLimit && !isLastAttempt) {
+          console.log(`[createEmbedding] Rate limited, waiting ${retryDelay}ms before retry ${attempt + 1}/${maxRetries}...`);
+          await new Promise((resolve) => setTimeout(resolve, retryDelay * attempt));
+          continue;
         }
-      });
-      await PGVectorStore.fromDocuments(
-        [doc],
-        this.embeddings,
-        this.vectorStoreConfig
-      );
-      const result = await this.pool.query(
-        `SELECT id FROM embeddings_documents
-         WHERE metadata->>'id' = $1
-         ORDER BY id DESC LIMIT 1`,
-        [docData.id]
-      );
-      return {
-        embeddingId: result.rows[0]?.id || "",
-        embedding: embeddingVector
-      };
-    } catch (error) {
-      console.error(`Failed to create embedding: ${error}`);
-      throw new Error(`Failed to create embedding: ${error}`);
+        console.error(`[createEmbedding] Failed (attempt ${attempt}/${maxRetries}):`, error.message || error);
+        if (isLastAttempt) {
+          throw new Error(`Failed to create embedding after ${maxRetries} attempts: ${error.message || error}`);
+        }
+      }
     }
+    throw new Error("Failed to create embedding: unexpected error");
   }
   async deleteEmbedding(strapiId) {
     if (!this.pool) {
@@ -228,8 +234,14 @@ class PluginManager {
         this.vectorStoreConfig
       );
       const resultsWithScores = await vectorStore.similaritySearchWithScore(query, 6);
-      const SIMILARITY_THRESHOLD = 0.5;
+      console.log(`[queryEmbedding] Query: "${query}"`);
+      console.log(`[queryEmbedding] Found ${resultsWithScores.length} results:`);
+      resultsWithScores.forEach(([doc, score], i) => {
+        console.log(`  ${i + 1}. Score: ${score.toFixed(4)}, Title: ${doc.metadata?.title || "N/A"}`);
+      });
+      const SIMILARITY_THRESHOLD = 1;
       const relevantResults = resultsWithScores.filter(([_, score]) => score < SIMILARITY_THRESHOLD);
+      console.log(`[queryEmbedding] ${relevantResults.length} results passed threshold (< ${SIMILARITY_THRESHOLD})`);
       const topResults = relevantResults.slice(0, 3);
       const sourceDocuments = topResults.map(([doc]) => doc);
       const bestMatchForDisplay = topResults.length > 0 ? [topResults[0][0]] : [];
@@ -348,6 +360,59 @@ Context:
     this.chat = null;
     this.vectorStoreConfig = null;
   }
+  /**
+   * Clear all embeddings from Neon DB
+   * Returns the number of deleted rows
+   */
+  async clearAllNeonEmbeddings() {
+    if (!this.pool) {
+      throw new Error("Plugin manager not initialized");
+    }
+    try {
+      const result = await this.pool.query(`
+        DELETE FROM embeddings_documents
+        RETURNING id
+      `);
+      console.log(`[clearAllNeonEmbeddings] Deleted ${result.rowCount} embeddings from Neon`);
+      return result.rowCount || 0;
+    } catch (error) {
+      console.error(`Failed to clear Neon embeddings: ${error}`);
+      throw new Error(`Failed to clear Neon embeddings: ${error}`);
+    }
+  }
+  /**
+   * Debug method to inspect raw data in Neon DB
+   */
+  async debugNeonEmbeddings() {
+    if (!this.pool) {
+      throw new Error("Plugin manager not initialized");
+    }
+    try {
+      const result = await this.pool.query(`
+        SELECT
+          id,
+          content,
+          metadata,
+          pg_typeof(metadata) as metadata_type,
+          embedding IS NOT NULL as has_embedding,
+          CASE WHEN embedding IS NOT NULL THEN array_length(embedding::float[], 1) ELSE 0 END as embedding_length
+        FROM embeddings_documents
+        ORDER BY id
+        LIMIT 20
+      `);
+      return result.rows.map((row) => ({
+        id: row.id,
+        content: row.content?.substring(0, 200) + (row.content?.length > 200 ? "..." : ""),
+        metadata: row.metadata,
+        metadataType: row.metadata_type,
+        hasEmbedding: row.has_embedding,
+        embeddingLength: row.embedding_length || 0
+      }));
+    } catch (error) {
+      console.error(`Failed to debug Neon embeddings: ${error}`);
+      throw new Error(`Failed to debug Neon embeddings: ${error}`);
+    }
+  }
 }
 const pluginManager = new PluginManager();
 const SemanticSearchSchema = z.object({
@@ -994,9 +1059,12 @@ const PLUGIN_ID$3 = "strapi-content-embeddings";
 const controller = ({ strapi }) => ({
   async createEmbedding(ctx) {
     try {
+      console.log("[createEmbedding] Starting, autoChunk:", ctx.request.body?.data?.autoChunk);
       const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").createEmbedding(ctx.request.body);
+      console.log("[createEmbedding] Completed, documentId:", result?.documentId);
       ctx.body = result;
     } catch (error) {
+      console.error("[createEmbedding] Error:", error.message);
       ctx.throw(500, error.message || "Failed to create embedding");
     }
   },
@@ -1055,6 +1123,23 @@ const controller = ({ strapi }) => ({
       ctx.throw(500, error.message || "Failed to query embeddings");
     }
   },
+  /**
+   * Get all chunks related to a document
+   * GET /api/strapi-content-embeddings/embeddings/related-chunks/:id
+   */
+  async getRelatedChunks(ctx) {
+    try {
+      const { id } = ctx.params;
+      const result = await strapi.plugin(PLUGIN_ID$3).service("embeddings").findRelatedChunks(id);
+      console.log(`[getRelatedChunks] Found ${result.length} chunks for document ${id}`);
+      ctx.body = {
+        data: result,
+        count: result.length
+      };
+    } catch (error) {
+      ctx.throw(500, error.message || "Failed to get related chunks");
+    }
+  },
   /**
    * Sync embeddings from Neon DB to Strapi DB
    * GET /api/strapi-content-embeddings/sync
@@ -1086,6 +1171,39 @@ const controller = ({ strapi }) => ({
     } catch (error) {
       ctx.throw(500, error.message || "Failed to get sync status");
     }
+  },
+  /**
+   * Debug endpoint to inspect Neon DB contents
+   * GET /api/strapi-content-embeddings/debug/neon
+   */
+  async debugNeon(ctx) {
+    try {
+      const { pluginManager: pluginManager2 } = require("../plugin-manager");
+      const result = await pluginManager2.debugNeonEmbeddings();
+      ctx.body = {
+        count: result.length,
+        embeddings: result
+      };
+    } catch (error) {
+      ctx.throw(500, error.message || "Failed to debug Neon");
+    }
+  },
+  /**
+   * Recreate all embeddings in Neon from Strapi data
+   * POST /api/strapi-content-embeddings/recreate
+   *
+   * Use this when embeddings were created with incorrect metadata format
+   * WARNING: This will delete ALL existing Neon embeddings and recreate them
+   */
+  async recreateEmbeddings(ctx) {
+    try {
+      console.log("[recreateEmbeddings] Starting recreation of all embeddings...");
+      const result = await strapi.plugin(PLUGIN_ID$3).service("sync").recreateAllEmbeddings();
+      ctx.body = result;
+    } catch (error) {
+      console.error("[recreateEmbeddings] Error:", error.message);
+      ctx.throw(500, error.message || "Failed to recreate embeddings");
+    }
   }
 });
 const PLUGIN_ID$2 = "strapi-content-embeddings";
@@ -1354,6 +1472,45 @@ const admin = [
         }
       ]
     }
+  },
+  {
+    method: "GET",
+    path: "/embeddings/related-chunks/:id",
+    handler: "controller.getRelatedChunks",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.read"] }
+        }
+      ]
+    }
+  },
+  {
+    method: "GET",
+    path: "/debug/neon",
+    handler: "controller.debugNeon",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.read"] }
+        }
+      ]
+    }
+  },
+  {
+    method: "POST",
+    path: "/recreate",
+    handler: "controller.recreateEmbeddings",
+    config: {
+      policies: [
+        {
+          name: "admin::hasPermissions",
+          config: { actions: ["plugin::strapi-content-embeddings.update"] }
+        }
+      ]
+    }
   }
 ];
 const routes = {
@@ -1606,10 +1763,11 @@ const embeddings = ({ strapi }) => ({
         wasChunked: false
       };
     }
-    console.log(`Chunking content into ${chunks.length} parts (chunkSize: ${chunkSize}, overlap: ${chunkOverlap})`);
+    console.log(`[createChunkedEmbedding] Chunking content into ${chunks.length} parts (chunkSize: ${chunkSize}, overlap: ${chunkOverlap})`);
     const createdChunks = [];
     let parentDocumentId = null;
     for (const chunk of chunks) {
+      console.log(`[createChunkedEmbedding] Processing chunk ${chunk.chunkIndex + 1}/${chunks.length}`);
       const chunkTitle = formatChunkTitle(title, chunk.chunkIndex, chunk.totalChunks);
       const chunkMetadata = {
         ...metadata,
@@ -1632,12 +1790,15 @@ const embeddings = ({ strapi }) => ({
       if (chunk.chunkIndex === 0 && related && related.__type && related.id) {
         entityData.related = related;
       }
+      console.log(`[chunk ${chunk.chunkIndex + 1}] Creating entity in DB...`);
       const entity = await strapi.documents(CONTENT_TYPE_UID$1).create({
         data: entityData
       });
+      console.log(`[chunk ${chunk.chunkIndex + 1}] Entity created: ${entity.documentId}`);
       if (chunk.chunkIndex === 0) {
         parentDocumentId = entity.documentId;
       } else {
+        console.log(`[chunk ${chunk.chunkIndex + 1}] Updating metadata with parent ref...`);
         await strapi.documents(CONTENT_TYPE_UID$1).update({
           documentId: entity.documentId,
           data: {
@@ -1647,9 +1808,11 @@ const embeddings = ({ strapi }) => ({
             }
           }
         });
+        console.log(`[chunk ${chunk.chunkIndex + 1}] Metadata updated`);
       }
       if (pluginManager.isInitialized()) {
         try {
+          console.log(`[chunk ${chunk.chunkIndex + 1}] Creating OpenAI embedding...`);
           const result = await pluginManager.createEmbedding({
             id: entity.documentId,
             title: chunkTitle,
@@ -1657,6 +1820,8 @@ const embeddings = ({ strapi }) => ({
             collectionType: collectionType || "standalone",
             fieldName: fieldName || "content"
           });
+          console.log(`[chunk ${chunk.chunkIndex + 1}] OpenAI embedding created`);
+          console.log(`[chunk ${chunk.chunkIndex + 1}] Saving embedding to DB...`);
           const updatedEntity = await strapi.documents(CONTENT_TYPE_UID$1).update({
             documentId: entity.documentId,
             data: {
@@ -1664,15 +1829,17 @@ const embeddings = ({ strapi }) => ({
               embedding: result.embedding
             }
           });
+          console.log(`[chunk ${chunk.chunkIndex + 1}] Chunk complete`);
           createdChunks.push(updatedEntity);
         } catch (error) {
-          console.error(`Failed to create embedding for chunk ${chunk.chunkIndex}:`, error);
+          console.error(`[chunk ${chunk.chunkIndex + 1}] FAILED:`, error.message || error);
           createdChunks.push(entity);
         }
       } else {
         createdChunks.push(entity);
       }
     }
+    console.log(`[createChunkedEmbedding] Completed, created ${createdChunks.length} chunks, first documentId: ${createdChunks[0]?.documentId}`);
     return {
       entity: createdChunks[0],
       chunks: createdChunks,
@@ -1719,8 +1886,11 @@ const embeddings = ({ strapi }) => ({
           metadata: {
             $containsi: `"parentDocumentId":"${documentId}"`
           }
-        }
+        },
+        limit: -1
+        // No limit - get all
       });
+      console.log(`[findRelatedChunks] Found ${children.length} children for parent ${documentId}`);
       if (children.length === 0) {
         return [entry];
       }
@@ -1736,8 +1906,11 @@ const embeddings = ({ strapi }) => ({
             }
           }
         ]
-      }
+      },
+      limit: -1
+      // No limit - get all
     });
+    console.log(`[findRelatedChunks] Found ${allChunks.length} total chunks for parent ${parentId}`);
     return allChunks.sort((a, b) => {
       const aIndex = a.metadata?.chunkIndex ?? 0;
       const bIndex = b.metadata?.chunkIndex ?? 0;
@@ -2134,6 +2307,94 @@ const sync = ({ strapi }) => ({
       missingInNeon,
       contentDifferences
     };
+  },
+  /**
+   * Recreate all embeddings in Neon DB from Strapi data
+   *
+   * This will:
+   * 1. Delete ALL embeddings from Neon DB
+   * 2. Re-create embeddings for each Strapi embedding entry
+   * 3. Update Strapi entries with new embedding IDs
+   *
+   * Use this when embeddings were created with incorrect metadata format
+   */
+  async recreateAllEmbeddings() {
+    const result = {
+      success: false,
+      timestamp: (/* @__PURE__ */ new Date()).toISOString(),
+      deletedFromNeon: 0,
+      processedFromStrapi: 0,
+      recreatedInNeon: 0,
+      errors: [],
+      details: {
+        recreated: [],
+        failed: []
+      }
+    };
+    if (!pluginManager.isInitialized()) {
+      result.errors.push(
+        "Plugin manager not initialized. Check your Neon and OpenAI configuration."
+      );
+      return result;
+    }
+    try {
+      console.log("[recreateAllEmbeddings] Step 1: Clearing Neon DB...");
+      result.deletedFromNeon = await pluginManager.clearAllNeonEmbeddings();
+      console.log(`[recreateAllEmbeddings] Deleted ${result.deletedFromNeon} embeddings from Neon`);
+      console.log("[recreateAllEmbeddings] Step 2: Fetching Strapi embeddings...");
+      const strapiEmbeddings = await strapi.documents(CONTENT_TYPE_UID).findMany({
+        limit: -1
+        // Get all
+      });
+      result.processedFromStrapi = strapiEmbeddings.length;
+      console.log(`[recreateAllEmbeddings] Found ${strapiEmbeddings.length} embeddings in Strapi`);
+      if (strapiEmbeddings.length === 0) {
+        result.success = true;
+        return result;
+      }
+      console.log("[recreateAllEmbeddings] Step 3: Recreating embeddings in Neon...");
+      for (let i = 0; i < strapiEmbeddings.length; i++) {
+        const entry = strapiEmbeddings[i];
+        const progress = `[${i + 1}/${strapiEmbeddings.length}]`;
+        if (!entry.content) {
+          console.log(`${progress} Skipping ${entry.documentId} - no content`);
+          result.details.failed.push(`${entry.documentId}: no content`);
+          continue;
+        }
+        try {
+          console.log(`${progress} Creating embedding for: ${entry.title || entry.documentId}`);
+          const embeddingResult = await pluginManager.createEmbedding({
+            id: entry.documentId,
+            title: entry.title || "",
+            content: entry.content,
+            collectionType: entry.collectionType || "standalone",
+            fieldName: entry.fieldName || "content"
+          });
+          await strapi.documents(CONTENT_TYPE_UID).update({
+            documentId: entry.documentId,
+            data: {
+              embeddingId: embeddingResult.embeddingId,
+              embedding: embeddingResult.embedding
+            }
+          });
+          result.recreatedInNeon++;
+          result.details.recreated.push(`${entry.documentId} (${entry.title || "untitled"})`);
+          if (i < strapiEmbeddings.length - 1) {
+            await new Promise((resolve) => setTimeout(resolve, 500));
+          }
+        } catch (error) {
+          console.error(`${progress} Failed:`, error.message || error);
+          result.errors.push(`${entry.documentId}: ${error.message || error}`);
+          result.details.failed.push(`${entry.documentId}: ${error.message || error}`);
+        }
+      }
+      result.success = result.errors.length === 0;
+      console.log(`[recreateAllEmbeddings] Complete. Recreated: ${result.recreatedInNeon}, Failed: ${result.details.failed.length}`);
+      return result;
+    } catch (error) {
+      result.errors.push(`Recreate failed: ${error.message || error}`);
+      return result;
+    }
   }
 });
 const services = {
@@ -2155,4 +2416,3 @@ const index = {
 export {
   index as default
 };
-//# sourceMappingURL=index.mjs.map

package/dist/server/src/controllers/controller.d.ts

@@ -8,6 +8,11 @@ declare const controller: ({ strapi }: {
     getEmbeddings(ctx: any): Promise<void>;
     getEmbedding(ctx: any): Promise<void>;
     queryEmbeddings(ctx: any): Promise<void>;
+    /**
+     * Get all chunks related to a document
+     * GET /api/strapi-content-embeddings/embeddings/related-chunks/:id
+     */
+    getRelatedChunks(ctx: any): Promise<void>;
     /**
      * Sync embeddings from Neon DB to Strapi DB
      * GET /api/strapi-content-embeddings/sync
@@ -22,5 +27,18 @@ declare const controller: ({ strapi }: {
      * GET /api/strapi-content-embeddings/sync/status
      */
     getSyncStatus(ctx: any): Promise<void>;
+    /**
+     * Debug endpoint to inspect Neon DB contents
+     * GET /api/strapi-content-embeddings/debug/neon
+     */
+    debugNeon(ctx: any): Promise<void>;
+    /**
+     * Recreate all embeddings in Neon from Strapi data
+     * POST /api/strapi-content-embeddings/recreate
+     *
+     * Use this when embeddings were created with incorrect metadata format
+     * WARNING: This will delete ALL existing Neon embeddings and recreate them
+     */
+    recreateEmbeddings(ctx: any): Promise<void>;
 };
 export default controller;

package/dist/server/src/controllers/index.d.ts

@@ -8,8 +8,11 @@ declare const _default: {
         getEmbeddings(ctx: any): Promise<void>;
         getEmbedding(ctx: any): Promise<void>;
         queryEmbeddings(ctx: any): Promise<void>;
+        getRelatedChunks(ctx: any): Promise<void>;
         syncFromNeon(ctx: any): Promise<void>;
         getSyncStatus(ctx: any): Promise<void>;
+        debugNeon(ctx: any): Promise<void>;
+        recreateEmbeddings(ctx: any): Promise<void>;
     };
     mcp: ({ strapi }: {
         strapi: import("@strapi/types/dist/core").Strapi;

package/dist/server/src/index.d.ts

@@ -29,8 +29,11 @@ declare const _default: {
             getEmbeddings(ctx: any): Promise<void>;
             getEmbedding(ctx: any): Promise<void>;
             queryEmbeddings(ctx: any): Promise<void>;
+            getRelatedChunks(ctx: any): Promise<void>;
             syncFromNeon(ctx: any): Promise<void>;
             getSyncStatus(ctx: any): Promise<void>;
+            debugNeon(ctx: any): Promise<void>;
+            recreateEmbeddings(ctx: any): Promise<void>;
         };
         mcp: ({ strapi }: {
             strapi: import("@strapi/types/dist/core").Strapi;
@@ -127,6 +130,7 @@ declare const _default: {
                 missingInNeon: number;
                 contentDifferences: number;
             }>;
+            recreateAllEmbeddings(): Promise<import("./services/sync").RecreateResult>;
         };
     };
     contentTypes: {

package/dist/server/src/plugin-manager.d.ts

@@ -56,6 +56,22 @@ declare class PluginManager {
      */
     deleteNeonEmbeddingById(neonId: string): Promise<void>;
     destroy(): Promise<void>;
+    /**
+     * Clear all embeddings from Neon DB
+     * Returns the number of deleted rows
+     */
+    clearAllNeonEmbeddings(): Promise<number>;
+    /**
+     * Debug method to inspect raw data in Neon DB
+     */
+    debugNeonEmbeddings(): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+        metadataType: string;
+        hasEmbedding: boolean;
+        embeddingLength: number;
+    }>>;
 }
 export declare const pluginManager: PluginManager;
 export type { PluginConfig, EmbeddingDocument, QueryResponse, CreateEmbeddingResult };

package/dist/server/src/services/index.d.ts

@@ -42,6 +42,7 @@ declare const _default: {
             missingInNeon: number;
             contentDifferences: number;
         }>;
+        recreateAllEmbeddings(): Promise<import("./sync").RecreateResult>;
     };
 };
 export default _default;

package/dist/server/src/services/sync.d.ts

@@ -16,6 +16,18 @@ export interface SyncResult {
     };
     errors: string[];
 }
+export interface RecreateResult {
+    success: boolean;
+    timestamp: string;
+    deletedFromNeon: number;
+    processedFromStrapi: number;
+    recreatedInNeon: number;
+    errors: string[];
+    details: {
+        recreated: string[];
+        failed: string[];
+    };
+}
 declare const sync: ({ strapi }: {
     strapi: Core.Strapi;
 }) => {
@@ -44,5 +56,16 @@ declare const sync: ({ strapi }: {
         missingInNeon: number;
         contentDifferences: number;
     }>;
+    /**
+     * Recreate all embeddings in Neon DB from Strapi data
+     *
+     * This will:
+     * 1. Delete ALL embeddings from Neon DB
+     * 2. Re-create embeddings for each Strapi embedding entry
+     * 3. Update Strapi entries with new embedding IDs
+     *
+     * Use this when embeddings were created with incorrect metadata format
+     */
+    recreateAllEmbeddings(): Promise<RecreateResult>;
 };
 export default sync;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strapi-content-embeddings",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "Strapi v5 plugin for vector embeddings with OpenAI and Neon PostgreSQL. Enables semantic search, RAG chat, and MCP (Model Context Protocol) integration.",
   "keywords": [
     "strapi",