strapi-content-embeddings 0.1.4 → 0.1.6

package/dist/server/src/config/index.d.ts

@@ -14,12 +14,21 @@ export interface PluginConfigSchema {
     openAIApiKey?: string;
     neonConnectionString?: string;
     embeddingModel?: EmbeddingModelName;
+    /** Maximum characters per chunk (default: 4000, roughly ~1000 tokens) */
+    chunkSize?: number;
+    /** Number of characters to overlap between chunks (default: 200) */
+    chunkOverlap?: number;
+    /** Automatically chunk content that exceeds chunkSize (default: false) */
+    autoChunk?: boolean;
 }
 declare const _default: {
     default: {
         openAIApiKey: string;
         neonConnectionString: string;
         embeddingModel: "text-embedding-3-small" | "text-embedding-3-large" | "text-embedding-ada-002";
+        chunkSize: number;
+        chunkOverlap: number;
+        autoChunk: boolean;
     };
     validator(config: PluginConfigSchema): void;
 };

package/dist/server/src/controllers/controller.d.ts

@@ -8,5 +8,37 @@ declare const controller: ({ strapi }: {
     getEmbeddings(ctx: any): Promise<void>;
     getEmbedding(ctx: any): Promise<void>;
     queryEmbeddings(ctx: any): Promise<void>;
+    /**
+     * Get all chunks related to a document
+     * GET /api/strapi-content-embeddings/embeddings/related-chunks/:id
+     */
+    getRelatedChunks(ctx: any): Promise<void>;
+    /**
+     * Sync embeddings from Neon DB to Strapi DB
+     * GET /api/strapi-content-embeddings/sync
+     *
+     * Query params:
+     * - removeOrphans: boolean (default: false) - Remove Strapi entries that don't exist in Neon
+     * - dryRun: boolean (default: false) - Preview changes without applying them
+     */
+    syncFromNeon(ctx: any): Promise<void>;
+    /**
+     * Get sync status - compare Neon and Strapi without making changes
+     * GET /api/strapi-content-embeddings/sync/status
+     */
+    getSyncStatus(ctx: any): Promise<void>;
+    /**
+     * Debug endpoint to inspect Neon DB contents
+     * GET /api/strapi-content-embeddings/debug/neon
+     */
+    debugNeon(ctx: any): Promise<void>;
+    /**
+     * Recreate all embeddings in Neon from Strapi data
+     * POST /api/strapi-content-embeddings/recreate
+     *
+     * Use this when embeddings were created with incorrect metadata format
+     * WARNING: This will delete ALL existing Neon embeddings and recreate them
+     */
+    recreateEmbeddings(ctx: any): Promise<void>;
 };
 export default controller;

package/dist/server/src/controllers/index.d.ts

@@ -8,6 +8,11 @@ declare const _default: {
         getEmbeddings(ctx: any): Promise<void>;
         getEmbedding(ctx: any): Promise<void>;
         queryEmbeddings(ctx: any): Promise<void>;
+        getRelatedChunks(ctx: any): Promise<void>;
+        syncFromNeon(ctx: any): Promise<void>;
+        getSyncStatus(ctx: any): Promise<void>;
+        debugNeon(ctx: any): Promise<void>;
+        recreateEmbeddings(ctx: any): Promise<void>;
     };
     mcp: ({ strapi }: {
         strapi: import("@strapi/types/dist/core").Strapi;

package/dist/server/src/index.d.ts

@@ -13,6 +13,9 @@ declare const _default: {
             openAIApiKey: string;
             neonConnectionString: string;
             embeddingModel: "text-embedding-3-small" | "text-embedding-3-large" | "text-embedding-ada-002";
+            chunkSize: number;
+            chunkOverlap: number;
+            autoChunk: boolean;
         };
         validator(config: import("./config").PluginConfigSchema): void;
     };
@@ -26,6 +29,11 @@ declare const _default: {
             getEmbeddings(ctx: any): Promise<void>;
             getEmbedding(ctx: any): Promise<void>;
             queryEmbeddings(ctx: any): Promise<void>;
+            getRelatedChunks(ctx: any): Promise<void>;
+            syncFromNeon(ctx: any): Promise<void>;
+            getSyncStatus(ctx: any): Promise<void>;
+            debugNeon(ctx: any): Promise<void>;
+            recreateEmbeddings(ctx: any): Promise<void>;
         };
         mcp: ({ strapi }: {
             strapi: import("@strapi/types/dist/core").Strapi;
@@ -41,6 +49,15 @@ declare const _default: {
                 path: string;
                 handler: string;
                 config?: undefined;
+            } | {
+                method: string;
+                path: string;
+                handler: string;
+                config: {
+                    description: string;
+                    auth?: undefined;
+                    policies?: undefined;
+                };
             } | {
                 method: string;
                 path: string;
@@ -48,6 +65,7 @@ declare const _default: {
                 config: {
                     auth: boolean;
                     policies: any[];
+                    description?: undefined;
                 };
             })[];
         };
@@ -72,12 +90,17 @@ declare const _default: {
         embeddings: ({ strapi }: {
             strapi: import("@strapi/types/dist/core").Strapi;
         }) => {
-            createEmbedding(data: import("./services/embeddings").CreateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+            getConfig(): import("./config").PluginConfigSchema;
+            createEmbedding(data: import("./services/embeddings").CreateEmbeddingData): Promise<any>;
+            createChunkedEmbedding(data: import("./services/embeddings").CreateEmbeddingData): Promise<import("./services/embeddings").ChunkedEmbeddingResult>;
             deleteEmbedding(id: string | number): Promise<{
                 documentId: string;
                 entries: import("@strapi/types/dist/modules/documents").AnyDocument[];
             }>;
-            updateEmbedding(id: string, data: import("./services/embeddings").UpdateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+            findRelatedChunks(documentId: string): Promise<any[]>;
+            deleteRelatedChunks(documentId: string): Promise<number>;
+            updateChunkedEmbedding(id: string, data: import("./services/embeddings").UpdateEmbeddingData): Promise<import("./services/embeddings").ChunkedEmbeddingResult>;
+            updateEmbedding(id: string, data: import("./services/embeddings").UpdateEmbeddingData): Promise<any>;
             queryEmbeddings(query: string): Promise<import("./plugin-manager").QueryResponse | {
                 error: string;
             }>;
@@ -92,6 +115,23 @@ declare const _default: {
                 totalCount: number;
             }>;
         };
+        sync: ({ strapi }: {
+            strapi: import("@strapi/types/dist/core").Strapi;
+        }) => {
+            syncFromNeon(options?: {
+                removeOrphans?: boolean;
+                dryRun?: boolean;
+            }): Promise<import("./services/sync").SyncResult>;
+            getSyncStatus(): Promise<{
+                neonCount: number;
+                strapiCount: number;
+                inSync: boolean;
+                missingInStrapi: number;
+                missingInNeon: number;
+                contentDifferences: number;
+            }>;
+            recreateAllEmbeddings(): Promise<import("./services/sync").RecreateResult>;
+        };
     };
     contentTypes: {
         embedding: {

package/dist/server/src/mcp/tools/create-embedding.d.ts

@@ -2,6 +2,7 @@
  * Create Embedding Tool
  *
  * Creates a new embedding from text content.
+ * Supports automatic chunking for large content.
  */
 import type { Core } from '@strapi/strapi';
 export declare const createEmbeddingTool: {
@@ -22,6 +23,10 @@ export declare const createEmbeddingTool: {
                 type: string;
                 description: string;
             };
+            autoChunk: {
+                type: string;
+                description: string;
+            };
         };
         required: string[];
     };
@@ -30,6 +35,7 @@ export declare function handleCreateEmbedding(strapi: Core.Strapi, args: {
     title: string;
     content: string;
     metadata?: Record<string, any>;
+    autoChunk?: boolean;
 }): Promise<{
     content: {
         type: string;

package/dist/server/src/mcp/tools/index.d.ts

@@ -99,6 +99,10 @@ export declare const tools: ({
                 type: string;
                 description: string;
             };
+            autoChunk: {
+                type: string;
+                description: string;
+            };
         };
         required: string[];
     };

package/dist/server/src/plugin-manager.d.ts

@@ -39,7 +39,39 @@ declare class PluginManager {
     queryEmbedding(query: string): Promise<QueryResponse>;
     similaritySearch(query: string, k?: number): Promise<Document[]>;
     isInitialized(): boolean;
+    /**
+     * Get all embeddings from Neon DB
+     * Returns the metadata (including Strapi documentId) for each embedding
+     */
+    getAllNeonEmbeddings(): Promise<Array<{
+        id: string;
+        strapiId: string;
+        title: string;
+        content: string;
+        collectionType: string;
+        fieldName: string;
+    }>>;
+    /**
+     * Delete an embedding from Neon by its Neon UUID (not Strapi ID)
+     */
+    deleteNeonEmbeddingById(neonId: string): Promise<void>;
     destroy(): Promise<void>;
+    /**
+     * Clear all embeddings from Neon DB
+     * Returns the number of deleted rows
+     */
+    clearAllNeonEmbeddings(): Promise<number>;
+    /**
+     * Debug method to inspect raw data in Neon DB
+     */
+    debugNeonEmbeddings(): Promise<Array<{
+        id: string;
+        content: string;
+        metadata: any;
+        metadataType: string;
+        hasEmbedding: boolean;
+        embeddingLength: number;
+    }>>;
 }
 export declare const pluginManager: PluginManager;
 export type { PluginConfig, EmbeddingDocument, QueryResponse, CreateEmbeddingResult };

package/dist/server/src/routes/content-api.d.ts

@@ -3,6 +3,15 @@ declare const _default: ({
     path: string;
     handler: string;
     config?: undefined;
+} | {
+    method: string;
+    path: string;
+    handler: string;
+    config: {
+        description: string;
+        auth?: undefined;
+        policies?: undefined;
+    };
 } | {
     method: string;
     path: string;
@@ -10,6 +19,7 @@ declare const _default: ({
     config: {
         auth: boolean;
         policies: any[];
+        description?: undefined;
     };
 })[];
 export default _default;

package/dist/server/src/routes/index.d.ts

@@ -6,6 +6,15 @@ declare const _default: {
             path: string;
             handler: string;
             config?: undefined;
+        } | {
+            method: string;
+            path: string;
+            handler: string;
+            config: {
+                description: string;
+                auth?: undefined;
+                policies?: undefined;
+            };
         } | {
             method: string;
             path: string;
@@ -13,6 +22,7 @@ declare const _default: {
             config: {
                 auth: boolean;
                 policies: any[];
+                description?: undefined;
             };
         })[];
     };

package/dist/server/src/services/embeddings.d.ts

@@ -1,4 +1,5 @@
 import type { Core } from "@strapi/strapi";
+import type { PluginConfigSchema } from "../config";
 export interface CreateEmbeddingData {
     data: {
         title: string;
@@ -10,6 +11,8 @@ export interface CreateEmbeddingData {
             __type: string;
             id: number;
         };
+        /** Enable chunking for large content (overrides config.autoChunk) */
+        autoChunk?: boolean;
     };
 }
 export interface UpdateEmbeddingData {
@@ -17,17 +20,55 @@ export interface UpdateEmbeddingData {
         title?: string;
         content?: string;
         metadata?: Record<string, any>;
+        /** Enable chunking for large content on update (overrides config.autoChunk) */
+        autoChunk?: boolean;
     };
 }
+export interface ChunkedEmbeddingResult {
+    /** The parent/first embedding entity */
+    entity: any;
+    /** All chunk entities created */
+    chunks: any[];
+    /** Total number of chunks created */
+    totalChunks: number;
+    /** Whether content was chunked */
+    wasChunked: boolean;
+}
 declare const embeddings: ({ strapi }: {
     strapi: Core.Strapi;
 }) => {
-    createEmbedding(data: CreateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+    /**
+     * Get plugin config with defaults
+     */
+    getConfig(): PluginConfigSchema;
+    /**
+     * Create a single embedding (no chunking)
+     */
+    createEmbedding(data: CreateEmbeddingData): Promise<any>;
+    /**
+     * Create embeddings with automatic chunking for large content
+     * Creates multiple embedding entities, one per chunk
+     */
+    createChunkedEmbedding(data: CreateEmbeddingData): Promise<ChunkedEmbeddingResult>;
     deleteEmbedding(id: number | string): Promise<{
         documentId: string;
         entries: import("@strapi/types/dist/modules/documents").AnyDocument[];
     }>;
-    updateEmbedding(id: string, data: UpdateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+    /**
+     * Find all chunks related to a parent document
+     * Returns chunks including the parent itself
+     */
+    findRelatedChunks(documentId: string): Promise<any[]>;
+    /**
+     * Delete all chunks related to a parent document
+     */
+    deleteRelatedChunks(documentId: string): Promise<number>;
+    /**
+     * Update embeddings with automatic chunking support
+     * Handles re-chunking when content changes and exceeds chunk size
+     */
+    updateChunkedEmbedding(id: string, data: UpdateEmbeddingData): Promise<ChunkedEmbeddingResult>;
+    updateEmbedding(id: string, data: UpdateEmbeddingData): Promise<any>;
     queryEmbeddings(query: string): Promise<import("../plugin-manager").QueryResponse | {
         error: string;
     }>;

package/dist/server/src/services/index.d.ts

@@ -2,12 +2,17 @@ declare const _default: {
     embeddings: ({ strapi }: {
         strapi: import("@strapi/types/dist/core").Strapi;
     }) => {
-        createEmbedding(data: import("./embeddings").CreateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+        getConfig(): import("../config").PluginConfigSchema;
+        createEmbedding(data: import("./embeddings").CreateEmbeddingData): Promise<any>;
+        createChunkedEmbedding(data: import("./embeddings").CreateEmbeddingData): Promise<import("./embeddings").ChunkedEmbeddingResult>;
         deleteEmbedding(id: string | number): Promise<{
             documentId: string;
             entries: import("@strapi/types/dist/modules/documents").AnyDocument[];
         }>;
-        updateEmbedding(id: string, data: import("./embeddings").UpdateEmbeddingData): Promise<import("@strapi/types/dist/modules/documents").AnyDocument>;
+        findRelatedChunks(documentId: string): Promise<any[]>;
+        deleteRelatedChunks(documentId: string): Promise<number>;
+        updateChunkedEmbedding(id: string, data: import("./embeddings").UpdateEmbeddingData): Promise<import("./embeddings").ChunkedEmbeddingResult>;
+        updateEmbedding(id: string, data: import("./embeddings").UpdateEmbeddingData): Promise<any>;
         queryEmbeddings(query: string): Promise<import("../plugin-manager").QueryResponse | {
             error: string;
         }>;
@@ -22,5 +27,22 @@ declare const _default: {
             totalCount: number;
         }>;
     };
+    sync: ({ strapi }: {
+        strapi: import("@strapi/types/dist/core").Strapi;
+    }) => {
+        syncFromNeon(options?: {
+            removeOrphans?: boolean;
+            dryRun?: boolean;
+        }): Promise<import("./sync").SyncResult>;
+        getSyncStatus(): Promise<{
+            neonCount: number;
+            strapiCount: number;
+            inSync: boolean;
+            missingInStrapi: number;
+            missingInNeon: number;
+            contentDifferences: number;
+        }>;
+        recreateAllEmbeddings(): Promise<import("./sync").RecreateResult>;
+    };
 };
 export default _default;

package/dist/server/src/services/sync.d.ts

@@ -0,0 +1,71 @@
+import type { Core } from "@strapi/strapi";
+export interface SyncResult {
+    success: boolean;
+    timestamp: string;
+    neonCount: number;
+    strapiCount: number;
+    actions: {
+        created: number;
+        updated: number;
+        orphansRemoved: number;
+    };
+    details: {
+        created: string[];
+        updated: string[];
+        orphansRemoved: string[];
+    };
+    errors: string[];
+}
+export interface RecreateResult {
+    success: boolean;
+    timestamp: string;
+    deletedFromNeon: number;
+    processedFromStrapi: number;
+    recreatedInNeon: number;
+    errors: string[];
+    details: {
+        recreated: string[];
+        failed: string[];
+    };
+}
+declare const sync: ({ strapi }: {
+    strapi: Core.Strapi;
+}) => {
+    /**
+     * Sync embeddings from Neon DB to Strapi DB
+     *
+     * This performs the following operations:
+     * 1. Fetches all embeddings from Neon DB (source of truth)
+     * 2. Fetches all embeddings from Strapi DB
+     * 3. Creates missing entries in Strapi that exist in Neon
+     * 4. Updates Strapi entries where content differs from Neon
+     * 5. Optionally removes orphaned Strapi entries (no matching Neon record)
+     */
+    syncFromNeon(options?: {
+        removeOrphans?: boolean;
+        dryRun?: boolean;
+    }): Promise<SyncResult>;
+    /**
+     * Get sync status - compare Neon and Strapi without making changes
+     */
+    getSyncStatus(): Promise<{
+        neonCount: number;
+        strapiCount: number;
+        inSync: boolean;
+        missingInStrapi: number;
+        missingInNeon: number;
+        contentDifferences: number;
+    }>;
+    /**
+     * Recreate all embeddings in Neon DB from Strapi data
+     *
+     * This will:
+     * 1. Delete ALL embeddings from Neon DB
+     * 2. Re-create embeddings for each Strapi embedding entry
+     * 3. Update Strapi entries with new embedding IDs
+     *
+     * Use this when embeddings were created with incorrect metadata format
+     */
+    recreateAllEmbeddings(): Promise<RecreateResult>;
+};
+export default sync;

package/dist/server/src/utils/chunking.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Text chunking utilities for splitting large content into embeddable chunks
+ */
+export interface ChunkOptions {
+    /** Maximum characters per chunk (default: 4000, roughly ~1000 tokens) */
+    chunkSize?: number;
+    /** Number of characters to overlap between chunks (default: 200) */
+    chunkOverlap?: number;
+    /** Separator to use when splitting (default: splits on paragraphs, sentences, then words) */
+    separators?: string[];
+}
+export interface TextChunk {
+    /** The chunk text content */
+    text: string;
+    /** Zero-based chunk index */
+    chunkIndex: number;
+    /** Total number of chunks */
+    totalChunks: number;
+    /** Character offset in original text */
+    startOffset: number;
+    /** Character end offset in original text */
+    endOffset: number;
+}
+/**
+ * Estimate token count from character count
+ * OpenAI models average ~4 characters per token for English text
+ */
+export declare function estimateTokens(text: string): number;
+/**
+ * Check if content exceeds the recommended chunk size
+ */
+export declare function needsChunking(content: string, maxChars?: number): boolean;
+/**
+ * Split content into chunks suitable for embedding
+ *
+ * @param content - The text content to split
+ * @param options - Chunking options
+ * @returns Array of TextChunk objects
+ */
+export declare function chunkContent(content: string, options?: ChunkOptions): TextChunk[];
+/**
+ * Format chunk title with index information
+ */
+export declare function formatChunkTitle(baseTitle: string, chunkIndex: number, totalChunks: number): string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strapi-content-embeddings",
-  "version": "0.1.4",
+  "version": "0.1.6",
   "description": "Strapi v5 plugin for vector embeddings with OpenAI and Neon PostgreSQL. Enables semantic search, RAG chat, and MCP (Model Context Protocol) integration.",
   "keywords": [
     "strapi",