s3db.js 12.2.3 → 12.2.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "12.2.3",
+  "version": "12.2.4",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",

package/src/plugins/vector.plugin.js

@@ -86,6 +86,7 @@ export class VectorPlugin extends Plugin {
    *
    * Detects large vector fields and warns if proper behavior is not set.
    * Can optionally auto-fix by setting body-overflow behavior.
+   * Auto-creates partitions for optional embedding fields to enable O(1) filtering.
    */
   validateVectorStorage() {
     for (const resource of Object.values(this.database.resources)) {
@@ -131,7 +132,278 @@ export class VectorPlugin extends Plugin {
           }
         }
       }
+
+      // Auto-create partitions for optional embedding fields
+      this.setupEmbeddingPartitions(resource, vectorFields);
+    }
+  }
+
+  /**
+   * Setup automatic partitions for optional embedding fields
+   *
+   * Creates a partition that separates records with embeddings from those without.
+   * This enables O(1) filtering instead of O(n) full scans when searching/clustering.
+   *
+   * @param {Resource} resource - Resource instance
+   * @param {Array} vectorFields - Detected vector fields with metadata
+   */
+  setupEmbeddingPartitions(resource, vectorFields) {
+    // Skip if resource doesn't have config (e.g., mocked resources)
+    if (!resource.config) return;
+
+    for (const vectorField of vectorFields) {
+      // Check if the vector field is optional
+      const isOptional = this.isFieldOptional(resource.schema.attributes, vectorField.name);
+
+      if (!isOptional) continue;
+
+      // Generate partition name
+      const partitionName = `byHas${this.capitalize(vectorField.name.replace(/\./g, '_'))}`;
+      const trackingFieldName = `_has${this.capitalize(vectorField.name.replace(/\./g, '_'))}`;
+
+      // Check if partition already exists
+      if (resource.config.partitions && resource.config.partitions[partitionName]) {
+        this.emit('vector:partition-exists', {
+          resource: resource.name,
+          vectorField: vectorField.name,
+          partition: partitionName,
+          timestamp: Date.now()
+        });
+        continue;
+      }
+
+      // Create partition configuration
+      if (!resource.config.partitions) {
+        resource.config.partitions = {};
+      }
+
+      resource.config.partitions[partitionName] = {
+        fields: {
+          [trackingFieldName]: 'boolean'
+        }
+      };
+
+      // Add tracking field to schema if not present
+      if (!resource.schema.attributes[trackingFieldName]) {
+        resource.schema.attributes[trackingFieldName] = {
+          type: 'boolean',
+          optional: true,
+          default: false
+        };
+      }
+
+      // Emit event
+      this.emit('vector:partition-created', {
+        resource: resource.name,
+        vectorField: vectorField.name,
+        partition: partitionName,
+        trackingField: trackingFieldName,
+        timestamp: Date.now()
+      });
+
+      console.log(`✅ VectorPlugin: Created partition '${partitionName}' for optional embedding field '${vectorField.name}' in resource '${resource.name}'`);
+
+      // Install hooks to maintain the partition
+      this.installEmbeddingHooks(resource, vectorField.name, trackingFieldName);
+    }
+  }
+
+  /**
+   * Check if a field is optional in the schema
+   *
+   * @param {Object} attributes - Resource attributes
+   * @param {string} fieldPath - Field path (supports dot notation)
+   * @returns {boolean} True if field is optional
+   */
+  isFieldOptional(attributes, fieldPath) {
+    const parts = fieldPath.split('.');
+    let current = attributes;
+
+    for (let i = 0; i < parts.length; i++) {
+      const part = parts[i];
+      const attr = current[part];
+
+      if (!attr) return true; // Field doesn't exist = optional
+
+      // Shorthand notation (e.g., 'string|required', 'embedding:1536')
+      if (typeof attr === 'string') {
+        const flags = attr.split('|');
+        // If it has 'required' flag, it's not optional
+        if (flags.includes('required')) return false;
+        // If it has 'optional' flag, it's optional
+        if (flags.includes('optional') || flags.some(f => f.startsWith('optional:'))) return true;
+        // By default, fields without 'required' are optional
+        return !flags.includes('required');
+      }
+
+      // Expanded notation (e.g., { type: 'string', optional: true })
+      if (typeof attr === 'object') {
+        // If we're at the last part, check if it's optional
+        if (i === parts.length - 1) {
+          // Explicit optional field
+          if (attr.optional === true) return true;
+          // Explicit required field
+          if (attr.optional === false) return false;
+          // Check for 'required' in nested object structure
+          // Default: optional unless explicitly marked as required
+          return attr.optional !== false;
+        }
+
+        // Navigate into nested object
+        if (attr.type === 'object' && attr.props) {
+          current = attr.props;
+        } else {
+          return true; // Can't navigate further = assume optional
+        }
+      }
     }
+
+    return true; // Default to optional
+  }
+
+  /**
+   * Capitalize first letter of string
+   *
+   * @param {string} str - Input string
+   * @returns {string} Capitalized string
+   */
+  capitalize(str) {
+    return str.charAt(0).toUpperCase() + str.slice(1);
+  }
+
+  /**
+   * Install hooks to maintain embedding partition tracking field
+   *
+   * @param {Resource} resource - Resource instance
+   * @param {string} vectorField - Vector field name
+   * @param {string} trackingField - Tracking field name
+   */
+  installEmbeddingHooks(resource, vectorField, trackingField) {
+    // beforeInsert: Set tracking field based on vector presence
+    resource.registerHook('beforeInsert', async (data) => {
+      const hasVector = this.hasVectorValue(data, vectorField);
+      this.setNestedValue(data, trackingField, hasVector);
+      return data;
+    });
+
+    // beforeUpdate: Update tracking field if vector changes
+    resource.registerHook('beforeUpdate', async (id, updates) => {
+      // Check if the vector field is being updated
+      if (vectorField in updates || this.hasNestedKey(updates, vectorField)) {
+        const hasVector = this.hasVectorValue(updates, vectorField);
+        this.setNestedValue(updates, trackingField, hasVector);
+      }
+      return updates;
+    });
+
+    this.emit('vector:hooks-installed', {
+      resource: resource.name,
+      vectorField,
+      trackingField,
+      hooks: ['beforeInsert', 'beforeUpdate'],
+      timestamp: Date.now()
+    });
+  }
+
+  /**
+   * Check if data has a valid vector value for the given field
+   *
+   * @param {Object} data - Data object
+   * @param {string} fieldPath - Field path (supports dot notation)
+   * @returns {boolean} True if vector exists and is valid
+   */
+  hasVectorValue(data, fieldPath) {
+    const value = this.getNestedValue(data, fieldPath);
+    return value != null && Array.isArray(value) && value.length > 0;
+  }
+
+  /**
+   * Check if object has a nested key
+   *
+   * @param {Object} obj - Object to check
+   * @param {string} path - Dot-notation path
+   * @returns {boolean} True if key exists
+   */
+  hasNestedKey(obj, path) {
+    const parts = path.split('.');
+    let current = obj;
+
+    for (const part of parts) {
+      if (current == null || typeof current !== 'object') return false;
+      if (!(part in current)) return false;
+      current = current[part];
+    }
+
+    return true;
+  }
+
+  /**
+   * Get nested value from object using dot notation
+   *
+   * @param {Object} obj - Object to traverse
+   * @param {string} path - Dot-notation path
+   * @returns {*} Value at path or undefined
+   */
+  getNestedValue(obj, path) {
+    const parts = path.split('.');
+    let current = obj;
+
+    for (const part of parts) {
+      if (current == null || typeof current !== 'object') return undefined;
+      current = current[part];
+    }
+
+    return current;
+  }
+
+  /**
+   * Set nested value in object using dot notation
+   *
+   * @param {Object} obj - Object to modify
+   * @param {string} path - Dot-notation path
+   * @param {*} value - Value to set
+   */
+  setNestedValue(obj, path, value) {
+    const parts = path.split('.');
+    let current = obj;
+
+    for (let i = 0; i < parts.length - 1; i++) {
+      const part = parts[i];
+      if (!(part in current) || typeof current[part] !== 'object') {
+        current[part] = {};
+      }
+      current = current[part];
+    }
+
+    current[parts[parts.length - 1]] = value;
+  }
+
+  /**
+   * Get auto-created embedding partition for a vector field
+   *
+   * Returns partition configuration if an auto-partition exists for the given vector field.
+   * Auto-partitions enable O(1) filtering to only records with embeddings.
+   *
+   * @param {Resource} resource - Resource instance
+   * @param {string} vectorField - Vector field name
+   * @returns {Object|null} Partition config or null
+   */
+  getAutoEmbeddingPartition(resource, vectorField) {
+    // Skip if resource doesn't have config (e.g., mocked resources)
+    if (!resource.config) return null;
+
+    const partitionName = `byHas${this.capitalize(vectorField.replace(/\./g, '_'))}`;
+    const trackingFieldName = `_has${this.capitalize(vectorField.replace(/\./g, '_'))}`;
+
+    // Check if auto-partition exists
+    if (resource.config.partitions && resource.config.partitions[partitionName]) {
+      return {
+        partitionName,
+        partitionValues: { [trackingFieldName]: true }
+      };
+    }
+
+    return null;
   }
 
   /**
@@ -313,11 +585,12 @@ export class VectorPlugin extends Plugin {
         vectorField = 'vector'; // Fallback to default
       }
 
-      const {
+      let {
         limit = 10,
         distanceMetric = this.config.distanceMetric,
         threshold = null,
-        partition = null
+        partition = null,
+        partitionValues = null
       } = options;
 
       const distanceFn = this.distanceFunctions[distanceMetric];
@@ -337,6 +610,23 @@ export class VectorPlugin extends Plugin {
         throw error;
       }
 
+      // Auto-use embedding partition if available and no custom partition specified
+      if (!partition) {
+        const autoPartition = this.getAutoEmbeddingPartition(resource, vectorField);
+        if (autoPartition) {
+          partition = autoPartition.partitionName;
+          partitionValues = autoPartition.partitionValues;
+
+          this._emitEvent('vector:auto-partition-used', {
+            resource: resource.name,
+            vectorField,
+            partition,
+            partitionValues,
+            timestamp: Date.now()
+          });
+        }
+      }
+
       // Emit start event
       this._emitEvent('vector:search-start', {
         resource: resource.name,
@@ -344,6 +634,7 @@ export class VectorPlugin extends Plugin {
         limit,
         distanceMetric,
         partition,
+        partitionValues,
         threshold,
         queryDimensions: queryVector.length,
         timestamp: startTime
@@ -352,21 +643,41 @@ export class VectorPlugin extends Plugin {
       try {
         // Get all records (with optional partition filter)
         let allRecords;
-        if (partition) {
+        if (partition && partitionValues) {
           this._emitEvent('vector:partition-filter', {
             resource: resource.name,
             partition,
+            partitionValues,
             timestamp: Date.now()
           });
-          allRecords = await resource.list({ partition, partitionValues: partition });
+          allRecords = await resource.list({ partition, partitionValues });
         } else {
-          allRecords = await resource.getAll();
+          // Fallback to list() if getAll() doesn't exist (for mocked resources in tests)
+          allRecords = resource.getAll ? await resource.getAll() : await resource.list();
         }
 
         const totalRecords = allRecords.length;
         let processedRecords = 0;
         let dimensionMismatches = 0;
 
+        // Performance warning for large resources without partition
+        if (!partition && totalRecords > 1000) {
+          const warning = {
+            resource: resource.name,
+            operation: 'vectorSearch',
+            totalRecords,
+            vectorField,
+            recommendation: 'Use partitions to filter data before vector search for better performance'
+          };
+
+          this._emitEvent('vector:performance-warning', warning);
+
+          console.warn(`⚠️  VectorPlugin: Performing vectorSearch on ${totalRecords} records without partition filter`);
+          console.warn(`   Resource: '${resource.name}'`);
+          console.warn(`   Recommendation: Use partition parameter to reduce search space`);
+          console.warn(`   Example: resource.vectorSearch(vector, { partition: 'byCategory', partitionValues: { category: 'books' } })`);
+        }
+
         // Calculate distances
         const results = allRecords
           .filter(record => record[vectorField] && Array.isArray(record[vectorField]))
@@ -473,10 +784,11 @@ export class VectorPlugin extends Plugin {
         vectorField = 'vector'; // Fallback to default
       }
 
-      const {
+      let {
         k = 5,
         distanceMetric = this.config.distanceMetric,
         partition = null,
+        partitionValues = null,
         ...kmeansOptions
       } = options;
 
@@ -497,6 +809,23 @@ export class VectorPlugin extends Plugin {
         throw error;
       }
 
+      // Auto-use embedding partition if available and no custom partition specified
+      if (!partition) {
+        const autoPartition = this.getAutoEmbeddingPartition(resource, vectorField);
+        if (autoPartition) {
+          partition = autoPartition.partitionName;
+          partitionValues = autoPartition.partitionValues;
+
+          this._emitEvent('vector:auto-partition-used', {
+            resource: resource.name,
+            vectorField,
+            partition,
+            partitionValues,
+            timestamp: Date.now()
+          });
+        }
+      }
+
       // Emit start event
       this._emitEvent('vector:cluster-start', {
         resource: resource.name,
@@ -504,6 +833,7 @@ export class VectorPlugin extends Plugin {
         k,
         distanceMetric,
         partition,
+        partitionValues,
         maxIterations: kmeansOptions.maxIterations || 100,
         timestamp: startTime
       });
@@ -511,15 +841,17 @@ export class VectorPlugin extends Plugin {
       try {
         // Get all records (with optional partition filter)
         let allRecords;
-        if (partition) {
+        if (partition && partitionValues) {
           this._emitEvent('vector:partition-filter', {
             resource: resource.name,
             partition,
+            partitionValues,
             timestamp: Date.now()
           });
-          allRecords = await resource.list({ partition, partitionValues: partition });
+          allRecords = await resource.list({ partition, partitionValues });
         } else {
-          allRecords = await resource.getAll();
+          // Fallback to list() if getAll() doesn't exist (for mocked resources in tests)
+          allRecords = resource.getAll ? await resource.getAll() : await resource.list();
         }
 
         // Extract vectors
@@ -527,6 +859,26 @@ export class VectorPlugin extends Plugin {
           record => record[vectorField] && Array.isArray(record[vectorField])
         );
 
+        // Performance warning for large resources without partition
+        if (!partition && allRecords.length > 1000) {
+          const warning = {
+            resource: resource.name,
+            operation: 'cluster',
+            totalRecords: allRecords.length,
+            recordsWithVectors: recordsWithVectors.length,
+            vectorField,
+            recommendation: 'Use partitions to filter data before clustering for better performance'
+          };
+
+          this._emitEvent('vector:performance-warning', warning);
+
+          console.warn(`⚠️  VectorPlugin: Performing clustering on ${allRecords.length} records without partition filter`);
+          console.warn(`   Resource: '${resource.name}'`);
+          console.warn(`   Records with vectors: ${recordsWithVectors.length}`);
+          console.warn(`   Recommendation: Use partition parameter to reduce clustering space`);
+          console.warn(`   Example: resource.cluster({ k: 5, partition: 'byCategory', partitionValues: { category: 'books' } })`);
+        }
+
         if (recordsWithVectors.length === 0) {
           const error = new VectorError('No vectors found in resource', {
             operation: 'cluster',