s3db.js 12.2.4 → 12.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "12.2.4",
+  "version": "12.3.0",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",

package/src/behaviors/body-only.js

@@ -59,10 +59,15 @@ export async function handleInsert({ resource, data, mappedData }) {
     '_v': mappedData._v || String(resource.version)
   };
   metadataOnly._map = JSON.stringify(resource.schema.map);
-  
+
+  // Store pluginMap for backwards compatibility when plugins are added/removed
+  if (resource.schema.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
+    metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
+  }
+
   // Use the original object for the body
   const body = JSON.stringify(mappedData);
-  
+
   return { mappedData: metadataOnly, body };
 }
 
@@ -70,16 +75,21 @@ export async function handleUpdate({ resource, id, data, mappedData }) {
   // For updates, we need to merge with existing data
   // Since we can't easily read the existing body during update,
   // we'll put the update data in the body and let the resource handle merging
-  
+
   // Keep only the version field in metadata
   const metadataOnly = {
     '_v': mappedData._v || String(resource.version)
   };
   metadataOnly._map = JSON.stringify(resource.schema.map);
-  
+
+  // Store pluginMap for backwards compatibility when plugins are added/removed
+  if (resource.schema.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
+    metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
+  }
+
   // Use the original object for the body
   const body = JSON.stringify(mappedData);
-  
+
   return { mappedData: metadataOnly, body };
 }
 

package/src/behaviors/body-overflow.js

@@ -90,6 +90,15 @@ export async function handleInsert({ resource, data, mappedData, originalData })
     currentSize += attributeSizes._v;
   }
 
+  // Always include plugin map for backwards compatibility when plugins are added/removed
+  // Note: _pluginMap is not in mappedData, it's added separately by behaviors
+  if (resource.schema?.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
+    const pluginMapStr = JSON.stringify(resource.schema.pluginMap);
+    const pluginMapSize = calculateUTF8Bytes('_pluginMap') + calculateUTF8Bytes(pluginMapStr);
+    metadataFields._pluginMap = pluginMapStr;
+    currentSize += pluginMapSize;
+  }
+
   // Reserve space for $overflow if overflow is possible
   let reservedLimit = effectiveLimit;
   for (const [fieldName, size] of sortedFields) {

package/src/behaviors/user-managed.js

@@ -88,7 +88,14 @@ export async function handleInsert({ resource, data, mappedData, originalData })
       data: originalData || data
     });
     // If data exceeds limit, store in body
-    return { mappedData: { _v: mappedData._v }, body: JSON.stringify(mappedData) };
+    const metadataOnly = { _v: mappedData._v };
+
+    // Store pluginMap for backwards compatibility when plugins are added/removed
+    if (resource.schema?.pluginMap && Object.keys(resource.schema.pluginMap).length > 0) {
+      metadataOnly._pluginMap = JSON.stringify(resource.schema.pluginMap);
+    }
+
+    return { mappedData: metadataOnly, body: JSON.stringify(mappedData) };
   }
   
   // If data fits in metadata, store only in metadata

package/src/concerns/typescript-generator.js

@@ -190,10 +190,20 @@ export async function generateTypes(database, options = {}) {
   const resourceInterfaces = [];
 
   for (const [name, resource] of Object.entries(database.resources)) {
-    const attributes = resource.config?.attributes || resource.attributes || {};
+    const allAttributes = resource.config?.attributes || resource.attributes || {};
     const timestamps = resource.config?.timestamps || false;
 
-    const interfaceDef = generateResourceInterface(name, attributes, timestamps);
+    // Filter out plugin attributes - they are internal implementation details
+    // and should not be exposed in public TypeScript interfaces
+    const pluginAttrNames = resource.schema?._pluginAttributes
+      ? Object.values(resource.schema._pluginAttributes).flat()
+      : [];
+
+    const userAttributes = Object.fromEntries(
+      Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+    );
+
+    const interfaceDef = generateResourceInterface(name, userAttributes, timestamps);
     lines.push(interfaceDef);
 
     resourceInterfaces.push({

package/src/plugins/api/utils/openapi-generator.js

@@ -98,7 +98,17 @@ function generateResourceSchema(resource) {
   const properties = {};
   const required = [];
 
-  const attributes = resource.config?.attributes || resource.attributes || {};
+  const allAttributes = resource.config?.attributes || resource.attributes || {};
+
+  // Filter out plugin attributes - they are internal implementation details
+  // and should not be exposed in public API documentation
+  const pluginAttrNames = resource.schema?._pluginAttributes
+    ? Object.values(resource.schema._pluginAttributes).flat()
+    : [];
+
+  const attributes = Object.fromEntries(
+    Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+  );
 
   // Extract resource description (supports both string and object format)
   const resourceDescription = resource.config?.description;
@@ -254,7 +264,16 @@ function generateResourcePaths(resource, version, config = {}) {
     }
 
     // Create query parameters only for partition fields
-    const attributes = resource.config?.attributes || resource.attributes || {};
+    const allAttributes = resource.config?.attributes || resource.attributes || {};
+
+    // Filter out plugin attributes
+    const pluginAttrNames = resource.schema?._pluginAttributes
+      ? Object.values(resource.schema._pluginAttributes).flat()
+      : [];
+
+    const attributes = Object.fromEntries(
+      Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+    );
 
     for (const fieldName of partitionFieldsSet) {
       const fieldDef = attributes[fieldName];

package/src/plugins/replicators/bigquery-replicator.class.js

@@ -186,7 +186,15 @@ class BigqueryReplicator extends BaseReplicator {
         continue;
       }
 
-      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      // Filter out plugin attributes - they are internal and should not be replicated
+      const pluginAttrNames = resource.schema?._pluginAttributes
+        ? Object.values(resource.schema._pluginAttributes).flat()
+        : [];
+      const attributes = Object.fromEntries(
+        Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+      );
 
       for (const tableConfig of tableConfigs) {
         const tableName = tableConfig.table;

package/src/plugins/replicators/mysql-replicator.class.js

@@ -221,7 +221,15 @@ class MySQLReplicator extends BaseReplicator {
         continue;
       }
 
-      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      // Filter out plugin attributes - they are internal and should not be replicated
+      const pluginAttrNames = resource.schema?._pluginAttributes
+        ? Object.values(resource.schema._pluginAttributes).flat()
+        : [];
+      const attributes = Object.fromEntries(
+        Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+      );
 
       for (const tableConfig of tableConfigs) {
         const tableName = tableConfig.table;

package/src/plugins/replicators/planetscale-replicator.class.js

@@ -183,7 +183,15 @@ class PlanetScaleReplicator extends BaseReplicator {
         continue;
       }
 
-      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      // Filter out plugin attributes - they are internal and should not be replicated
+      const pluginAttrNames = resource.schema?._pluginAttributes
+        ? Object.values(resource.schema._pluginAttributes).flat()
+        : [];
+      const attributes = Object.fromEntries(
+        Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+      );
 
       for (const tableConfig of tableConfigs) {
         const tableName = tableConfig.table;

package/src/plugins/replicators/postgres-replicator.class.js

@@ -225,7 +225,15 @@ class PostgresReplicator extends BaseReplicator {
       }
 
       // Get resource attributes from current version
-      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      // Filter out plugin attributes - they are internal and should not be replicated
+      const pluginAttrNames = resource.schema?._pluginAttributes
+        ? Object.values(resource.schema._pluginAttributes).flat()
+        : [];
+      const attributes = Object.fromEntries(
+        Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+      );
 
       // Sync each table configured for this resource
       for (const tableConfig of tableConfigs) {

package/src/plugins/replicators/schema-sync.helper.js

@@ -7,6 +7,25 @@
 
 import tryFn from "#src/concerns/try-fn.js";
 
+/**
+ * Filter out plugin attributes from attributes object
+ * Plugin attributes are internal implementation details and should not be replicated
+ * @param {Object} attributes - All attributes including plugin attributes
+ * @param {Object} resource - Resource instance with schema._pluginAttributes
+ * @returns {Object} Filtered attributes (user attributes only)
+ */
+function filterPluginAttributes(attributes, resource) {
+  if (!resource?.schema?._pluginAttributes) {
+    return attributes;
+  }
+
+  const pluginAttrNames = Object.values(resource.schema._pluginAttributes).flat();
+
+  return Object.fromEntries(
+    Object.entries(attributes).filter(([name]) => !pluginAttrNames.includes(name))
+  );
+}
+
 /**
  * Parse s3db field type notation (e.g., 'string|required|maxlength:50')
  */

package/src/plugins/replicators/turso-replicator.class.js

@@ -177,7 +177,15 @@ class TursoReplicator extends BaseReplicator {
         continue;
       }
 
-      const attributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+      const allAttributes = resource.config.versions[resource.config.currentVersion]?.attributes || {};
+
+      // Filter out plugin attributes - they are internal and should not be replicated
+      const pluginAttrNames = resource.schema?._pluginAttributes
+        ? Object.values(resource.schema._pluginAttributes).flat()
+        : [];
+      const attributes = Object.fromEntries(
+        Object.entries(allAttributes).filter(([name]) => !pluginAttrNames.includes(name))
+      );
 
       for (const tableConfig of tableConfigs) {
         const tableName = tableConfig.table;

package/src/plugins/vector.plugin.js

@@ -183,13 +183,13 @@ export class VectorPlugin extends Plugin {
         }
       };
 
-      // Add tracking field to schema if not present
+      // Add tracking field to schema if not present using plugin API
       if (!resource.schema.attributes[trackingFieldName]) {
-        resource.schema.attributes[trackingFieldName] = {
+        resource.addPluginAttribute(trackingFieldName, {
           type: 'boolean',
           optional: true,
           default: false
-        };
+        }, 'VectorPlugin');
       }
 
       // Emit event

package/src/resource.class.js

@@ -7,6 +7,7 @@ import { PromisePool } from "@supercharge/promise-pool";
 import { chunk, cloneDeep, merge, isEmpty, isObject } from "lodash-es";
 
 import Schema from "./schema.class.js";
+import { ValidatorManager } from "./validator.class.js";
 import { streamToString } from "./stream/index.js";
 import tryFn, { tryFnSync } from "./concerns/try-fn.js";
 import { ResourceReader, ResourceWriter } from "./stream/index.js"
@@ -425,6 +426,164 @@ export class Resource extends AsyncEventEmitter {
     return { oldAttributes, newAttributes };
   }
 
+  /**
+   * Add a plugin-created attribute to the resource schema
+   * This ensures plugin attributes don't interfere with user-defined attributes
+   * by using a separate mapping namespace (p0, p1, p2, ...)
+   *
+   * @param {string} name - Attribute name (e.g., '_hasEmbedding', 'clusterId')
+   * @param {Object|string} definition - Attribute definition
+   * @param {string} pluginName - Name of plugin adding the attribute
+   * @returns {void}
+   *
+   * @example
+   * // VectorPlugin adding tracking field
+   * resource.addPluginAttribute('_hasEmbedding', {
+   *   type: 'boolean',
+   *   optional: true,
+   *   default: false
+   * }, 'VectorPlugin');
+   *
+   * // Shorthand notation
+   * resource.addPluginAttribute('clusterId', 'string|optional', 'VectorPlugin');
+   */
+  addPluginAttribute(name, definition, pluginName) {
+    if (!pluginName) {
+      throw new ResourceError(
+        'Plugin name is required when adding plugin attributes',
+        { resource: this.name, attribute: name }
+      );
+    }
+
+    // If attribute already exists and is not a plugin attribute, throw error
+    const existingDef = this.schema.getAttributeDefinition(name);
+    if (existingDef && (!existingDef.__plugin__ || existingDef.__plugin__ !== pluginName)) {
+      throw new ResourceError(
+        `Attribute '${name}' already exists and is not from plugin '${pluginName}'`,
+        { resource: this.name, attribute: name, plugin: pluginName }
+      );
+    }
+
+    // Use the definition as-is (string or object)
+    // The schema preprocessor will handle string notation validation
+    let defObject = definition;
+    if (typeof definition === 'object' && definition !== null) {
+      // Clone to avoid mutation
+      defObject = { ...definition };
+    }
+
+    // Mark as plugin-created with metadata
+    // For string definitions, we need to preserve them but track plugin ownership separately
+    if (typeof defObject === 'object' && defObject !== null) {
+      defObject.__plugin__ = pluginName;
+      defObject.__pluginCreated__ = Date.now();
+    }
+
+    // Add to schema attributes
+    // Store original definition (string or object) as the validator expects
+    this.schema.attributes[name] = defObject;
+
+    // Also update resource.attributes to keep them in sync
+    this.attributes[name] = defObject;
+
+    // For string definitions, add metadata separately
+    if (typeof defObject === 'string') {
+      // Create a marker object to track plugin ownership in a parallel structure
+      if (!this.schema._pluginAttributeMetadata) {
+        this.schema._pluginAttributeMetadata = {};
+      }
+      this.schema._pluginAttributeMetadata[name] = {
+        __plugin__: pluginName,
+        __pluginCreated__: Date.now()
+      };
+    }
+
+    // Regenerate plugin mapping only (not user mapping)
+    this.schema.regeneratePluginMapping();
+
+    // Regenerate hooks for the new attribute
+    if (this.schema.options.generateAutoHooks) {
+      this.schema.generateAutoHooks();
+    }
+
+    // Recompile validator to include new attribute
+    const processedAttributes = this.schema.preprocessAttributesForValidation(this.schema.attributes);
+    this.schema.validator = new ValidatorManager({ autoEncrypt: false }).compile(merge(
+      { $$async: true, $$strict: false },
+      processedAttributes
+    ));
+
+    // Emit event
+    if (this.database) {
+      this.database.emit('plugin-attribute-added', {
+        resource: this.name,
+        attribute: name,
+        plugin: pluginName,
+        definition: defObject
+      });
+    }
+  }
+
+  /**
+   * Remove a plugin-created attribute from the resource schema
+   * Called when a plugin is uninstalled or no longer needs the attribute
+   *
+   * @param {string} name - Attribute name to remove
+   * @param {string} [pluginName] - Optional plugin name for safety check
+   * @returns {boolean} True if attribute was removed, false if not found
+   *
+   * @example
+   * resource.removePluginAttribute('_hasEmbedding', 'VectorPlugin');
+   */
+  removePluginAttribute(name, pluginName = null) {
+    const attrDef = this.schema.getAttributeDefinition(name);
+
+    // Check metadata for string definitions
+    const metadata = this.schema._pluginAttributeMetadata?.[name];
+    const isPluginAttr = (typeof attrDef === 'object' && attrDef?.__plugin__) || metadata;
+
+    // Check if attribute exists and is a plugin attribute
+    if (!attrDef || !isPluginAttr) {
+      return false;
+    }
+
+    // Get plugin name from either object or metadata
+    const actualPlugin = attrDef?.__plugin__ || metadata?.__plugin__;
+
+    // Safety check: if pluginName provided, ensure it matches
+    if (pluginName && actualPlugin !== pluginName) {
+      throw new ResourceError(
+        `Attribute '${name}' belongs to plugin '${actualPlugin}', not '${pluginName}'`,
+        { resource: this.name, attribute: name, actualPlugin, requestedPlugin: pluginName }
+      );
+    }
+
+    // Remove from schema
+    delete this.schema.attributes[name];
+
+    // Also remove from resource.attributes to keep them in sync
+    delete this.attributes[name];
+
+    // Remove metadata if it exists
+    if (this.schema._pluginAttributeMetadata?.[name]) {
+      delete this.schema._pluginAttributeMetadata[name];
+    }
+
+    // Regenerate plugin mapping
+    this.schema.regeneratePluginMapping();
+
+    // Emit event
+    if (this.database) {
+      this.database.emit('plugin-attribute-removed', {
+        resource: this.name,
+        attribute: name,
+        plugin: actualPlugin
+      });
+    }
+
+    return true;
+  }
+
   /**
    * Add a hook function for a specific event
    * @param {string} event - Hook event (beforeInsert, afterInsert, etc.)
@@ -3284,11 +3443,17 @@ export class Resource extends AsyncEventEmitter {
     unmappedMetadata = ok ? unmapped : metadata;
     // Helper function to filter out internal S3DB fields
     // Preserve geo-related fields (_geohash, _geohash_zoom*) for GeoPlugin
+    // Preserve plugin attributes (fields in _pluginAttributes)
     const filterInternalFields = (obj) => {
       if (!obj || typeof obj !== 'object') return obj;
       const filtered = {};
+      const pluginAttrNames = this.schema._pluginAttributes
+        ? Object.values(this.schema._pluginAttributes).flat()
+        : [];
+
       for (const [key, value] of Object.entries(obj)) {
-        if (!key.startsWith('_') || key === '_geohash' || key.startsWith('_geohash_zoom')) {
+        // Keep field if it doesn't start with _, or if it's a special field, or if it's a plugin attribute
+        if (!key.startsWith('_') || key === '_geohash' || key.startsWith('_geohash_zoom') || pluginAttrNames.includes(key)) {
           filtered[key] = value;
         }
       }
@@ -3315,7 +3480,19 @@ export class Resource extends AsyncEventEmitter {
       if (hasOverflow && body) {
         const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(JSON.parse(body)));
         if (okBody) {
-          const [okUnmap, errUnmap, unmappedBody] = await tryFn(() => this.schema.unmapper(parsedBody));
+          // Extract pluginMap for backwards compatibility when plugins are added/removed
+          let pluginMapFromMeta = null;
+          // S3 metadata keys are case-insensitive and stored as lowercase
+          if (metadata && metadata._pluginmap) {
+            const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(() =>
+              Promise.resolve(typeof metadata._pluginmap === 'string' ? JSON.parse(metadata._pluginmap) : metadata._pluginmap)
+            );
+            pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
+          }
+
+          const [okUnmap, errUnmap, unmappedBody] = await tryFn(() =>
+            this.schema.unmapper(parsedBody, undefined, pluginMapFromMeta)
+          );
           bodyData = okUnmap ? unmappedBody : {};
         }
       }
@@ -3330,11 +3507,21 @@ export class Resource extends AsyncEventEmitter {
     if (behavior === 'body-only') {
       const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(body ? JSON.parse(body) : {}));
       let mapFromMeta = this.schema.map;
+      let pluginMapFromMeta = null;
+
       if (metadata && metadata._map) {
         const [okMap, errMap, parsedMap] = await tryFn(() => Promise.resolve(typeof metadata._map === 'string' ? JSON.parse(metadata._map) : metadata._map));
         mapFromMeta = okMap ? parsedMap : this.schema.map;
       }
-      const [okUnmap, errUnmap, unmappedBody] = await tryFn(() => this.schema.unmapper(parsedBody, mapFromMeta));
+
+      // S3 metadata keys are case-insensitive and stored as lowercase
+      // So _pluginMap becomes _pluginmap
+      if (metadata && metadata._pluginmap) {
+        const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(() => Promise.resolve(typeof metadata._pluginmap === 'string' ? JSON.parse(metadata._pluginmap) : metadata._pluginmap));
+        pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
+      }
+
+      const [okUnmap, errUnmap, unmappedBody] = await tryFn(() => this.schema.unmapper(parsedBody, mapFromMeta, pluginMapFromMeta));
       const result = okUnmap ? { ...unmappedBody, id } : { id };
       Object.keys(result).forEach(k => { result[k] = fixValue(result[k]); });
       return result;
@@ -3344,7 +3531,19 @@ export class Resource extends AsyncEventEmitter {
     if (behavior === 'user-managed' && body && body.trim() !== '') {
       const [okBody, errBody, parsedBody] = await tryFn(() => Promise.resolve(JSON.parse(body)));
       if (okBody) {
-        const [okUnmap, errUnmap, unmappedBody] = await tryFn(() => this.schema.unmapper(parsedBody));
+        // Extract pluginMap for backwards compatibility when plugins are added/removed
+        let pluginMapFromMeta = null;
+        // S3 metadata keys are case-insensitive and stored as lowercase
+        if (metadata && metadata._pluginmap) {
+          const [okPluginMap, errPluginMap, parsedPluginMap] = await tryFn(() =>
+            Promise.resolve(typeof metadata._pluginmap === 'string' ? JSON.parse(metadata._pluginmap) : metadata._pluginmap)
+          );
+          pluginMapFromMeta = okPluginMap ? parsedPluginMap : null;
+        }
+
+        const [okUnmap, errUnmap, unmappedBody] = await tryFn(() =>
+          this.schema.unmapper(parsedBody, undefined, pluginMapFromMeta)
+        );
         const bodyData = okUnmap ? unmappedBody : {};
         const merged = { ...bodyData, ...unmappedMetadata, id };
         Object.keys(merged).forEach(k => { merged[k] = fixValue(merged[k]); });