s3db.js 6.2.0 → 7.0.0

package/src/behaviors/truncate-data.js

@@ -0,0 +1,204 @@
+import { calculateTotalSize, calculateAttributeSizes, calculateUTF8Bytes } from '../concerns/calculator.js';
+import { calculateEffectiveLimit } from '../concerns/calculator.js';
+import { S3_METADATA_LIMIT_BYTES } from './enforce-limits.js';
+
+const TRUNCATED_FLAG = '$truncated';
+const TRUNCATED_FLAG_VALUE = 'true';
+const TRUNCATED_FLAG_BYTES = calculateUTF8Bytes(TRUNCATED_FLAG) + calculateUTF8Bytes(TRUNCATED_FLAG_VALUE);
+
+/**
+ * Data Truncate Behavior Configuration Documentation
+ *
+ * The `truncate-data` behavior optimizes metadata usage by sorting attributes by size
+ * in ascending order and truncating the last attribute that fits within the available
+ * space. This ensures all data stays in metadata for fast access while respecting
+ * S3 metadata size limits.
+ *
+ * ## Purpose & Use Cases
+ * - When you need fast access to all data (no body reads required)
+ * - For objects that slightly exceed metadata limits
+ * - When data loss through truncation is acceptable
+ * - For frequently accessed data where performance is critical
+ *
+ * ## How It Works
+ * 1. Calculates the size of each attribute
+ * 2. Sorts attributes by size in ascending order (smallest first)
+ * 3. Fills metadata with small attributes until limit is approached
+ * 4. Truncates the last attribute that fits to maximize data retention
+ * 5. Adds a `$truncated` flag to indicate truncation occurred
+ *
+ * ## Performance Characteristics
+ * - Fastest possible access (all data in metadata)
+ * - No body reads required
+ * - Potential data loss through truncation
+ * - Optimal for frequently accessed data
+ *
+ * @example
+ * // Create a resource with truncate-data behavior
+ * const resource = await db.createResource({
+ *   name: 'fast_access_data',
+ *   attributes: { ... },
+ *   behavior: 'truncate-data'
+ * });
+ *
+ * // Small fields stay intact, large fields get truncated
+ * const doc = await resource.insert({
+ *   id: 'doc123',           // Small -> intact
+ *   title: 'Short Title',   // Small -> intact
+ *   content: 'Very long...', // Large -> truncated
+ *   metadata: { ... }       // Large -> truncated
+ * });
+ *
+ * ## Comparison to Other Behaviors
+ * | Behavior         | Metadata Usage | Body Usage | Size Limits | Performance |
+ * |------------------|----------------|------------|-------------|-------------|
+ * | truncate-data    | All (truncated)| None       | 2KB metadata | Fast reads   |
+ * | body-overflow    | Optimized      | Overflow   | 2KB metadata | Balanced     |
+ * | body-only        | Minimal (_v)   | All data   | 5TB         | Slower reads |
+ * | enforce-limits   | All (limited)  | None       | 2KB metadata | Fast reads   |
+ * | user-managed     | All (unlimited)| None       | S3 limit    | Fast reads   |
+ *
+ * @typedef {Object} DataTruncateBehaviorConfig
+ * @property {boolean} [enabled=true] - Whether the behavior is active
+ * @property {string} [truncateIndicator='...'] - String to append when truncating
+ * @property {string[]} [priorityFields] - Fields that should not be truncated
+ * @property {boolean} [preserveStructure=true] - Whether to preserve JSON structure
+ */
+export async function handleInsert({ resource, data, mappedData, originalData }) {
+  const effectiveLimit = calculateEffectiveLimit({
+    s3Limit: S3_METADATA_LIMIT_BYTES,
+    systemConfig: {
+      version: resource.version,
+      timestamps: resource.config.timestamps,
+      id: data.id
+    }
+  });
+
+  const attributeSizes = calculateAttributeSizes(mappedData);
+  const sortedFields = Object.entries(attributeSizes)
+    .sort(([, a], [, b]) => a - b);
+
+  const resultFields = {};
+  let currentSize = 0;
+  let truncated = false;
+
+  // Always include version field first
+  if (mappedData._v) {
+    resultFields._v = mappedData._v;
+    currentSize += attributeSizes._v;
+  }
+
+  // Add fields to metadata until we reach the limit
+  for (const [fieldName, size] of sortedFields) {
+    if (fieldName === '_v') continue;
+    
+    const fieldValue = mappedData[fieldName];
+    const spaceNeeded = size + (truncated ? 0 : TRUNCATED_FLAG_BYTES);
+    
+    if (currentSize + spaceNeeded <= effectiveLimit) {
+      // Field fits completely
+      resultFields[fieldName] = fieldValue;
+      currentSize += size;
+    } else {
+      // Field needs to be truncated
+      const availableSpace = effectiveLimit - currentSize - (truncated ? 0 : TRUNCATED_FLAG_BYTES);
+      if (availableSpace > 0) {
+        // We can fit part of this field
+        const truncatedValue = truncateValue(fieldValue, availableSpace);
+        resultFields[fieldName] = truncatedValue;
+        truncated = true;
+        currentSize += calculateUTF8Bytes(truncatedValue);
+      } else {
+        // Field doesn't fit at all, but keep it as empty string
+        resultFields[fieldName] = '';
+        truncated = true;
+      }
+      // Stop processing - we've reached the limit
+      break;
+    }
+  }
+
+  // Verify we're within limits and adjust if necessary
+  let finalSize = calculateTotalSize(resultFields) + (truncated ? TRUNCATED_FLAG_BYTES : 0);
+  
+  // If still over limit, keep removing/truncating fields until we fit
+  while (finalSize > effectiveLimit) {
+    const fieldNames = Object.keys(resultFields).filter(f => f !== '_v' && f !== '$truncated');
+    if (fieldNames.length === 0) {
+      // Only version field remains, this shouldn't happen but just in case
+      break;
+    }
+    
+    // Remove the last field but keep it as empty string
+    const lastField = fieldNames[fieldNames.length - 1];
+    resultFields[lastField] = '';
+    
+    // Recalculate size
+    finalSize = calculateTotalSize(resultFields) + TRUNCATED_FLAG_BYTES;
+    truncated = true;
+  }
+
+  if (truncated) {
+    resultFields[TRUNCATED_FLAG] = TRUNCATED_FLAG_VALUE;
+  }
+
+  return { mappedData: resultFields, body: JSON.stringify(mappedData) };
+}
+
+export async function handleUpdate({ resource, id, data, mappedData, originalData }) {
+  return handleInsert({ resource, data, mappedData, originalData });
+}
+
+export async function handleUpsert({ resource, id, data, mappedData }) {
+  return handleInsert({ resource, data, mappedData });
+}
+
+export async function handleGet({ resource, metadata, body }) {
+  // For truncate-data, all data is in metadata, no body processing needed
+  return { metadata, body };
+}
+
+/**
+ * Truncate a value to fit within the specified byte limit
+ * @param {any} value - The value to truncate
+ * @param {number} maxBytes - Maximum bytes allowed
+ * @returns {any} - Truncated value
+ */
+function truncateValue(value, maxBytes) {
+  if (typeof value === 'string') {
+    return truncateString(value, maxBytes);
+  } else if (typeof value === 'object' && value !== null) {
+    // Truncar objeto como JSON string truncada
+    const jsonStr = JSON.stringify(value);
+    return truncateString(jsonStr, maxBytes);
+  } else {
+    // Para números, booleanos, etc., converte para string e trunca
+    const stringValue = String(value);
+    return truncateString(stringValue, maxBytes);
+  }
+}
+
+/**
+ * Truncate a string to fit within byte limit
+ * @param {string} str - String to truncate
+ * @param {number} maxBytes - Maximum bytes allowed
+ * @returns {string} - Truncated string
+ */
+function truncateString(str, maxBytes) {
+  const encoder = new TextEncoder();
+  let bytes = encoder.encode(str);
+  if (bytes.length <= maxBytes) {
+    return str;
+  }
+  // Trunca sem adicionar '...'
+  let length = str.length;
+  while (length > 0) {
+    const truncated = str.substring(0, length);
+    bytes = encoder.encode(truncated);
+    if (bytes.length <= maxBytes) {
+      return truncated;
+    }
+    length--;
+  }
+  return '';
+}

package/src/behaviors/user-managed.js

@@ -0,0 +1,147 @@
+import { calculateTotalSize } from '../concerns/calculator.js';
+import { calculateEffectiveLimit } from '../concerns/calculator.js';
+import { S3_METADATA_LIMIT_BYTES } from './enforce-limits.js';
+
+/**
+ * User Managed Behavior Configuration Documentation
+ *
+ * The `user-managed` behavior is the default for s3db resources. It provides no automatic enforcement
+ * of S3 metadata or body size limits, and does not modify or truncate data. Instead, it emits warnings
+ * via the `exceedsLimit` event when S3 metadata limits are exceeded, but allows all operations to proceed.
+ *
+ * ## Purpose & Use Cases
+ * - For development, testing, or advanced users who want full control over resource metadata and body size.
+ * - Useful when you want to handle S3 metadata limits yourself, or implement custom logic for warnings.
+ * - Not recommended for production unless you have custom enforcement or validation in place.
+ *
+ * ## How It Works
+ * - Emits an `exceedsLimit` event (with details) when a resource's metadata size exceeds the S3 2KB limit.
+ * - Does NOT block, truncate, or modify data—operations always proceed.
+ * - No automatic enforcement of any limits; user is responsible for handling warnings and data integrity.
+ *
+ * ## Event Emission
+ * - Event: `exceedsLimit`
+ * - Payload:
+ *   - `operation`: 'insert' | 'update' | 'upsert'
+ *   - `id` (for update/upsert): resource id
+ *   - `totalSize`: total metadata size in bytes
+ *   - `limit`: S3 metadata limit (2048 bytes)
+ *   - `excess`: number of bytes over the limit
+ *   - `data`: the offending data object
+ *
+ * @example
+ * // Listen for warnings on a resource
+ * resource.on('exceedsLimit', (info) => {
+ *   console.warn(`Resource exceeded S3 metadata limit:`, info);
+ * });
+ *
+ * @example
+ * // Create a resource with user-managed behavior (default)
+ * const resource = await db.createResource({
+ *   name: 'my_resource',
+ *   attributes: { ... },
+ *   behavior: 'user-managed' // or omit for default
+ * });
+ *
+ * ## Comparison to Other Behaviors
+ * | Behavior         | Enforcement | Data Loss | Event Emission | Use Case                |
+ * |------------------|-------------|-----------|----------------|-------------------------|
+ * | user-managed     | None        | Possible  | Warns          | Dev/Test/Advanced users |
+ * | enforce-limits   | Strict      | No        | Throws         | Production              |
+ * | truncate-data    | Truncates   | Yes       | Warns          | Content Mgmt            |
+ * | body-overflow    | Truncates/Splits | Yes   | Warns          | Large objects           |
+ *
+ * ## Best Practices & Warnings
+ * - Exceeding S3 metadata limits will cause silent data loss or errors at the storage layer.
+ * - Use this behavior only if you have custom logic to handle warnings and enforce limits.
+ * - For production, prefer `enforce-limits` or `truncate-data` to avoid data loss.
+ *
+ * ## Migration Tips
+ * - To migrate to a stricter behavior, change the resource's behavior to `enforce-limits` or `truncate-data`.
+ * - Review emitted warnings to identify resources at risk of exceeding S3 limits.
+ *
+ * @typedef {Object} UserManagedBehaviorConfig
+ * @property {boolean} [enabled=true] - Whether the behavior is active
+ */
+export async function handleInsert({ resource, data, mappedData, originalData }) {
+  const totalSize = calculateTotalSize(mappedData);
+  
+  // Calculate effective limit considering system overhead
+  const effectiveLimit = calculateEffectiveLimit({
+    s3Limit: S3_METADATA_LIMIT_BYTES,
+    systemConfig: {
+      version: resource.version,
+      timestamps: resource.config.timestamps,
+      id: data.id
+    }
+  });
+  
+  if (totalSize > effectiveLimit) {
+    resource.emit('exceedsLimit', {
+      operation: 'insert',
+      totalSize,
+      limit: 2047,
+      excess: totalSize - 2047,
+      data: originalData || data
+    });
+  }
+  return { mappedData, body: JSON.stringify(data) };
+}
+
+export async function handleUpdate({ resource, id, data, mappedData, originalData }) {
+  const totalSize = calculateTotalSize(mappedData);
+  
+  // Calculate effective limit considering system overhead
+  const effectiveLimit = calculateEffectiveLimit({
+    s3Limit: S3_METADATA_LIMIT_BYTES,
+    systemConfig: {
+      version: resource.version,
+      timestamps: resource.config.timestamps,
+      id
+    }
+  });
+  
+  if (totalSize > effectiveLimit) {
+    resource.emit('exceedsLimit', {
+      operation: 'update',
+      id,
+      totalSize,
+      limit: 2047,
+      excess: totalSize - 2047,
+      data: originalData || data
+    });
+  }
+  return { mappedData, body: JSON.stringify(data) };
+}
+
+export async function handleUpsert({ resource, id, data, mappedData, originalData }) {
+  const totalSize = calculateTotalSize(mappedData);
+  
+  // Calculate effective limit considering system overhead
+  const effectiveLimit = calculateEffectiveLimit({
+    s3Limit: S3_METADATA_LIMIT_BYTES,
+    systemConfig: {
+      version: resource.version,
+      timestamps: resource.config.timestamps,
+      id
+    }
+  });
+  
+  if (totalSize > effectiveLimit) {
+    resource.emit('exceedsLimit', {
+      operation: 'upsert',
+      id,
+      totalSize,
+      limit: 2047,
+      excess: totalSize - 2047,
+      data: originalData || data
+    });
+  }
+  return { mappedData, body: JSON.stringify(data) };
+}
+
+export async function handleGet({ resource, metadata, body }) {
+  // No special handling needed for user-managed behavior
+  // User is responsible for handling metadata as received
+  return { metadata, body };
+}