s3db.js 11.3.2 → 12.0.1

package/src/client.class.js

@@ -145,21 +145,19 @@ export class Client extends EventEmitter {
     if (contentLength !== undefined) options.ContentLength = contentLength
     if (ifMatch !== undefined) options.IfMatch = ifMatch
 
-    let response, error;
-    try {
-      response = await this.sendCommand(new PutObjectCommand(options));
-      return response;
-    } catch (err) {
-      error = err;
+    const [ok, err, response] = await tryFn(() => this.sendCommand(new PutObjectCommand(options)));
+    this.emit('putObject', err || response, { key, metadata, contentType, body, contentEncoding, contentLength });
+
+    if (!ok) {
       throw mapAwsError(err, {
         bucket: this.config.bucket,
         key,
         commandName: 'PutObjectCommand',
         commandInput: options,
       });
-    } finally {
-      this.emit('putObject', error || response, { key, metadata, contentType, body, contentEncoding, contentLength });
     }
+
+    return response;
   }
 
   async getObject(key) {
@@ -168,32 +166,34 @@ export class Client extends EventEmitter {
       Bucket: this.config.bucket,
       Key: keyPrefix ? path.join(keyPrefix, key) : key,
     };
- 
-    let response, error;
-    try {
-      response = await this.sendCommand(new GetObjectCommand(options));
-      
+
+    const [ok, err, response] = await tryFn(async () => {
+      const res = await this.sendCommand(new GetObjectCommand(options));
+
       // Smart decode metadata values
-      if (response.Metadata) {
+      if (res.Metadata) {
         const decodedMetadata = {};
-        for (const [key, value] of Object.entries(response.Metadata)) {
+        for (const [key, value] of Object.entries(res.Metadata)) {
           decodedMetadata[key] = metadataDecode(value);
         }
-        response.Metadata = decodedMetadata;
+        res.Metadata = decodedMetadata;
       }
-      
-      return response;
-    } catch (err) {
-      error = err;
+
+      return res;
+    });
+
+    this.emit('getObject', err || response, { key });
+
+    if (!ok) {
       throw mapAwsError(err, {
         bucket: this.config.bucket,
         key,
         commandName: 'GetObjectCommand',
         commandInput: options,
       });
-    } finally {
-      this.emit('getObject', error || response, { key });
     }
+
+    return response;
   }
 
   async headObject(key) {
@@ -202,45 +202,77 @@ export class Client extends EventEmitter {
       Bucket: this.config.bucket,
       Key: keyPrefix ? path.join(keyPrefix, key) : key,
     };
-    let response, error;
-    try {
-      response = await this.sendCommand(new HeadObjectCommand(options));
-      return response;
-    } catch (err) {
-      error = err;
+
+    const [ok, err, response] = await tryFn(async () => {
+      const res = await this.sendCommand(new HeadObjectCommand(options));
+
+      // Smart decode metadata values (same as getObject)
+      if (res.Metadata) {
+        const decodedMetadata = {};
+        for (const [key, value] of Object.entries(res.Metadata)) {
+          decodedMetadata[key] = metadataDecode(value);
+        }
+        res.Metadata = decodedMetadata;
+      }
+
+      return res;
+    });
+
+    this.emit('headObject', err || response, { key });
+
+    if (!ok) {
       throw mapAwsError(err, {
         bucket: this.config.bucket,
         key,
         commandName: 'HeadObjectCommand',
         commandInput: options,
       });
-    } finally {
-      this.emit('headObject', error || response, { key });
     }
+
+    return response;
   }
 
-  async copyObject({ from, to }) {
+  async copyObject({ from, to, metadata, metadataDirective, contentType }) {
+    const keyPrefix = typeof this.config.keyPrefix === 'string' ? this.config.keyPrefix : '';
     const options = {
       Bucket: this.config.bucket,
-      Key: this.config.keyPrefix ? path.join(this.config.keyPrefix, to) : to,
-      CopySource: path.join(this.config.bucket, this.config.keyPrefix ? path.join(this.config.keyPrefix, from) : from),
+      Key: keyPrefix ? path.join(keyPrefix, to) : to,
+      CopySource: path.join(this.config.bucket, keyPrefix ? path.join(keyPrefix, from) : from),
     };
 
-    let response, error;
-    try {
-      response = await this.sendCommand(new CopyObjectCommand(options));
-      return response;
-    } catch (err) {
-      error = err;
+    // Add metadata directive if specified
+    if (metadataDirective) {
+      options.MetadataDirective = metadataDirective; // 'COPY' or 'REPLACE'
+    }
+
+    // Add metadata if specified (and encode values)
+    if (metadata && typeof metadata === 'object') {
+      const encodedMetadata = {};
+      for (const [key, value] of Object.entries(metadata)) {
+        const { encoded } = metadataEncode(value);
+        encodedMetadata[key] = encoded;
+      }
+      options.Metadata = encodedMetadata;
+    }
+
+    // Add content type if specified
+    if (contentType) {
+      options.ContentType = contentType;
+    }
+
+    const [ok, err, response] = await tryFn(() => this.sendCommand(new CopyObjectCommand(options)));
+    this.emit('copyObject', err || response, { from, to, metadataDirective });
+
+    if (!ok) {
       throw mapAwsError(err, {
         bucket: this.config.bucket,
         key: to,
         commandName: 'CopyObjectCommand',
         commandInput: options,
       });
-    } finally {
-      this.emit('copyObject', error || response, { from, to });
     }
+
+    return response;
   }
 
   async exists(key) {
@@ -258,21 +290,19 @@ export class Client extends EventEmitter {
       Key: keyPrefix ? path.join(keyPrefix, key) : key,
     };
 
-    let response, error;
-    try {
-      response = await this.sendCommand(new DeleteObjectCommand(options));
-      return response;
-    } catch (err) {
-      error = err;
+    const [ok, err, response] = await tryFn(() => this.sendCommand(new DeleteObjectCommand(options)));
+    this.emit('deleteObject', err || response, { key });
+
+    if (!ok) {
       throw mapAwsError(err, {
         bucket: this.config.bucket,
         key,
         commandName: 'DeleteObjectCommand',
         commandInput: options,
       });
-    } finally {
-      this.emit('deleteObject', error || response, { key });
     }
+
+    return response;
   }
 
   async deleteObjects(keys) {

package/src/concerns/base62.js

@@ -129,3 +129,88 @@ export const decodeFixedPoint = (s, precision = 6) => {
   const scaled = negative ? -r : r;
   return scaled / scale;
 };
+
+/**
+ * Batch encoding for arrays of fixed-point numbers (optimized for embeddings)
+ *
+ * Achieves ~17% additional compression vs individual encodeFixedPoint by using
+ * a single prefix for the entire array instead of one prefix per value.
+ *
+ * For 1536-dim embedding: ~1533 bytes saved (17.4%)
+ * For 3072-dim embedding: ~3069 bytes saved (17.5%)
+ *
+ * @param {number[]} values - Array of numbers to encode
+ * @param {number} precision - Decimal places to preserve (default: 6)
+ * @returns {string} Batch-encoded string with format: ^[val1,val2,val3,...]
+ *
+ * Examples:
+ *   [0.123, -0.456, 0.789] → "^[w7f,-3sdz,oHb]"
+ *   [] → "^[]"
+ */
+export const encodeFixedPointBatch = (values, precision = 6) => {
+  if (!Array.isArray(values)) return '';
+  if (values.length === 0) return '^[]';
+
+  const scale = Math.pow(10, precision);
+
+  const encoded = values.map(n => {
+    if (typeof n !== 'number' || isNaN(n) || !isFinite(n)) return '';
+
+    const scaled = Math.round(n * scale);
+    if (scaled === 0) return '0';
+
+    const negative = scaled < 0;
+    let num = Math.abs(scaled);
+    let s = '';
+
+    while (num > 0) {
+      s = alphabet[num % base] + s;
+      num = Math.floor(num / base);
+    }
+
+    return (negative ? '-' : '') + s;
+  });
+
+  // Single prefix for entire batch, comma-separated
+  return '^[' + encoded.join(',') + ']';
+};
+
+/**
+ * Decodes batch-encoded fixed-point arrays
+ *
+ * @param {string} s - Batch-encoded string (format: ^[val1,val2,...])
+ * @param {number} precision - Decimal places used in encoding (default: 6)
+ * @returns {number[]} Decoded array of numbers
+ */
+export const decodeFixedPointBatch = (s, precision = 6) => {
+  if (typeof s !== 'string') return [];
+  if (!s.startsWith('^[')) return [];
+
+  s = s.slice(2, -1); // Remove ^[ and ]
+
+  if (s === '') return [];
+
+  const parts = s.split(',');
+  const scale = Math.pow(10, precision);
+
+  return parts.map(part => {
+    if (part === '0') return 0;
+    if (part === '') return NaN;
+
+    let negative = false;
+    if (part[0] === '-') {
+      negative = true;
+      part = part.slice(1);
+    }
+
+    let r = 0;
+    for (let i = 0; i < part.length; i++) {
+      const idx = charToValue[part[i]];
+      if (idx === undefined) return NaN;
+      r = r * base + idx;
+    }
+
+    const scaled = negative ? -r : r;
+    return scaled / scale;
+  });
+};

package/src/concerns/dictionary-encoding.js

@@ -0,0 +1,294 @@
+/**
+ * Dictionary Encoding for Common Metadata Values
+ *
+ * Provides massive compression for frequently-used long strings:
+ * - Content-Types: application/json (16B) → j (1B) = -93.75%
+ * - URL Prefixes: https://api.example.com/ (24B) → @a (2B) = -91.7%
+ * - Status Messages: processing (10B) → p (1B) = -90%
+ *
+ * Encoding format: 'd:{code}' where {code} is 1-2 characters
+ * Example: 'd:j' = application/json (3B vs 16B = -81% with prefix!)
+ */
+
+/**
+ * Content-Type Dictionary
+ * Most common MIME types with massive savings potential
+ * Format: 'original_value' → 'single_char_code'
+ */
+const CONTENT_TYPE_DICT = {
+  // JSON/XML (most common, highest savings)
+  'application/json': 'j',              // 16B → 1B = -93.75%
+  'application/xml': 'X',               // 15B → 1B = -93.3% (changed from 'x' to avoid conflict)
+  'application/ld+json': 'J',           // 20B → 1B = -95%
+
+  // Text types
+  'text/html': 'H',                     // 9B → 1B = -88.9% (changed from 'h' to avoid conflict)
+  'text/plain': 'T',                    // 10B → 1B = -90% (changed from 'p' to avoid conflict)
+  'text/css': 'C',                      // 8B → 1B = -87.5% (changed from 'c' to avoid conflict)
+  'text/javascript': 'V',               // 15B → 1B = -93.3% (changed from 's' to avoid conflict)
+  'text/csv': 'v',                      // 8B → 1B = -87.5%
+
+  // Images
+  'image/png': 'P',                     // 9B → 1B = -88.9%
+  'image/jpeg': 'I',                    // 10B → 1B = -90%
+  'image/gif': 'G',                     // 9B → 1B = -88.9%
+  'image/svg+xml': 'S',                 // 13B → 1B = -92.3%
+  'image/webp': 'W',                    // 10B → 1B = -90%
+
+  // Application types
+  'application/pdf': 'Q',               // 15B → 1B = -93.3% (changed from 'd' to avoid conflict)
+  'application/zip': 'z',               // 15B → 1B = -93.3%
+  'application/octet-stream': 'o',      // 24B → 1B = -95.8%
+  'application/x-www-form-urlencoded': 'u', // 33B → 1B = -97%
+  'multipart/form-data': 'F',           // 19B → 1B = -94.7% (changed from 'f' to avoid conflict)
+
+  // Font types
+  'font/woff': 'w',                     // 9B → 1B = -88.9%
+  'font/woff2': 'f'                     // 10B → 1B = -90% (changed from 'F')
+};
+
+/**
+ * URL Prefix Dictionary
+ * Common URL prefixes that appear in paths, webhooks, API endpoints
+ * Format: 'prefix' → '@{code}'
+ */
+const URL_PREFIX_DICT = {
+  // API endpoints (very common)
+  '/api/v1/': '@1',                     // 8B → 2B = -75%
+  '/api/v2/': '@2',                     // 8B → 2B = -75%
+  '/api/v3/': '@3',                     // 8B → 2B = -75%
+  '/api/': '@a',                        // 5B → 2B = -60%
+
+  // HTTPS prefixes
+  'https://api.example.com/': '@A',     // 24B → 2B = -91.7%
+  'https://api.': '@H',                 // 11B → 2B = -81.8%
+  'https://www.': '@W',                 // 12B → 2B = -83.3%
+  'https://': '@h',                     // 8B → 2B = -75%
+  'http://': '@t',                      // 7B → 2B = -71.4%
+
+  // AWS/S3 (common in s3db.js context)
+  'https://s3.amazonaws.com/': '@s',    // 26B → 2B = -92.3%
+  'https://s3-': '@S',                  // 10B → 2B = -80%
+
+  // Localhost (development)
+  'http://localhost:': '@L',            // 17B → 2B = -88.2%
+  'http://localhost': '@l',             // 16B → 2B = -87.5%
+
+  // Common paths
+  '/v1/': '@v',                         // 4B → 2B = -50%
+  '/users/': '@u',                      // 7B → 2B = -71.4%
+  '/products/': '@p'                    // 10B → 2B = -80%
+};
+
+/**
+ * Status Message Dictionary
+ * Common status/state strings
+ * Format: 'status' → 'code'
+ */
+const STATUS_MESSAGE_DICT = {
+  // Processing states (very common, good savings)
+  'processing': 'p',                    // 10B → 1B = -90%
+  'completed': 'c',                     // 9B → 1B = -88.9%
+  'succeeded': 's',                     // 9B → 1B = -88.9%
+  'failed': 'f',                        // 6B → 1B = -83.3%
+  'cancelled': 'x',                     // 9B → 1B = -88.9%
+  'timeout': 't',                       // 7B → 1B = -85.7%
+  'retrying': 'r',                      // 8B → 1B = -87.5%
+
+  // Payment states
+  'authorized': 'a',                    // 10B → 1B = -90%
+  'captured': 'K',                      // 8B → 1B = -87.5% (changed from C to avoid conflict)
+  'refunded': 'R',                      // 8B → 1B = -87.5%
+  'declined': 'd',                      // 8B → 1B = -87.5%
+
+  // Order/delivery states
+  'shipped': 'h',                       // 7B → 1B = -85.7% (changed from S to avoid conflict)
+  'delivered': 'D',                     // 9B → 1B = -88.9%
+  'returned': 'e',                      // 8B → 1B = -87.5% (changed from T to avoid conflict)
+  'in_transit': 'i',                    // 10B → 1B = -90%
+
+  // Generic states
+  'initialized': 'n',                   // 11B → 1B = -90.9% (changed from I to avoid conflict)
+  'terminated': 'm'                     // 10B → 1B = -90% (changed from X to avoid conflict)
+};
+
+/**
+ * Reverse dictionaries for decoding
+ * Built automatically from forward dictionaries
+ */
+const CONTENT_TYPE_REVERSE = Object.fromEntries(
+  Object.entries(CONTENT_TYPE_DICT).map(([k, v]) => [v, k])
+);
+
+const URL_PREFIX_REVERSE = Object.fromEntries(
+  Object.entries(URL_PREFIX_DICT).map(([k, v]) => [v, k])
+);
+
+const STATUS_MESSAGE_REVERSE = Object.fromEntries(
+  Object.entries(STATUS_MESSAGE_DICT).map(([k, v]) => [v, k])
+);
+
+/**
+ * Combined dictionaries for easier lookup
+ * All dictionaries merged into one for encoding
+ */
+const COMBINED_DICT = {
+  ...CONTENT_TYPE_DICT,
+  ...STATUS_MESSAGE_DICT
+  // URL prefixes handled separately (prefix matching)
+};
+
+const COMBINED_REVERSE = {
+  ...CONTENT_TYPE_REVERSE,
+  ...STATUS_MESSAGE_REVERSE
+  // URL prefixes handled separately
+};
+
+/**
+ * Encode a value using dictionary if available
+ * @param {string} value - Value to encode
+ * @returns {Object|null} Encoded result or null if not in dictionary
+ */
+export function dictionaryEncode(value) {
+  if (typeof value !== 'string' || !value) {
+    return null;
+  }
+
+  // Check exact match first (content-types, status messages)
+  if (COMBINED_DICT[value]) {
+    return {
+      encoded: 'd:' + COMBINED_DICT[value],
+      encoding: 'dictionary',
+      originalLength: value.length,
+      encodedLength: 2 + COMBINED_DICT[value].length,
+      dictionaryType: 'exact',
+      savings: value.length - (2 + COMBINED_DICT[value].length)
+    };
+  }
+
+  // Check URL prefix matching (for paths, URLs)
+  // Sort prefixes by length (longest first) to prioritize specific matches
+  const sortedPrefixes = Object.entries(URL_PREFIX_DICT)
+    .sort(([a], [b]) => b.length - a.length);
+
+  for (const [prefix, code] of sortedPrefixes) {
+    if (value.startsWith(prefix)) {
+      const remainder = value.substring(prefix.length);
+      const encoded = 'd:' + code + remainder;
+
+      return {
+        encoded,
+        encoding: 'dictionary',
+        originalLength: value.length,
+        encodedLength: encoded.length,
+        dictionaryType: 'prefix',
+        prefix,
+        remainder,
+        savings: value.length - encoded.length
+      };
+    }
+  }
+
+  // Not in dictionary
+  return null;
+}
+
+/**
+ * Decode a dictionary-encoded value
+ * @param {string} encoded - Encoded value (starts with 'd:')
+ * @returns {string|null} Decoded value or null if not dictionary-encoded
+ */
+export function dictionaryDecode(encoded) {
+  if (typeof encoded !== 'string' || !encoded.startsWith('d:')) {
+    return null;
+  }
+
+  const payload = encoded.substring(2); // Remove 'd:' prefix
+
+  if (payload.length === 0) {
+    return null;
+  }
+
+  // Try exact match first (single character codes)
+  if (payload.length === 1) {
+    const decoded = COMBINED_REVERSE[payload];
+    if (decoded) {
+      return decoded;
+    }
+  }
+
+  // Try URL prefix match (starts with @)
+  if (payload.startsWith('@')) {
+    // Extract prefix code (1-2 chars after @)
+    const prefixCode = payload.substring(0, 2); // '@' + 1 char
+    const remainder = payload.substring(2);
+
+    const prefix = URL_PREFIX_REVERSE[prefixCode];
+    if (prefix) {
+      return prefix + remainder;
+    }
+  }
+
+  // Unknown dictionary code - return null (fall back to original)
+  return null;
+}
+
+/**
+ * Calculate compression ratio for a value
+ * @param {string} value - Original value
+ * @returns {Object} Compression statistics
+ */
+export function calculateDictionaryCompression(value) {
+  const result = dictionaryEncode(value);
+
+  if (!result) {
+    return {
+      compressible: false,
+      original: value.length,
+      encoded: value.length,
+      savings: 0,
+      ratio: 1.0
+    };
+  }
+
+  return {
+    compressible: true,
+    original: result.originalLength,
+    encoded: result.encodedLength,
+    savings: result.savings,
+    ratio: result.encodedLength / result.originalLength,
+    savingsPercent: ((result.savings / result.originalLength) * 100).toFixed(1) + '%'
+  };
+}
+
+/**
+ * Get dictionary statistics (for debugging/monitoring)
+ * @returns {Object} Statistics about dictionaries
+ */
+export function getDictionaryStats() {
+  return {
+    contentTypes: Object.keys(CONTENT_TYPE_DICT).length,
+    urlPrefixes: Object.keys(URL_PREFIX_DICT).length,
+    statusMessages: Object.keys(STATUS_MESSAGE_DICT).length,
+    total: Object.keys(COMBINED_DICT).length + Object.keys(URL_PREFIX_DICT).length,
+    avgSavingsContentType:
+      Object.keys(CONTENT_TYPE_DICT).reduce((sum, key) =>
+        sum + (key.length - (2 + CONTENT_TYPE_DICT[key].length)), 0
+      ) / Object.keys(CONTENT_TYPE_DICT).length,
+    avgSavingsStatus:
+      Object.keys(STATUS_MESSAGE_DICT).reduce((sum, key) =>
+        sum + (key.length - (2 + STATUS_MESSAGE_DICT[key].length)), 0
+      ) / Object.keys(STATUS_MESSAGE_DICT).length
+  };
+}
+
+export default {
+  dictionaryEncode,
+  dictionaryDecode,
+  calculateDictionaryCompression,
+  getDictionaryStats,
+  // Export dictionaries for testing
+  CONTENT_TYPE_DICT,
+  URL_PREFIX_DICT,
+  STATUS_MESSAGE_DICT
+};