s3db.js 11.3.1 → 12.0.0

package/src/concerns/geo-encoding.js

@@ -0,0 +1,256 @@
+/**
+ * Geographic Coordinate Encoding - Normalized Fixed-Point
+ *
+ * Optimizes storage of latitude/longitude by:
+ * 1. Normalizing to positive range (eliminates negative sign)
+ * 2. Using fixed-point integer encoding
+ * 3. Base62 compression
+ *
+ * Achieves 45-55% compression vs JSON floats.
+ *
+ * Examples:
+ *   Latitude -23.550519 → "~18kPxZ" (8 bytes vs 15 bytes = 47% savings)
+ *   Longitude -46.633309 → "~36WqLj" (8 bytes vs 16 bytes = 50% savings)
+ *
+ * Precision:
+ *   6 decimals = ~11cm accuracy (GPS standard)
+ *   5 decimals = ~1.1m accuracy (sufficient for most apps)
+ *   4 decimals = ~11m accuracy (building-level)
+ */
+
+import { encode, decode } from './base62.js';
+
+/**
+ * Encode latitude with normalized range
+ * Range: -90 to +90 → normalized to 0 to 180
+ *
+ * @param {number} lat - Latitude value (-90 to 90)
+ * @param {number} precision - Decimal places to preserve (default: 6)
+ * @returns {string} Encoded string with '~' prefix
+ *
+ * @throws {Error} If latitude is out of valid range
+ *
+ * @example
+ * encodeGeoLat(-23.550519, 6)  // → "~18kPxZ"
+ * encodeGeoLat(40.7128, 6)     // → "~2i8pYw"
+ */
+export function encodeGeoLat(lat, precision = 6) {
+  if (lat === null || lat === undefined) return lat;
+  if (typeof lat !== 'number' || isNaN(lat)) return lat;
+  if (!isFinite(lat)) return lat;
+
+  // Validate range
+  if (lat < -90 || lat > 90) {
+    throw new Error(`Latitude out of range [-90, 90]: ${lat}`);
+  }
+
+  // Normalize: -90 to +90 → 0 to 180
+  const normalized = lat + 90;
+
+  // Convert to fixed-point integer
+  const scale = Math.pow(10, precision);
+  const scaled = Math.round(normalized * scale);
+
+  // Encode with '~' prefix to identify as geo coordinate
+  return '~' + encode(scaled);
+}
+
+/**
+ * Decode latitude from encoded string
+ *
+ * @param {string} encoded - Encoded string (must start with '~')
+ * @param {number} precision - Decimal places used in encoding (default: 6)
+ * @returns {number} Decoded latitude value
+ *
+ * @example
+ * decodeGeoLat('~18kPxZ', 6)  // → -23.550519
+ */
+export function decodeGeoLat(encoded, precision = 6) {
+  if (typeof encoded !== 'string') return encoded;
+  if (!encoded.startsWith('~')) return encoded;
+
+  const scaled = decode(encoded.slice(1));
+  if (isNaN(scaled)) return NaN;
+
+  const scale = Math.pow(10, precision);
+  const normalized = scaled / scale;
+
+  // Denormalize: 0 to 180 → -90 to +90
+  return normalized - 90;
+}
+
+/**
+ * Encode longitude with normalized range
+ * Range: -180 to +180 → normalized to 0 to 360
+ *
+ * @param {number} lon - Longitude value (-180 to 180)
+ * @param {number} precision - Decimal places to preserve (default: 6)
+ * @returns {string} Encoded string with '~' prefix
+ *
+ * @throws {Error} If longitude is out of valid range
+ *
+ * @example
+ * encodeGeoLon(-46.633309, 6)  // → "~36WqLj"
+ * encodeGeoLon(-74.0060, 6)    // → "~2xKqrO"
+ */
+export function encodeGeoLon(lon, precision = 6) {
+  if (lon === null || lon === undefined) return lon;
+  if (typeof lon !== 'number' || isNaN(lon)) return lon;
+  if (!isFinite(lon)) return lon;
+
+  // Validate range
+  if (lon < -180 || lon > 180) {
+    throw new Error(`Longitude out of range [-180, 180]: ${lon}`);
+  }
+
+  // Normalize: -180 to +180 → 0 to 360
+  const normalized = lon + 180;
+
+  // Convert to fixed-point integer
+  const scale = Math.pow(10, precision);
+  const scaled = Math.round(normalized * scale);
+
+  // Encode with '~' prefix
+  return '~' + encode(scaled);
+}
+
+/**
+ * Decode longitude from encoded string
+ *
+ * @param {string} encoded - Encoded string (must start with '~')
+ * @param {number} precision - Decimal places used in encoding (default: 6)
+ * @returns {number} Decoded longitude value
+ *
+ * @example
+ * decodeGeoLon('~36WqLj', 6)  // → -46.633309
+ */
+export function decodeGeoLon(encoded, precision = 6) {
+  if (typeof encoded !== 'string') return encoded;
+  if (!encoded.startsWith('~')) return encoded;
+
+  const scaled = decode(encoded.slice(1));
+  if (isNaN(scaled)) return NaN;
+
+  const scale = Math.pow(10, precision);
+  const normalized = scaled / scale;
+
+  // Denormalize: 0 to 360 → -180 to +180
+  return normalized - 180;
+}
+
+/**
+ * Encode a lat/lon point as a single string
+ * Format: {lat}{lon} (both with '~' prefix)
+ *
+ * @param {number} lat - Latitude
+ * @param {number} lon - Longitude
+ * @param {number} precision - Decimal places (default: 6)
+ * @returns {string} Encoded point
+ *
+ * @example
+ * encodeGeoPoint(-23.550519, -46.633309, 6)
+ * // → "~18kPxZ~36WqLj"
+ */
+export function encodeGeoPoint(lat, lon, precision = 6) {
+  const latEncoded = encodeGeoLat(lat, precision);
+  const lonEncoded = encodeGeoLon(lon, precision);
+
+  // Return concatenated (both have '~' prefix for easy parsing)
+  return latEncoded + lonEncoded;
+}
+
+/**
+ * Decode a lat/lon point from encoded string
+ *
+ * @param {string} encoded - Encoded point string
+ * @param {number} precision - Decimal places (default: 6)
+ * @returns {Object} { latitude, longitude }
+ *
+ * @example
+ * decodeGeoPoint('~18kPxZ~36WqLj', 6)
+ * // → { latitude: -23.550519, longitude: -46.633309 }
+ */
+export function decodeGeoPoint(encoded, precision = 6) {
+  if (typeof encoded !== 'string') return { latitude: NaN, longitude: NaN };
+
+  // Split by '~' and filter empty strings
+  const parts = encoded.split('~').filter(p => p.length > 0);
+
+  if (parts.length !== 2) {
+    return { latitude: NaN, longitude: NaN };
+  }
+
+  // Decode each part (re-add '~' prefix)
+  const latitude = decodeGeoLat('~' + parts[0], precision);
+  const longitude = decodeGeoLon('~' + parts[1], precision);
+
+  return { latitude, longitude };
+}
+
+/**
+ * Validate if coordinates are within valid ranges
+ * @param {number} lat - Latitude
+ * @param {number} lon - Longitude
+ * @returns {boolean} True if valid
+ */
+export function isValidCoordinate(lat, lon) {
+  return (
+    typeof lat === 'number' &&
+    typeof lon === 'number' &&
+    !isNaN(lat) &&
+    !isNaN(lon) &&
+    isFinite(lat) &&
+    isFinite(lon) &&
+    lat >= -90 &&
+    lat <= 90 &&
+    lon >= -180 &&
+    lon <= 180
+  );
+}
+
+/**
+ * Calculate precision level based on desired accuracy
+ *
+ * @param {number} accuracyMeters - Desired accuracy in meters
+ * @returns {number} Recommended decimal places
+ *
+ * Precision levels:
+ * - 0 decimals: ~111 km
+ * - 1 decimal: ~11 km
+ * - 2 decimals: ~1.1 km
+ * - 3 decimals: ~110 m
+ * - 4 decimals: ~11 m
+ * - 5 decimals: ~1.1 m (GPS consumer)
+ * - 6 decimals: ~11 cm (GPS precision)
+ * - 7 decimals: ~1.1 cm
+ */
+export function getPrecisionForAccuracy(accuracyMeters) {
+  if (accuracyMeters >= 111000) return 0;
+  if (accuracyMeters >= 11000) return 1;
+  if (accuracyMeters >= 1100) return 2;
+  if (accuracyMeters >= 110) return 3;
+  if (accuracyMeters >= 11) return 4;
+  if (accuracyMeters >= 1.1) return 5;
+  if (accuracyMeters >= 0.11) return 6;
+  return 7;
+}
+
+/**
+ * Get accuracy in meters for a precision level
+ * @param {number} precision - Decimal places
+ * @returns {number} Approximate accuracy in meters
+ */
+export function getAccuracyForPrecision(precision) {
+  const accuracies = {
+    0: 111000,
+    1: 11000,
+    2: 1100,
+    3: 110,
+    4: 11,
+    5: 1.1,
+    6: 0.11,
+    7: 0.011
+  };
+
+  return accuracies[precision] || 111000;
+}

package/src/concerns/high-performance-inserter.js

@@ -79,8 +79,8 @@ export class HighPerformanceInserter {
     // Take current buffer and reset
     const batch = this.insertBuffer.splice(0, this.batchSize);
     const startTime = Date.now();
-    
-    try {
+
+    const [ok, err] = await tryFn(async () => {
       // Process inserts in parallel with connection pooling
       const { results, errors } = await PromisePool
         .for(batch)
@@ -88,25 +88,25 @@ export class HighPerformanceInserter {
         .process(async (item) => {
           return await this.performInsert(item);
         });
-      
+
       // Update stats
       const duration = Date.now() - startTime;
       this.stats.inserted += results.filter(r => r.success).length;
       this.stats.failed += errors.length;
       this.stats.avgInsertTime = duration / batch.length;
-      
+
       // Process partition queue separately (non-blocking)
       if (!this.disablePartitions && this.partitionQueue.length > 0) {
         this.processPartitionsAsync();
       }
-      
-    } finally {
-      this.isProcessing = false;
-      
-      // Continue processing if more items
-      if (this.insertBuffer.length > 0) {
-        setImmediate(() => this.flush());
-      }
+    });
+
+    // Always execute (finally equivalent)
+    this.isProcessing = false;
+
+    // Continue processing if more items
+    if (this.insertBuffer.length > 0) {
+      setImmediate(() => this.flush());
     }
   }
 
@@ -115,43 +115,46 @@ export class HighPerformanceInserter {
    */
   async performInsert(item) {
     const { data } = item;
-    
-    try {
+
+    const [ok, error, result] = await tryFn(async () => {
       // Temporarily disable partitions for the insert
       const originalAsyncPartitions = this.resource.config.asyncPartitions;
       const originalPartitions = this.resource.config.partitions;
-      
+
       if (this.disablePartitions) {
         // Completely bypass partitions during insert
         this.resource.config.partitions = {};
       }
-      
+
       // Perform insert
-      const [ok, err, result] = await tryFn(() => this.resource.insert(data));
-      
-      if (!ok) {
-        return { success: false, error: err };
+      const [insertOk, insertErr, insertResult] = await tryFn(() => this.resource.insert(data));
+
+      if (!insertOk) {
+        throw insertErr; // Re-throw to be caught by outer tryFn
       }
-      
+
       // Queue partition creation for later (if not disabled)
       if (!this.disablePartitions && originalPartitions && Object.keys(originalPartitions).length > 0) {
         this.partitionQueue.push({
           operation: 'create',
-          data: result,
+          data: insertResult,
           partitions: originalPartitions
         });
         this.stats.partitionsPending++;
       }
-      
+
       // Restore original config
       this.resource.config.partitions = originalPartitions;
       this.resource.config.asyncPartitions = originalAsyncPartitions;
-      
-      return { success: true, data: result };
-      
-    } catch (error) {
+
+      return { success: true, data: insertResult };
+    });
+
+    if (!ok) {
       return { success: false, error };
     }
+
+    return result;
   }
 
   /**
@@ -173,10 +176,11 @@ export class HighPerformanceInserter {
         .for(batch)
         .withConcurrency(10) // Lower concurrency for partitions
         .process(async (item) => {
-          try {
-            await this.resource.createPartitionReferences(item.data);
+          const [ok, err] = await tryFn(() => this.resource.createPartitionReferences(item.data));
+
+          if (ok) {
             this.stats.partitionsPending--;
-          } catch (err) {
+          } else {
             // Silently handle partition errors
             this.resource.emit('partitionIndexError', {
               operation: 'bulk-insert',

package/src/concerns/ip.js

@@ -0,0 +1,325 @@
+/**
+ * IP Address Encoding/Decoding Utilities
+ *
+ * Provides compact binary encoding for IPv4 and IPv6 addresses
+ * to save space in S3 metadata.
+ *
+ * Savings:
+ * - IPv4: "192.168.1.1" (11-15 chars) → 4 bytes → ~8 chars Base64 (47% savings)
+ * - IPv6: "2001:db8::1" (up to 39 chars) → 16 bytes → ~22 chars Base64 (44% savings)
+ */
+
+import tryFn from './try-fn.js';
+
+/**
+ * Validate IPv4 address format
+ * @param {string} ip - IP address string
+ * @returns {boolean} True if valid IPv4
+ */
+export function isValidIPv4(ip) {
+  if (typeof ip !== 'string') return false;
+
+  const ipv4Regex = /^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/;
+  const match = ip.match(ipv4Regex);
+
+  if (!match) return false;
+
+  // Check each octet is 0-255
+  for (let i = 1; i <= 4; i++) {
+    const octet = parseInt(match[i], 10);
+    if (octet < 0 || octet > 255) return false;
+  }
+
+  return true;
+}
+
+/**
+ * Validate IPv6 address format
+ * @param {string} ip - IP address string
+ * @returns {boolean} True if valid IPv6
+ */
+export function isValidIPv6(ip) {
+  if (typeof ip !== 'string') return false;
+
+  // IPv6 regex (simplified, covers most cases)
+  const ipv6Regex = /^(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?[0-9])?[0-9])\.){3}(25[0-5]|(2[0-4]|1?[0-9])?[0-9]))$/;
+
+  return ipv6Regex.test(ip);
+}
+
+/**
+ * Encode IPv4 address to Base64 binary representation
+ * @param {string} ip - IPv4 address (e.g., "192.168.1.1")
+ * @returns {string} Base64-encoded binary (e.g., "wKgBAQ==")
+ */
+export function encodeIPv4(ip) {
+  if (!isValidIPv4(ip)) {
+    throw new Error(`Invalid IPv4 address: ${ip}`);
+  }
+
+  const octets = ip.split('.').map(octet => parseInt(octet, 10));
+  const buffer = Buffer.from(octets);
+
+  return buffer.toString('base64');
+}
+
+/**
+ * Decode Base64 binary to IPv4 address
+ * @param {string} encoded - Base64-encoded binary
+ * @returns {string} IPv4 address (e.g., "192.168.1.1")
+ */
+export function decodeIPv4(encoded) {
+  if (typeof encoded !== 'string') {
+    throw new Error('Encoded IPv4 must be a string');
+  }
+
+  const [ok, err, result] = tryFn(() => {
+    const buffer = Buffer.from(encoded, 'base64');
+
+    if (buffer.length !== 4) {
+      throw new Error(`Invalid encoded IPv4 length: ${buffer.length} (expected 4)`);
+    }
+
+    return Array.from(buffer).join('.');
+  });
+
+  if (!ok) {
+    throw new Error(`Failed to decode IPv4: ${err.message}`);
+  }
+
+  return result;
+}
+
+/**
+ * Normalize IPv6 address to full expanded form
+ * @param {string} ip - IPv6 address (may be compressed)
+ * @returns {string} Expanded IPv6 address
+ */
+export function expandIPv6(ip) {
+  if (!isValidIPv6(ip)) {
+    throw new Error(`Invalid IPv6 address: ${ip}`);
+  }
+
+  // Handle :: expansion
+  let expanded = ip;
+
+  // Special case: ::
+  if (expanded === '::') {
+    return '0000:0000:0000:0000:0000:0000:0000:0000';
+  }
+
+  // Expand ::
+  if (expanded.includes('::')) {
+    const parts = expanded.split('::');
+    const leftParts = parts[0] ? parts[0].split(':') : [];
+    const rightParts = parts[1] ? parts[1].split(':') : [];
+    const missingGroups = 8 - leftParts.length - rightParts.length;
+
+    const middleParts = Array(missingGroups).fill('0');
+    expanded = [...leftParts, ...middleParts, ...rightParts].join(':');
+  }
+
+  // Pad each group to 4 digits
+  const groups = expanded.split(':');
+  const paddedGroups = groups.map(group => group.padStart(4, '0'));
+
+  return paddedGroups.join(':');
+}
+
+/**
+ * Compress IPv6 address (remove leading zeros and use ::)
+ * @param {string} ip - Full IPv6 address
+ * @returns {string} Compressed IPv6 address
+ */
+export function compressIPv6(ip) {
+  // Remove leading zeros
+  let compressed = ip.split(':').map(group => {
+    return parseInt(group, 16).toString(16);
+  }).join(':');
+
+  // Find longest sequence of consecutive 0 groups
+  const zeroSequences = [];
+  let currentSequence = { start: -1, length: 0 };
+
+  compressed.split(':').forEach((group, index) => {
+    if (group === '0') {
+      if (currentSequence.start === -1) {
+        currentSequence.start = index;
+        currentSequence.length = 1;
+      } else {
+        currentSequence.length++;
+      }
+    } else {
+      if (currentSequence.length > 0) {
+        zeroSequences.push({ ...currentSequence });
+        currentSequence = { start: -1, length: 0 };
+      }
+    }
+  });
+
+  if (currentSequence.length > 0) {
+    zeroSequences.push(currentSequence);
+  }
+
+  // Find longest sequence (must be at least 2 groups)
+  const longestSequence = zeroSequences
+    .filter(seq => seq.length >= 2)
+    .sort((a, b) => b.length - a.length)[0];
+
+  if (longestSequence) {
+    const parts = compressed.split(':');
+    const before = parts.slice(0, longestSequence.start).join(':');
+    const after = parts.slice(longestSequence.start + longestSequence.length).join(':');
+
+    if (before && after) {
+      compressed = `${before}::${after}`;
+    } else if (before) {
+      compressed = `${before}::`;
+    } else if (after) {
+      compressed = `::${after}`;
+    } else {
+      compressed = '::';
+    }
+  }
+
+  return compressed;
+}
+
+/**
+ * Encode IPv6 address to Base64 binary representation
+ *
+ * Always encodes to ensure consistent format in storage.
+ * IPv6 addresses are normalized to 16 bytes and Base64-encoded to 24 characters.
+ *
+ * @param {string} ip - IPv6 address (e.g., "2001:db8::1")
+ * @returns {string} Base64-encoded binary (24 chars)
+ */
+export function encodeIPv6(ip) {
+  if (!isValidIPv6(ip)) {
+    throw new Error(`Invalid IPv6 address: ${ip}`);
+  }
+
+  // Always encode for consistency (like IPv4)
+  // Expand to full notation first
+  const expanded = expandIPv6(ip);
+  const groups = expanded.split(':');
+
+  // Convert each group to 2 bytes
+  const bytes = [];
+  for (const group of groups) {
+    const value = parseInt(group, 16);
+    bytes.push((value >> 8) & 0xFF); // High byte
+    bytes.push(value & 0xFF);         // Low byte
+  }
+
+  const buffer = Buffer.from(bytes);
+  return buffer.toString('base64');
+}
+
+/**
+ * Decode Base64 binary to IPv6 address
+ *
+ * Handles both encoded and unencoded IPv6 addresses for backwards compatibility.
+ * - If input is a valid unencoded IPv6 address → return it (optionally expanded)
+ * - Otherwise → decode from Base64 binary (24 chars)
+ *
+ * @param {string} encoded - Base64-encoded binary (24 chars) or unencoded IPv6
+ * @param {boolean} compress - Whether to compress the output (default: true)
+ * @returns {string} IPv6 address
+ */
+export function decodeIPv6(encoded, compress = true) {
+  if (typeof encoded !== 'string') {
+    throw new Error('Encoded IPv6 must be a string');
+  }
+
+  // SMART DETECTION: Check if this is unencoded IPv6
+  // If it's not 24 chars AND it's a valid IPv6, treat as unencoded
+  if (encoded.length !== 24 && isValidIPv6(encoded)) {
+    // Not encoded - was kept as original compressed form
+    // Respect the compress parameter
+    return compress ? encoded : expandIPv6(encoded);
+  }
+
+  // Try to decode as Base64 - works for both 24-char encoded AND invalid inputs
+  const [ok, err, result] = tryFn(() => {
+    const buffer = Buffer.from(encoded, 'base64');
+
+    if (buffer.length !== 16) {
+      throw new Error(`Invalid encoded IPv6 length: ${buffer.length} (expected 16)`);
+    }
+
+    const groups = [];
+    for (let i = 0; i < 16; i += 2) {
+      const value = (buffer[i] << 8) | buffer[i + 1];
+      groups.push(value.toString(16).padStart(4, '0'));
+    }
+
+    const fullAddress = groups.join(':');
+
+    return compress ? compressIPv6(fullAddress) : fullAddress;
+  });
+
+  if (!ok) {
+    throw new Error(`Failed to decode IPv6: ${err.message}`);
+  }
+
+  return result;
+}
+
+/**
+ * Detect IP version from string
+ * @param {string} ip - IP address string
+ * @returns {'ipv4'|'ipv6'|null} IP version or null if invalid
+ */
+export function detectIPVersion(ip) {
+  if (isValidIPv4(ip)) return 'ipv4';
+  if (isValidIPv6(ip)) return 'ipv6';
+  return null;
+}
+
+/**
+ * Calculate savings percentage for IP encoding
+ * @param {string} ip - IP address
+ * @returns {Object} Savings information
+ */
+export function calculateIPSavings(ip) {
+  const version = detectIPVersion(ip);
+
+  if (!version) {
+    return { version: null, originalSize: 0, encodedSize: 0, savings: 0 };
+  }
+
+  const originalSize = ip.length;
+  let encodedSize;
+
+  if (version === 'ipv4') {
+    const encoded = encodeIPv4(ip);
+    encodedSize = encoded.length;
+  } else {
+    const encoded = encodeIPv6(ip);
+    encodedSize = encoded.length;
+  }
+
+  const savings = ((originalSize - encodedSize) / originalSize) * 100;
+
+  return {
+    version,
+    originalSize,
+    encodedSize,
+    savings: Math.round(savings * 100) / 100,
+    savingsPercent: `${Math.round(savings)}%`
+  };
+}
+
+export default {
+  isValidIPv4,
+  isValidIPv6,
+  encodeIPv4,
+  decodeIPv4,
+  encodeIPv6,
+  decodeIPv6,
+  expandIPv6,
+  compressIPv6,
+  detectIPVersion,
+  calculateIPSavings
+};