s3db.js 11.2.4 → 11.2.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "s3db.js",
-  "version": "11.2.4",
+  "version": "11.2.6",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
@@ -103,6 +103,7 @@
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
     "@types/node": "24.7.0",
+    "@xenova/transformers": "^2.17.2",
     "babel-loader": "^10.0.0",
     "chalk": "^5.6.2",
     "cli-table3": "^0.6.5",

package/src/concerns/base62.js

@@ -59,3 +59,73 @@ export const decodeDecimal = s => {
   const num = decPart ? Number(decodedInt + '.' + decPart) : decodedInt;
   return negative ? -num : num;
 };
+
+/**
+ * Fixed-point encoding optimized for normalized values (typically -1 to 1)
+ * Common in embeddings, similarity scores, probabilities, etc.
+ *
+ * Achieves ~77% compression vs encodeDecimal for embedding vectors.
+ *
+ * @param {number} n - Number to encode (works for any range, optimized for [-1, 1])
+ * @param {number} precision - Decimal places to preserve (default: 6)
+ * @returns {string} Base62-encoded string with '^' prefix to indicate fixed-point encoding
+ *
+ * Examples:
+ *   0.123456 → "^w7f" (4 bytes vs 8 bytes with encodeDecimal)
+ *   -0.8234567 → "^-3sdz" (6 bytes vs 10 bytes)
+ *   1.5 → "^98v9" (for values outside [-1,1], still works but less optimal)
+ */
+export const encodeFixedPoint = (n, precision = 6) => {
+  if (typeof n !== 'number' || isNaN(n)) return 'undefined';
+  if (!isFinite(n)) return 'undefined';
+
+  const scale = Math.pow(10, precision);
+  const scaled = Math.round(n * scale);
+
+  if (scaled === 0) return '^0';
+
+  const negative = scaled < 0;
+  let num = Math.abs(scaled);
+  let s = '';
+
+  while (num > 0) {
+    s = alphabet[num % base] + s;
+    num = Math.floor(num / base);
+  }
+
+  // Prefix with ^ to distinguish from regular base62
+  return '^' + (negative ? '-' : '') + s;
+};
+
+/**
+ * Decodes fixed-point encoded values
+ *
+ * @param {string} s - Encoded string (must start with '^')
+ * @param {number} precision - Decimal places used in encoding (default: 6)
+ * @returns {number} Decoded number
+ */
+export const decodeFixedPoint = (s, precision = 6) => {
+  if (typeof s !== 'string') return NaN;
+  if (!s.startsWith('^')) return NaN; // Safety check
+
+  s = s.slice(1); // Remove ^ prefix
+
+  if (s === '0') return 0;
+
+  let negative = false;
+  if (s[0] === '-') {
+    negative = true;
+    s = s.slice(1);
+  }
+
+  let r = 0;
+  for (let i = 0; i < s.length; i++) {
+    const idx = charToValue[s[i]];
+    if (idx === undefined) return NaN;
+    r = r * base + idx;
+  }
+
+  const scale = Math.pow(10, precision);
+  const scaled = negative ? -r : r;
+  return scaled / scale;
+};

package/src/plugins/index.js

@@ -15,3 +15,4 @@ export * from './replicator.plugin.js'
 export * from './s3-queue.plugin.js'
 export * from './scheduler.plugin.js'
 export * from './state-machine.plugin.js'
+export * from './vector.plugin.js'

package/src/plugins/vector/distances.js

@@ -0,0 +1,173 @@
+/**
+ * Vector Distance Functions
+ *
+ * Provides distance/similarity calculations for vector operations.
+ * All distance functions return lower values for more similar vectors.
+ */
+
+/**
+ * Calculate cosine distance between two vectors
+ *
+ * Range: 0 (identical) to 2 (opposite direction)
+ * Best for: Normalized vectors, semantic similarity
+ *
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @returns {number} Cosine distance
+ * @throws {Error} If vectors have different dimensions
+ */
+export function cosineDistance(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+
+  // Handle zero vectors
+  if (denominator === 0) {
+    return a.every(v => v === 0) && b.every(v => v === 0) ? 0 : 1;
+  }
+
+  const similarity = dotProduct / denominator;
+
+  // Convert similarity [-1, 1] to distance [0, 2]
+  return 1 - similarity;
+}
+
+/**
+ * Calculate euclidean (L2) distance between two vectors
+ *
+ * Range: [0, ∞)
+ * Best for: Geometric proximity, continuous data
+ *
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @returns {number} Euclidean distance
+ * @throws {Error} If vectors have different dimensions
+ */
+export function euclideanDistance(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i] - b[i];
+    sum += diff * diff;
+  }
+
+  return Math.sqrt(sum);
+}
+
+/**
+ * Calculate manhattan (L1) distance between two vectors
+ *
+ * Range: [0, ∞)
+ * Best for: Grid-based movement, faster computation
+ *
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @returns {number} Manhattan distance
+ * @throws {Error} If vectors have different dimensions
+ */
+export function manhattanDistance(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += Math.abs(a[i] - b[i]);
+  }
+
+  return sum;
+}
+
+/**
+ * Calculate dot product of two vectors
+ *
+ * Higher values indicate more similarity (for normalized vectors)
+ *
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @returns {number} Dot product
+ * @throws {Error} If vectors have different dimensions
+ */
+export function dotProduct(a, b) {
+  if (a.length !== b.length) {
+    throw new Error(`Dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  let sum = 0;
+  for (let i = 0; i < a.length; i++) {
+    sum += a[i] * b[i];
+  }
+
+  return sum;
+}
+
+/**
+ * Normalize a vector to unit length (L2 normalization)
+ *
+ * Converts vector to unit vector pointing in same direction.
+ * Useful for cosine similarity calculations.
+ *
+ * @param {number[]} vector - Vector to normalize
+ * @returns {number[]} Normalized vector
+ */
+export function normalize(vector) {
+  const magnitude = Math.sqrt(
+    vector.reduce((sum, val) => sum + val * val, 0)
+  );
+
+  // Handle zero vector
+  if (magnitude === 0) {
+    return vector.slice(); // Return copy of zero vector
+  }
+
+  return vector.map(val => val / magnitude);
+}
+
+/**
+ * Calculate the magnitude (length) of a vector
+ *
+ * @param {number[]} vector - Input vector
+ * @returns {number} Magnitude
+ */
+export function magnitude(vector) {
+  return Math.sqrt(
+    vector.reduce((sum, val) => sum + val * val, 0)
+  );
+}
+
+/**
+ * Check if two vectors are equal within a tolerance
+ *
+ * @param {number[]} a - First vector
+ * @param {number[]} b - Second vector
+ * @param {number} epsilon - Tolerance for floating point comparison
+ * @returns {boolean} True if vectors are equal within tolerance
+ */
+export function vectorsEqual(a, b, epsilon = 1e-10) {
+  if (a.length !== b.length) {
+    return false;
+  }
+
+  for (let i = 0; i < a.length; i++) {
+    if (Math.abs(a[i] - b[i]) > epsilon) {
+      return false;
+    }
+  }
+
+  return true;
+}

package/src/plugins/vector/kmeans.js

@@ -0,0 +1,367 @@
+/**
+ * K-Means Clustering Implementation
+ *
+ * Provides k-means clustering with k-means++ initialization
+ * and comprehensive optimal K analysis using multiple metrics.
+ */
+
+import { euclideanDistance } from './distances.js';
+
+/**
+ * K-means clustering algorithm
+ *
+ * @param {number[][]} vectors - Array of vectors to cluster
+ * @param {number} k - Number of clusters
+ * @param {Object} options - Configuration options
+ * @param {number} options.maxIterations - Maximum iterations (default: 100)
+ * @param {number} options.tolerance - Convergence tolerance (default: 0.0001)
+ * @param {Function} options.distanceFn - Distance function (default: euclideanDistance)
+ * @param {number|null} options.seed - Random seed for reproducibility (default: null)
+ * @param {Function} options.onIteration - Callback for each iteration (iteration, inertia, converged) (default: null)
+ * @returns {Object} Clustering results
+ * @throws {Error} If invalid parameters
+ */
+export function kmeans(vectors, k, options = {}) {
+  const {
+    maxIterations = 100,
+    tolerance = 0.0001,
+    distanceFn = euclideanDistance,
+    seed = null,
+    onIteration = null
+  } = options;
+
+  if (vectors.length === 0) {
+    throw new Error('Cannot cluster empty vector array');
+  }
+
+  if (k < 1) {
+    throw new Error(`k must be at least 1, got ${k}`);
+  }
+
+  if (k > vectors.length) {
+    throw new Error(`k (${k}) cannot be greater than number of vectors (${vectors.length})`);
+  }
+
+  const dimensions = vectors[0].length;
+
+  // Validate all vectors have same dimensions
+  for (let i = 1; i < vectors.length; i++) {
+    if (vectors[i].length !== dimensions) {
+      throw new Error(`All vectors must have same dimensions. Expected ${dimensions}, got ${vectors[i].length} at index ${i}`);
+    }
+  }
+
+  // Initialize centroids using k-means++
+  const centroids = initializeCentroidsKMeansPlusPlus(vectors, k, distanceFn, seed);
+
+  let assignments = new Array(vectors.length);
+  let iterations = 0;
+  let converged = false;
+  let previousInertia = Infinity;
+
+  while (!converged && iterations < maxIterations) {
+    // Assign each vector to nearest centroid
+    const newAssignments = vectors.map(vector => {
+      let minDist = Infinity;
+      let nearestCluster = 0;
+
+      for (let i = 0; i < k; i++) {
+        const dist = distanceFn(vector, centroids[i]);
+        if (dist < minDist) {
+          minDist = dist;
+          nearestCluster = i;
+        }
+      }
+
+      return nearestCluster;
+    });
+
+    // Calculate inertia (sum of squared distances to centroids)
+    let inertia = 0;
+    vectors.forEach((vector, i) => {
+      const dist = distanceFn(vector, centroids[newAssignments[i]]);
+      inertia += dist * dist;
+    });
+
+    // Check for convergence
+    const inertiaChange = Math.abs(previousInertia - inertia);
+    converged = inertiaChange < tolerance;
+
+    assignments = newAssignments;
+    previousInertia = inertia;
+
+    // Call onIteration callback if provided
+    if (onIteration) {
+      onIteration(iterations + 1, inertia, converged);
+    }
+
+    if (!converged) {
+      // Recalculate centroids
+      const clusterSums = Array(k).fill(null).map(() => new Array(dimensions).fill(0));
+      const clusterCounts = new Array(k).fill(0);
+
+      vectors.forEach((vector, i) => {
+        const cluster = assignments[i];
+        clusterCounts[cluster]++;
+        vector.forEach((val, j) => {
+          clusterSums[cluster][j] += val;
+        });
+      });
+
+      // Update centroids
+      for (let i = 0; i < k; i++) {
+        if (clusterCounts[i] > 0) {
+          centroids[i] = clusterSums[i].map(sum => sum / clusterCounts[i]);
+        }
+        // If cluster is empty, reinitialize to a random vector
+        else {
+          const randomIdx = Math.floor(Math.random() * vectors.length);
+          centroids[i] = [...vectors[randomIdx]];
+        }
+      }
+    }
+
+    iterations++;
+  }
+
+  // Final inertia calculation
+  let inertia = 0;
+  vectors.forEach((vector, i) => {
+    const dist = distanceFn(vector, centroids[assignments[i]]);
+    inertia += dist * dist;
+  });
+
+  return {
+    centroids,
+    assignments,
+    iterations,
+    converged,
+    inertia
+  };
+}
+
+/**
+ * Initialize centroids using k-means++ algorithm
+ *
+ * K-means++ provides better initialization than random selection,
+ * leading to faster convergence and better final clustering.
+ *
+ * @param {number[][]} vectors - Input vectors
+ * @param {number} k - Number of centroids
+ * @param {Function} distanceFn - Distance function
+ * @param {number|null} seed - Random seed
+ * @returns {number[][]} Initial centroids
+ */
+function initializeCentroidsKMeansPlusPlus(vectors, k, distanceFn, seed) {
+  const centroids = [];
+  const n = vectors.length;
+
+  // Choose first centroid randomly
+  const firstIndex = seed !== null ? seed % n : Math.floor(Math.random() * n);
+  centroids.push([...vectors[firstIndex]]);
+
+  // Choose remaining centroids
+  for (let i = 1; i < k; i++) {
+    // Calculate distance to nearest existing centroid
+    const distances = vectors.map(vector => {
+      return Math.min(...centroids.map(c => distanceFn(vector, c)));
+    });
+
+    // Square distances for probability distribution
+    const squaredDistances = distances.map(d => d * d);
+    const totalSquared = squaredDistances.reduce((a, b) => a + b, 0);
+
+    if (totalSquared === 0) {
+      // All remaining points are identical to existing centroids
+      // Choose randomly
+      const randomIdx = Math.floor(Math.random() * n);
+      centroids.push([...vectors[randomIdx]]);
+      continue;
+    }
+
+    // Choose next centroid with probability proportional to squared distance
+    let threshold = Math.random() * totalSquared;
+    let cumulativeSum = 0;
+
+    for (let j = 0; j < n; j++) {
+      cumulativeSum += squaredDistances[j];
+      if (cumulativeSum >= threshold) {
+        centroids.push([...vectors[j]]);
+        break;
+      }
+    }
+  }
+
+  return centroids;
+}
+
+/**
+ * Find optimal K using multiple evaluation metrics
+ *
+ * Analyzes clustering quality across a range of K values using:
+ * - Elbow method (inertia)
+ * - Silhouette score
+ * - Davies-Bouldin index
+ * - Calinski-Harabasz index
+ * - Gap statistic
+ * - Clustering stability
+ *
+ * @param {number[][]} vectors - Vectors to analyze
+ * @param {Object} options - Configuration options
+ * @param {number} options.minK - Minimum K to test (default: 2)
+ * @param {number} options.maxK - Maximum K to test (default: sqrt(n/2))
+ * @param {Function} options.distanceFn - Distance function (default: euclideanDistance)
+ * @param {number} options.nReferences - Number of reference datasets for Gap statistic (default: 10)
+ * @param {number} options.stabilityRuns - Number of runs for stability analysis (default: 5)
+ * @returns {Promise<Object>} Analysis results with recommendations
+ */
+export async function findOptimalK(vectors, options = {}) {
+  const {
+    minK = 2,
+    maxK = Math.min(10, Math.floor(Math.sqrt(vectors.length / 2))),
+    distanceFn = euclideanDistance,
+    nReferences = 10,
+    stabilityRuns = 5,
+    ...kmeansOptions
+  } = options;
+
+  // Dynamic import to avoid circular dependency
+  const metricsModule = await import('./metrics.js');
+  const {
+    silhouetteScore,
+    daviesBouldinIndex,
+    calinskiHarabaszIndex,
+    gapStatistic,
+    clusteringStability
+  } = metricsModule;
+
+  const results = [];
+
+  for (let k = minK; k <= maxK; k++) {
+    // Run k-means
+    const kmeansResult = kmeans(vectors, k, { ...kmeansOptions, distanceFn });
+
+    // Calculate all metrics
+    const silhouette = silhouetteScore(
+      vectors,
+      kmeansResult.assignments,
+      kmeansResult.centroids,
+      distanceFn
+    );
+
+    const daviesBouldin = daviesBouldinIndex(
+      vectors,
+      kmeansResult.assignments,
+      kmeansResult.centroids,
+      distanceFn
+    );
+
+    const calinskiHarabasz = calinskiHarabaszIndex(
+      vectors,
+      kmeansResult.assignments,
+      kmeansResult.centroids,
+      distanceFn
+    );
+
+    const gap = await gapStatistic(
+      vectors,
+      kmeansResult.assignments,
+      kmeansResult.centroids,
+      distanceFn,
+      nReferences
+    );
+
+    const stability = clusteringStability(
+      vectors,
+      k,
+      { ...kmeansOptions, distanceFn, nRuns: stabilityRuns }
+    );
+
+    results.push({
+      k,
+      inertia: kmeansResult.inertia,
+      silhouette,
+      daviesBouldin,
+      calinskiHarabasz,
+      gap: gap.gap,
+      gapSk: gap.sk,
+      stability: stability.stability,
+      cvInertia: stability.cvInertia,
+      iterations: kmeansResult.iterations,
+      converged: kmeansResult.converged
+    });
+  }
+
+  // Calculate elbow point
+  const elbowK = findElbowPoint(results.map(r => r.inertia));
+
+  // Find best K by each metric
+  const recommendations = {
+    elbow: minK + elbowK,
+    silhouette: results.reduce((best, curr) =>
+      curr.silhouette > best.silhouette ? curr : best
+    ).k,
+    daviesBouldin: results.reduce((best, curr) =>
+      curr.daviesBouldin < best.daviesBouldin ? curr : best
+    ).k,
+    calinskiHarabasz: results.reduce((best, curr) =>
+      curr.calinskiHarabasz > best.calinskiHarabasz ? curr : best
+    ).k,
+    gap: results.reduce((best, curr) =>
+      curr.gap > best.gap ? curr : best
+    ).k,
+    stability: results.reduce((best, curr) =>
+      curr.stability > best.stability ? curr : best
+    ).k
+  };
+
+  // Calculate consensus (most common recommendation)
+  const votes = Object.values(recommendations);
+  const consensus = votes.reduce((acc, k) => {
+    acc[k] = (acc[k] || 0) + 1;
+    return acc;
+  }, {});
+
+  const consensusK = parseInt(
+    Object.entries(consensus).reduce((a, b) => b[1] > a[1] ? b : a)[0]
+  );
+
+  return {
+    results,
+    recommendations,
+    consensus: consensusK,
+    summary: {
+      analysisRange: `${minK}-${maxK}`,
+      totalVectors: vectors.length,
+      dimensions: vectors[0].length,
+      recommendation: consensusK,
+      confidence: consensus[consensusK] / votes.length
+    }
+  };
+}
+
+/**
+ * Find elbow point using method of maximum curvature
+ *
+ * @param {number[]} inertias - Array of inertia values
+ * @returns {number} Index of elbow point
+ */
+function findElbowPoint(inertias) {
+  const n = inertias.length;
+  if (n < 3) return 0;
+
+  let maxCurvature = -Infinity;
+  let elbowIndex = 0;
+
+  for (let i = 1; i < n - 1; i++) {
+    // Calculate second derivative (curvature approximation)
+    const curvature = inertias[i - 1] - 2 * inertias[i] + inertias[i + 1];
+
+    if (curvature > maxCurvature) {
+      maxCurvature = curvature;
+      elbowIndex = i;
+    }
+  }
+
+  return elbowIndex;
+}