s3db.js 8.2.0 → 9.1.0

package/package.json

@@ -1,13 +1,10 @@
 {
   "name": "s3db.js",
-  "version": "8.2.0",
+  "version": "9.1.0",
   "description": "Use AWS S3, the world's most reliable document storage, as a database with this ORM.",
   "main": "dist/s3db.cjs.js",
   "module": "dist/s3db.es.js",
-  "browser": "dist/s3db.iife.js",
   "types": "dist/s3db.d.ts",
-  "unpkg": "dist/s3db.iife.min.js",
-  "jsdelivr": "dist/s3db.iife.min.js",
   "author": "@stone/martech",
   "license": "UNLICENSED",
   "bin": {
@@ -26,7 +23,14 @@
   "keywords": [
     "s3",
     "aws",
-    "database"
+    "database",
+    "orm",
+    "nosql",
+    "document-store",
+    "cloud-database",
+    "metadata-encoding",
+    "s3-database",
+    "serverless"
   ],
   "type": "module",
   "sideEffects": false,
@@ -39,24 +43,25 @@
   },
   "exports": {
     ".": {
+      "types": "./dist/s3db.d.ts",
       "import": "./dist/s3db.es.js",
-      "require": "./dist/s3db.cjs.js",
-      "types": "./dist/s3db.d.ts"
+      "require": "./dist/s3db.cjs.js"
     }
   },
   "files": [
-    "dist",
-    "src",
+    "dist/",
+    "src/",
+    "bin/cli.js",
+    "mcp/server.js",
     "README.md",
     "PLUGINS.md",
     "UNLICENSE"
   ],
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.850.0",
+    "@aws-sdk/client-s3": "^3.864.0",
     "@modelcontextprotocol/sdk": "^1.17.3",
-    "@smithy/node-http-handler": "^4.1.0",
+    "@smithy/node-http-handler": "^4.1.1",
     "@supercharge/promise-pool": "^3.2.0",
-    "commander": "^12.1.0",
     "dotenv": "^17.2.1",
     "fastest-validator": "^1.19.1",
     "flat": "^6.0.1",
@@ -89,43 +94,54 @@
     }
   },
   "devDependencies": {
+    "@babel/core": "^7.28.3",
+    "@babel/preset-env": "^7.28.3",
     "@rollup/plugin-commonjs": "^28.0.6",
     "@rollup/plugin-json": "^6.1.0",
     "@rollup/plugin-node-resolve": "^16.0.1",
     "@rollup/plugin-replace": "^6.0.2",
     "@rollup/plugin-terser": "^0.4.4",
-    "@types/node": "24.1.0",
+    "@types/node": "24.3.0",
+    "babel-loader": "^10.0.0",
+    "chalk": "^5.5.0",
+    "cli-table3": "^0.6.5",
+    "commander": "^14.0.0",
+    "esbuild": "^0.25.9",
+    "inquirer": "^12.9.2",
     "jest": "^30.0.5",
-    "rollup": "^4.46.1",
+    "node-loader": "^2.1.0",
+    "ora": "^8.2.0",
+    "pkg": "^5.8.1",
+    "rollup": "^4.46.2",
     "rollup-plugin-copy": "^3.5.0",
     "rollup-plugin-esbuild": "^6.2.1",
     "rollup-plugin-polyfill-node": "^0.13.0",
+    "rollup-plugin-shebang-bin": "^0.1.0",
     "rollup-plugin-terser": "^7.0.2",
-    "typescript": "5.8.3"
+    "typescript": "5.9.2",
+    "webpack": "^5.101.2",
+    "webpack-cli": "^6.0.1"
   },
   "funding": [
     "https://github.com/sponsors/forattini-dev"
   ],
   "scripts": {
     "build": "rollup -c",
+    "build:cli": "rollup -c rollup.cli.config.mjs",
+    "build:binaries": "./scripts/scripts/build-binaries.sh",
     "dev": "rollup -c -w",
-    "test": "npm run test:js && npm run test:ts",
+    "test": "pnpm run test:js && pnpm run test:ts",
     "test:js": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --testTimeout=10000",
     "test:ts": "tsc --noEmit --project tests/typescript/tsconfig.json",
-    "test:js-converage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage --runInBand",
-    "test:js-ai": "node --max-old-space-size=4096 --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --testPathIgnorePatterns=\"plugin-audit.test.js|tests/typescript/\"",
-    "test:full": "npm run test:js-ai && npm run test:ts",
-    "test:audit": "node --max-old-space-size=8192 --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-audit.test.js --runInBand --testTimeout=60000",
-    "test:cache": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-cache*.test.js --runInBand",
+    "test:coverage": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --detectOpenHandles --coverage --runInBand",
     "test:quick": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js --runInBand --testTimeout=10000",
-    "test:batch": "./test-batch.sh",
     "test:plugins": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=60000",
-    "test:plugins:fast": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/ --runInBand --testTimeout=15000 --testPathIgnorePatterns='plugin-audit.test.js|plugin-replicator-s3db.test.js|plugin-fulltext.test.js'",
-    "test:slow": "node --no-warnings --experimental-vm-modules node_modules/jest/bin/jest.js tests/plugins/plugin-audit.test.js tests/plugins/plugin-replicator-s3db.test.js tests/plugins/plugin-fulltext.test.js --runInBand --testTimeout=120000",
-    "test:types": "tsc --noEmit --project tests/typescript/tsconfig.json",
-    "test:types:basic": "tsc --noEmit tests/typescript/basic-usage.test.ts",
-    "test:types:direct": "tsc --noEmit tests/typescript/direct-type-test.ts",
-    "test:types:watch": "tsc --noEmit --watch --project tests/typescript/tsconfig.json",
-    "validate:types": "npm run test:types && echo 'TypeScript definitions are valid!'"
+    "test:full": "pnpm run test:js && pnpm run test:ts",
+    "benchmark": "node benchmark-compression.js",
+    "version": "echo 'Use pnpm run release v<version> instead of npm version'",
+    "release:check": "./scripts/pre-release-check.sh",
+    "release:prepare": "pnpm run build:binaries && echo 'Binaries ready for GitHub release'",
+    "release": "./scripts/release.sh",
+    "validate:types": "pnpm run test:ts && echo 'TypeScript definitions are valid!'"
   }
 }

package/src/cli/index.js

@@ -0,0 +1,426 @@
+#!/usr/bin/env node
+
+import { Command } from 'commander';
+import chalk from 'chalk';
+import ora from 'ora';
+import Table from 'cli-table3';
+import inquirer from 'inquirer';
+import { S3db } from '../database.class.js';
+import fs from 'fs/promises';
+import path from 'path';
+import os from 'os';
+
+const program = new Command();
+const configPath = path.join(os.homedir(), '.s3db', 'config.json');
+
+// Helper to load config
+async function loadConfig() {
+  try {
+    const data = await fs.readFile(configPath, 'utf-8');
+    return JSON.parse(data);
+  } catch {
+    return {};
+  }
+}
+
+// Helper to save config
+async function saveConfig(config) {
+  const dir = path.dirname(configPath);
+  await fs.mkdir(dir, { recursive: true });
+  await fs.writeFile(configPath, JSON.stringify(config, null, 2));
+}
+
+// Connect to database
+async function getDatabase(options) {
+  const config = await loadConfig();
+  const connectionString = options.connection || config.connection || process.env.S3DB_CONNECTION;
+  
+  if (!connectionString) {
+    console.error(chalk.red('No connection string provided. Use --connection or s3db configure'));
+    process.exit(1);
+  }
+  
+  return new S3db({ connectionString });
+}
+
+program
+  .name('s3db')
+  .description('S3DB CLI - Transform AWS S3 into a powerful document database')
+  .version('9.0.0');
+
+// Configure command
+program
+  .command('configure')
+  .description('Configure S3DB connection')
+  .action(async () => {
+    const answers = await inquirer.prompt([
+      {
+        type: 'input',
+        name: 'connection',
+        message: 'Enter S3 connection string:',
+        default: 's3://KEY:SECRET@bucket/database'
+      },
+      {
+        type: 'list',
+        name: 'defaultBehavior',
+        message: 'Default behavior for resources:',
+        choices: ['user-managed', 'enforce-limits', 'body-overflow', 'body-only', 'truncate-data'],
+        default: 'user-managed'
+      }
+    ]);
+    
+    await saveConfig(answers);
+    console.log(chalk.green('✓ Configuration saved to ~/.s3db/config.json'));
+  });
+
+// List resources
+program
+  .command('list')
+  .description('List all resources')
+  .option('-c, --connection <string>', 'Connection string')
+  .action(async (options) => {
+    const spinner = ora('Connecting to S3DB...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const resources = await db.listResources();
+      spinner.stop();
+      
+      if (resources.length === 0) {
+        console.log(chalk.yellow('No resources found'));
+        return;
+      }
+      
+      const table = new Table({
+        head: ['Resource', 'Behavior', 'Timestamps', 'Paranoid', 'Partitions'],
+        style: { head: ['cyan'] }
+      });
+      
+      resources.forEach(r => {
+        table.push([
+          r.name,
+          r.config.behavior || 'user-managed',
+          r.config.timestamps ? '✓' : '✗',
+          r.config.paranoid ? '✓' : '✗',
+          Object.keys(r.config.partitions || {}).length
+        ]);
+      });
+      
+      console.log(table.toString());
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Query resource
+program
+  .command('query <resource>')
+  .description('Query a resource')
+  .option('-c, --connection <string>', 'Connection string')
+  .option('-l, --limit <number>', 'Limit results', '10')
+  .option('-f, --filter <json>', 'Filter as JSON')
+  .option('-p, --partition <name>', 'Partition name')
+  .option('--csv', 'Output as CSV')
+  .option('--json', 'Output as JSON')
+  .action(async (resourceName, options) => {
+    const spinner = ora('Querying...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const resource = await db.resource(resourceName);
+      
+      const queryOptions = {
+        limit: parseInt(options.limit)
+      };
+      
+      if (options.filter) {
+        queryOptions.filter = JSON.parse(options.filter);
+      }
+      
+      if (options.partition) {
+        queryOptions.partition = options.partition;
+      }
+      
+      const results = await resource.list(queryOptions);
+      spinner.stop();
+      
+      if (options.json) {
+        console.log(JSON.stringify(results, null, 2));
+      } else if (options.csv) {
+        if (results.length > 0) {
+          const headers = Object.keys(results[0]);
+          console.log(headers.join(','));
+          results.forEach(row => {
+            console.log(headers.map(h => JSON.stringify(row[h] || '')).join(','));
+          });
+        }
+      } else {
+        // Table output
+        if (results.length === 0) {
+          console.log(chalk.yellow('No results found'));
+          return;
+        }
+        
+        const headers = Object.keys(results[0]);
+        const table = new Table({
+          head: headers,
+          style: { head: ['cyan'] }
+        });
+        
+        results.forEach(row => {
+          table.push(headers.map(h => {
+            const val = row[h];
+            if (val === null || val === undefined) return '';
+            if (typeof val === 'object') return JSON.stringify(val);
+            return String(val).substring(0, 50);
+          }));
+        });
+        
+        console.log(table.toString());
+      }
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Insert data
+program
+  .command('insert <resource>')
+  .description('Insert data into a resource')
+  .option('-c, --connection <string>', 'Connection string')
+  .option('-d, --data <json>', 'Data as JSON')
+  .option('-f, --file <path>', 'Read data from file')
+  .action(async (resourceName, options) => {
+    const spinner = ora('Inserting...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const resource = await db.resource(resourceName);
+      
+      let data;
+      if (options.file) {
+        const content = await fs.readFile(options.file, 'utf-8');
+        data = JSON.parse(content);
+      } else if (options.data) {
+        data = JSON.parse(options.data);
+      } else {
+        spinner.fail('No data provided. Use --data or --file');
+        process.exit(1);
+      }
+      
+      const result = await resource.insert(data);
+      spinner.succeed(chalk.green(`✓ Inserted with ID: ${result.id}`));
+      
+      if (!options.quiet) {
+        console.log(JSON.stringify(result, null, 2));
+      }
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Update data
+program
+  .command('update <resource> <id>')
+  .description('Update a record')
+  .option('-c, --connection <string>', 'Connection string')
+  .option('-d, --data <json>', 'Data as JSON')
+  .action(async (resourceName, id, options) => {
+    const spinner = ora('Updating...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const resource = await db.resource(resourceName);
+      const data = JSON.parse(options.data || '{}');
+      
+      const result = await resource.update(id, data);
+      spinner.succeed(chalk.green(`✓ Updated ID: ${id}`));
+      
+      console.log(JSON.stringify(result, null, 2));
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Delete data
+program
+  .command('delete <resource> <id>')
+  .description('Delete a record')
+  .option('-c, --connection <string>', 'Connection string')
+  .option('--force', 'Force delete (no confirmation)')
+  .action(async (resourceName, id, options) => {
+    if (!options.force) {
+      const { confirm } = await inquirer.prompt([
+        {
+          type: 'confirm',
+          name: 'confirm',
+          message: `Are you sure you want to delete ${id} from ${resourceName}?`,
+          default: false
+        }
+      ]);
+      
+      if (!confirm) {
+        console.log(chalk.yellow('Cancelled'));
+        return;
+      }
+    }
+    
+    const spinner = ora('Deleting...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const resource = await db.resource(resourceName);
+      await resource.delete(id);
+      
+      spinner.succeed(chalk.green(`✓ Deleted ID: ${id}`));
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Create resource
+program
+  .command('create-resource <name>')
+  .description('Create a new resource')
+  .option('-c, --connection <string>', 'Connection string')
+  .option('-s, --schema <json>', 'Schema as JSON')
+  .option('-b, --behavior <type>', 'Behavior type', 'user-managed')
+  .option('--timestamps', 'Enable timestamps')
+  .option('--paranoid', 'Enable soft deletes')
+  .action(async (name, options) => {
+    const spinner = ora('Creating resource...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      const config = {
+        name,
+        behavior: options.behavior,
+        timestamps: options.timestamps,
+        paranoid: options.paranoid
+      };
+      
+      if (options.schema) {
+        config.attributes = JSON.parse(options.schema);
+      }
+      
+      const resource = await db.createResource(config);
+      spinner.succeed(chalk.green(`✓ Created resource: ${name}`));
+      
+      console.log(JSON.stringify(resource.config, null, 2));
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+// Interactive mode
+program
+  .command('interactive')
+  .description('Interactive REPL mode')
+  .option('-c, --connection <string>', 'Connection string')
+  .action(async (options) => {
+    console.log(chalk.cyan('S3DB Interactive Mode'));
+    console.log(chalk.gray('Type "help" for commands, "exit" to quit\n'));
+    
+    const db = await getDatabase(options);
+    await db.init();
+    
+    const repl = await import('repl');
+    const server = repl.start({
+      prompt: chalk.green('s3db> '),
+      eval: async (cmd, context, filename, callback) => {
+        try {
+          // Make db available in REPL
+          context.db = db;
+          
+          // Parse commands
+          const trimmed = cmd.trim();
+          if (trimmed === 'help') {
+            console.log(`
+Available commands:
+  db                    - Database instance
+  db.listResources()    - List all resources
+  db.resource('name')   - Get a resource
+  await ...             - Use await for async operations
+  .exit                 - Exit REPL
+            `);
+            callback(null);
+          } else {
+            // Default eval
+            const result = await eval(cmd);
+            callback(null, result);
+          }
+        } catch (error) {
+          callback(error);
+        }
+      }
+    });
+    
+    server.setupHistory(path.join(os.homedir(), '.s3db', 'history'), () => {});
+  });
+
+// Stats command
+program
+  .command('stats [resource]')
+  .description('Show statistics')
+  .option('-c, --connection <string>', 'Connection string')
+  .action(async (resourceName, options) => {
+    const spinner = ora('Gathering stats...').start();
+    
+    try {
+      const db = await getDatabase(options);
+      await db.init();
+      
+      if (resourceName) {
+        const resource = await db.resource(resourceName);
+        const count = await resource.count();
+        spinner.stop();
+        
+        console.log(chalk.cyan(`\nResource: ${resourceName}`));
+        console.log(`Total records: ${count}`);
+      } else {
+        const resources = await db.listResources();
+        spinner.stop();
+        
+        console.log(chalk.cyan('\nDatabase Statistics'));
+        console.log(`Total resources: ${resources.length}`);
+        
+        if (resources.length > 0) {
+          const table = new Table({
+            head: ['Resource', 'Count'],
+            style: { head: ['cyan'] }
+          });
+          
+          for (const r of resources) {
+            const resource = await db.resource(r.name);
+            const count = await resource.count();
+            table.push([r.name, count]);
+          }
+          
+          console.log(table.toString());
+        }
+      }
+    } catch (error) {
+      spinner.fail(chalk.red(error.message));
+      process.exit(1);
+    }
+  });
+
+program.parse(process.argv);

package/src/client.class.js

@@ -20,6 +20,7 @@ import {
 import tryFn from "./concerns/try-fn.js";
 import { md5 } from "./concerns/crypto.js";
 import { idGenerator } from "./concerns/id.js";
+import { metadataEncode, metadataDecode } from "./concerns/metadata-encoding.js";
 import { ConnectionString } from "./connection-string.class.js";
 import { mapAwsError, UnknownError, NoSuchKey, NotFound } from "./errors.js";
 
@@ -119,23 +120,16 @@ export class Client extends EventEmitter {
     const keyPrefix = typeof this.config.keyPrefix === 'string' ? this.config.keyPrefix : '';
     const fullKey = keyPrefix ? path.join(keyPrefix, key) : key;
     
-    // Ensure all metadata values are strings and keys are valid
+    // Ensure all metadata values are strings and use smart encoding
     const stringMetadata = {};
     if (metadata) {
       for (const [k, v] of Object.entries(metadata)) {
-        // Ensure key is a valid string and value is a string
+        // Ensure key is a valid string
         const validKey = String(k).replace(/[^a-zA-Z0-9\-_]/g, '_');
-        const stringValue = String(v);
         
-        // Check if value contains non-ASCII characters that might be corrupted by HTTP headers
-        const hasSpecialChars = /[^\x00-\x7F]/.test(stringValue);
-        
-        if (hasSpecialChars) {
-          // Encode as base64 without prefix - we'll detect it intelligently on read
-          stringMetadata[validKey] = Buffer.from(stringValue, 'utf8').toString('base64');
-        } else {
-          stringMetadata[validKey] = stringValue;
-        }
+        // Smart encode the value
+        const { encoded } = metadataEncode(v);
+        stringMetadata[validKey] = encoded;
       }
     }
     
@@ -178,30 +172,11 @@ export class Client extends EventEmitter {
     try {
       response = await this.sendCommand(new GetObjectCommand(options));
       
-      // Smart decode: try to detect base64-encoded UTF-8 values without prefix
+      // Smart decode metadata values
       if (response.Metadata) {
         const decodedMetadata = {};
         for (const [key, value] of Object.entries(response.Metadata)) {
-          if (typeof value === 'string') {
-            // Try to decode as base64 and check if it's valid UTF-8 with special chars
-            try {
-              const decoded = Buffer.from(value, 'base64').toString('utf8');
-              // Check if decoded string is different from original and contains non-ASCII chars
-              const hasSpecialChars = /[^\x00-\x7F]/.test(decoded);
-              const isValidBase64 = Buffer.from(decoded, 'utf8').toString('base64') === value;
-              
-              if (isValidBase64 && hasSpecialChars && decoded !== value) {
-                decodedMetadata[key] = decoded;
-              } else {
-                decodedMetadata[key] = value;
-              }
-            } catch (decodeError) {
-              // If decode fails, use original value
-              decodedMetadata[key] = value;
-            }
-          } else {
-            decodedMetadata[key] = value;
-          }
+          decodedMetadata[key] = metadataDecode(value);
         }
         response.Metadata = decodedMetadata;
       }