s3db.js 4.1.7 → 4.1.10

package/README.md

@@ -283,7 +283,7 @@ const googleUserIds = await users.listIds({
   partitionValues: { 'utm.source': 'google' }
 });
 
-// Paginar resultados
+// Paginar resultados com contagem total
 const page = await users.page({
   partition: 'byUtmSource',
   partitionValues: { 'utm.source': 'google' },
@@ -291,7 +291,20 @@ const page = await users.page({
   size: 10
 });
 console.log(page.items); // Array de usuários
-console.log(page.totalItems, page.totalPages);
+console.log(page.totalItems, page.totalPages); // Contagem total e páginas
+console.log(page.page, page.pageSize); // Página atual e tamanho
+
+// Paginar resultados sem contagem total (mais rápido para grandes coleções)
+const fastPage = await users.page({
+  partition: 'byUtmSource',
+  partitionValues: { 'utm.source': 'google' },
+  offset: 0,
+  size: 10,
+  skipCount: true // Pula a contagem total para melhor performance
+});
+console.log(fastPage.items); // Array de usuários
+console.log(fastPage.totalItems); // null (não contado)
+console.log(fastPage._debug); // Informações de debug
 
 // Contar documentos em uma partição
 const count = await users.count({
@@ -960,6 +973,73 @@ const count = await users.count();
 console.log(`Total users: ${count}`);
 ```
 
+#### Page Documents
+
+The `page()` method provides efficient pagination with optional total count for performance optimization.
+
+```javascript
+// Basic pagination with total count
+const page = await users.page({
+  offset: 0,
+  size: 10
+});
+
+console.log(page.items); // Array of user objects
+console.log(page.totalItems); // Total number of items
+console.log(page.totalPages); // Total number of pages
+console.log(page.page); // Current page number (0-based)
+console.log(page.pageSize); // Items per page
+console.log(page._debug); // Debug information
+
+// Pagination with partition filtering
+const googleUsersPage = await users.page({
+  partition: 'byUtmSource',
+  partitionValues: { 'utm.source': 'google' },
+  offset: 0,
+  size: 5
+});
+
+// Skip total count for better performance on large collections
+const fastPage = await users.page({
+  offset: 0,
+  size: 100,
+  skipCount: true // Skips counting total items
+});
+
+console.log(fastPage.totalItems); // null (not counted)
+console.log(fastPage.totalPages); // null (not calculated)
+console.log(fastPage._debug.skipCount); // true
+```
+
+**Page Response Structure:**
+
+```javascript
+{
+  items: Array,           // Array of document objects
+  totalItems: number,     // Total count (null if skipCount: true)
+  page: number,           // Current page number (0-based)
+  pageSize: number,       // Number of items per page
+  totalPages: number,     // Total pages (null if skipCount: true)
+  _debug: {               // Debug information
+    requestedSize: number,
+    requestedOffset: number,
+    actualItemsReturned: number,
+    skipCount: boolean,
+    hasTotalItems: boolean
+  }
+}
+```
+
+**Parameters:**
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `offset` | `number` | `0` | Number of items to skip |
+| `size` | `number` | `100` | Number of items per page |
+| `partition` | `string` | `null` | Partition name to filter by |
+| `partitionValues` | `object` | `{}` | Partition field values |
+| `skipCount` | `boolean` | `false` | Skip total count for performance |
+
 ### Bulk Operations
 
 #### Insert Many
@@ -1680,7 +1760,7 @@ await products.insert({
 | `listIds()` | Get array of document IDs | ✅ `{ partition, partitionValues }` |
 | `count()` | Count documents | ✅ `{ partition, partitionValues }` |
 | `listByPartition()` | List documents by partition | ✅ `{ partition, partitionValues }` |
-| `page()` | Paginate documents | ✅ `{ partition, partitionValues }` |
+| `page()` | Paginate documents | ✅ `{ partition, partitionValues, skipCount }` |
 | `getFromPartition()` | Get single document from partition | ✅ Direct partition access |
 | `query()` | Filter documents in memory | ❌ No partition support |
 
@@ -1767,19 +1847,6 @@ const s3db = new S3db({
 - **`enforce-limits`**: Strict validation to prevent oversized documents
 - **`user-management`**: Default behavior with warnings and monitoring
 
-### ✅ Recent Improvements
-
-**🔧 Enhanced Data Serialization (v3.3.2+)**
-
-s3db.js now handles complex data structures robustly:
-
-- **Empty Arrays**: `[]` correctly serialized and preserved
-- **Null Arrays**: `null` values maintained without corruption  
-- **Special Characters**: Arrays with pipe `|` characters properly escaped
-- **Empty Objects**: `{}` correctly mapped and stored
-- **Null Objects**: `null` object values preserved during serialization
-- **Nested Structures**: Complex nested objects with mixed empty/null values supported
-
 ### Best Practices
 
 #### 1. Design for Document Storage

package/dist/s3db.cjs.js

@@ -9236,36 +9236,97 @@ class Resource extends EventEmitter {
    */
   async get(id) {
     const key = this.getResourceKey(id);
-    const request = await this.client.headObject(key);
-    const objectVersion = this.extractVersionFromKey(key) || this.version;
-    const schema = await this.getSchemaForVersion(objectVersion);
-    let metadata = await schema.unmapper(request.Metadata);
-    const behaviorImpl = getBehavior(this.behavior);
-    let body = "";
-    if (request.ContentLength > 0) {
-      try {
-        const fullObject = await this.client.getObject(key);
-        body = await streamToString(fullObject.Body);
-      } catch (error) {
-        body = "";
+    try {
+      const request = await this.client.headObject(key);
+      const objectVersion = this.extractVersionFromKey(key) || this.version;
+      const schema = await this.getSchemaForVersion(objectVersion);
+      let metadata = await schema.unmapper(request.Metadata);
+      const behaviorImpl = getBehavior(this.behavior);
+      let body = "";
+      if (request.ContentLength > 0) {
+        try {
+          const fullObject = await this.client.getObject(key);
+          body = await streamToString(fullObject.Body);
+        } catch (error) {
+          console.warn(`Failed to read body for resource ${id}:`, error.message);
+          body = "";
+        }
       }
+      const { metadata: processedMetadata } = await behaviorImpl.handleGet({
+        resource: this,
+        metadata,
+        body
+      });
+      let data = processedMetadata;
+      data.id = id;
+      data._contentLength = request.ContentLength;
+      data._lastModified = request.LastModified;
+      data._hasContent = request.ContentLength > 0;
+      data._mimeType = request.ContentType || null;
+      if (request.VersionId) data._versionId = request.VersionId;
+      if (request.Expiration) data._expiresAt = request.Expiration;
+      data._definitionHash = this.getDefinitionHash();
+      this.emit("get", data);
+      return data;
+    } catch (error) {
+      if (error.message.includes("Cipher job failed") || error.message.includes("OperationError") || error.originalError?.message?.includes("Cipher job failed")) {
+        try {
+          console.warn(`Decryption failed for resource ${id}, attempting to get raw metadata`);
+          const request = await this.client.headObject(key);
+          const objectVersion = this.extractVersionFromKey(key) || this.version;
+          const tempSchema = new Schema({
+            name: this.name,
+            attributes: this.attributes,
+            passphrase: this.passphrase,
+            version: objectVersion,
+            options: {
+              ...this.options,
+              autoDecrypt: false,
+              // Disable decryption
+              autoEncrypt: false
+              // Disable encryption
+            }
+          });
+          let metadata = await tempSchema.unmapper(request.Metadata);
+          const behaviorImpl = getBehavior(this.behavior);
+          let body = "";
+          if (request.ContentLength > 0) {
+            try {
+              const fullObject = await this.client.getObject(key);
+              body = await streamToString(fullObject.Body);
+            } catch (bodyError) {
+              console.warn(`Failed to read body for resource ${id}:`, bodyError.message);
+              body = "";
+            }
+          }
+          const { metadata: processedMetadata } = await behaviorImpl.handleGet({
+            resource: this,
+            metadata,
+            body
+          });
+          let data = processedMetadata;
+          data.id = id;
+          data._contentLength = request.ContentLength;
+          data._lastModified = request.LastModified;
+          data._hasContent = request.ContentLength > 0;
+          data._mimeType = request.ContentType || null;
+          data._version = objectVersion;
+          data._decryptionFailed = true;
+          if (request.VersionId) data._versionId = request.VersionId;
+          if (request.Expiration) data._expiresAt = request.Expiration;
+          data._definitionHash = this.getDefinitionHash();
+          this.emit("get", data);
+          return data;
+        } catch (fallbackError) {
+          console.error(`Fallback attempt also failed for resource ${id}:`, fallbackError.message);
+        }
+      }
+      const enhancedError = new Error(`Failed to get resource with id '${id}': ${error.message}`);
+      enhancedError.originalError = error;
+      enhancedError.resourceId = id;
+      enhancedError.resourceKey = key;
+      throw enhancedError;
     }
-    const { metadata: processedMetadata } = await behaviorImpl.handleGet({
-      resource: this,
-      metadata,
-      body
-    });
-    let data = processedMetadata;
-    data.id = id;
-    data._contentLength = request.ContentLength;
-    data._lastModified = request.LastModified;
-    data._hasContent = request.ContentLength > 0;
-    data._mimeType = request.ContentType || null;
-    if (request.VersionId) data._versionId = request.VersionId;
-    if (request.Expiration) data._expiresAt = request.Expiration;
-    data._definitionHash = this.getDefinitionHash();
-    this.emit("get", data);
-    return data;
   }
   /**
    * Check if a resource exists by ID
@@ -9547,15 +9608,21 @@ class Resource extends EventEmitter {
     return { deletedCount, resource: this.name };
   }
   /**
-   * List resource IDs with optional partition filtering
+   * List resource IDs with optional partition filtering and pagination
    * @param {Object} [params] - List parameters
    * @param {string} [params.partition] - Partition name to list from
    * @param {Object} [params.partitionValues] - Partition field values to filter by
+   * @param {number} [params.limit] - Maximum number of results to return
+   * @param {number} [params.offset=0] - Offset for pagination
    * @returns {Promise<string[]>} Array of resource IDs (strings)
    * @example
    * // List all IDs
    * const allIds = await resource.listIds();
    * 
+   * // List IDs with pagination
+   * const firstPageIds = await resource.listIds({ limit: 10, offset: 0 });
+   * const secondPageIds = await resource.listIds({ limit: 10, offset: 10 });
+   * 
    * // List IDs from specific partition
    * const googleUserIds = await resource.listIds({
    *   partition: 'byUtmSource',
@@ -9568,7 +9635,7 @@ class Resource extends EventEmitter {
    *   partitionValues: { category: 'electronics', region: 'US' }
    * });
    */
-  async listIds({ partition = null, partitionValues = {} } = {}) {
+  async listIds({ partition = null, partitionValues = {}, limit, offset = 0 } = {}) {
     let prefix;
     if (partition && Object.keys(partitionValues).length > 0) {
       const partitionDef = this.options.partitions[partition];
@@ -9592,8 +9659,11 @@ class Resource extends EventEmitter {
     } else {
       prefix = `resource=${this.name}/v=${this.version}`;
     }
-    const keys = await this.client.getAllKeys({
-      prefix
+    const keys = await this.client.getKeysPage({
+      prefix,
+      offset,
+      amount: limit || 1e3
+      // Default to 1000 if no limit specified
     });
     const ids = keys.map((key) => {
       const parts = key.split("/");
@@ -9640,11 +9710,27 @@ class Resource extends EventEmitter {
       if (limit) {
         filteredIds2 = filteredIds2.slice(0, limit);
       }
-      const { results: results2 } = await promisePool.PromisePool.for(filteredIds2).withConcurrency(this.parallelism).process(async (id) => {
-        return await this.get(id);
+      const { results: results2, errors: errors2 } = await promisePool.PromisePool.for(filteredIds2).withConcurrency(this.parallelism).handleError(async (error, id) => {
+        console.warn(`Failed to get resource ${id}:`, error.message);
+        return null;
+      }).process(async (id) => {
+        try {
+          return await this.get(id);
+        } catch (error) {
+          if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
+            console.warn(`Decryption failed for ${id}, returning basic info`);
+            return {
+              id,
+              _decryptionFailed: true,
+              _error: error.message
+            };
+          }
+          throw error;
+        }
       });
-      this.emit("list", { partition, partitionValues, count: results2.length });
-      return results2;
+      const validResults2 = results2.filter((item) => item !== null);
+      this.emit("list", { partition, partitionValues, count: validResults2.length, errors: errors2.length });
+      return validResults2;
     }
     const partitionDef = this.options.partitions[partition];
     if (!partitionDef) {
@@ -9675,11 +9761,29 @@ class Resource extends EventEmitter {
     if (limit) {
       filteredIds = filteredIds.slice(0, limit);
     }
-    const { results } = await promisePool.PromisePool.for(filteredIds).withConcurrency(this.parallelism).process(async (id) => {
-      return await this.getFromPartition({ id, partitionName: partition, partitionValues });
+    const { results, errors } = await promisePool.PromisePool.for(filteredIds).withConcurrency(this.parallelism).handleError(async (error, id) => {
+      console.warn(`Failed to get partition resource ${id}:`, error.message);
+      return null;
+    }).process(async (id) => {
+      try {
+        return await this.getFromPartition({ id, partitionName: partition, partitionValues });
+      } catch (error) {
+        if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
+          console.warn(`Decryption failed for partition resource ${id}, returning basic info`);
+          return {
+            id,
+            _partition: partition,
+            _partitionValues: partitionValues,
+            _decryptionFailed: true,
+            _error: error.message
+          };
+        }
+        throw error;
+      }
     });
-    this.emit("list", { partition, partitionValues, count: results.length });
-    return results;
+    const validResults = results.filter((item) => item !== null);
+    this.emit("list", { partition, partitionValues, count: validResults.length, errors: errors.length });
+    return validResults;
   }
   /**
    * Get multiple resources by their IDs
@@ -9690,11 +9794,30 @@ class Resource extends EventEmitter {
    * users.forEach(user => console.log(user.name));
    */
   async getMany(ids) {
-    const { results } = await promisePool.PromisePool.for(ids).withConcurrency(this.client.parallelism).process(async (id) => {
+    const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.client.parallelism).handleError(async (error, id) => {
+      console.warn(`Failed to get resource ${id}:`, error.message);
+      return {
+        id,
+        _error: error.message,
+        _decryptionFailed: error.message.includes("Cipher job failed") || error.message.includes("OperationError")
+      };
+    }).process(async (id) => {
       this.emit("id", id);
-      const data = await this.get(id);
-      this.emit("data", data);
-      return data;
+      try {
+        const data = await this.get(id);
+        this.emit("data", data);
+        return data;
+      } catch (error) {
+        if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
+          console.warn(`Decryption failed for ${id}, returning basic info`);
+          return {
+            id,
+            _decryptionFailed: true,
+            _error: error.message
+          };
+        }
+        throw error;
+      }
     });
     this.emit("getMany", ids.length);
     return results;
@@ -9709,9 +9832,28 @@ class Resource extends EventEmitter {
   async getAll() {
     let ids = await this.listIds();
     if (ids.length === 0) return [];
-    const { results } = await promisePool.PromisePool.for(ids).withConcurrency(this.client.parallelism).process(async (id) => {
-      const data = await this.get(id);
-      return data;
+    const { results, errors } = await promisePool.PromisePool.for(ids).withConcurrency(this.client.parallelism).handleError(async (error, id) => {
+      console.warn(`Failed to get resource ${id}:`, error.message);
+      return {
+        id,
+        _error: error.message,
+        _decryptionFailed: error.message.includes("Cipher job failed") || error.message.includes("OperationError")
+      };
+    }).process(async (id) => {
+      try {
+        const data = await this.get(id);
+        return data;
+      } catch (error) {
+        if (error.message.includes("Cipher job failed") || error.message.includes("OperationError")) {
+          console.warn(`Decryption failed for ${id}, returning basic info`);
+          return {
+            id,
+            _decryptionFailed: true,
+            _error: error.message
+          };
+        }
+        throw error;
+      }
     });
     this.emit("getAll", results.length);
     return results;
@@ -9723,6 +9865,7 @@ class Resource extends EventEmitter {
    * @param {number} [params.size=100] - Page size
    * @param {string} [params.partition] - Partition name to page from
    * @param {Object} [params.partitionValues] - Partition field values to filter by
+   * @param {boolean} [params.skipCount=false] - Skip total count for performance (useful for large collections)
    * @returns {Promise<Object>} Page result with items and pagination info
    * @example
    * // Get first page of all resources
@@ -9737,22 +9880,43 @@ class Resource extends EventEmitter {
    *   offset: 0,
    *   size: 5
    * });
+   * 
+   * // Skip count for performance in large collections
+   * const fastPage = await resource.page({ 
+   *   offset: 0, 
+   *   size: 100, 
+   *   skipCount: true 
+   * });
+   * console.log(`Got ${fastPage.items.length} items`); // totalItems will be null
    */
-  async page({ offset = 0, size = 100, partition = null, partitionValues = {} } = {}) {
-    const ids = await this.listIds({ partition, partitionValues });
-    const totalItems = ids.length;
-    const totalPages = Math.ceil(totalItems / size);
+  async page({ offset = 0, size = 100, partition = null, partitionValues = {}, skipCount = false } = {}) {
+    let totalItems = null;
+    let totalPages = null;
+    if (!skipCount) {
+      totalItems = await this.count({ partition, partitionValues });
+      totalPages = Math.ceil(totalItems / size);
+    }
     const page = Math.floor(offset / size);
-    const pageIds = ids.slice(offset, offset + size);
-    const items = await Promise.all(
-      pageIds.map((id) => this.get(id))
-    );
+    const items = await this.list({
+      partition,
+      partitionValues,
+      limit: size,
+      offset
+    });
     const result = {
       items,
       totalItems,
       page,
       pageSize: size,
-      totalPages
+      totalPages,
+      // Add additional metadata for debugging
+      _debug: {
+        requestedSize: size,
+        requestedOffset: offset,
+        actualItemsReturned: items.length,
+        skipCount,
+        hasTotalItems: totalItems !== null
+      }
     };
     this.emit("page", result);
     return result;
@@ -9904,7 +10068,27 @@ class Resource extends EventEmitter {
    * @returns {Object} Schema object for the version
    */
   async getSchemaForVersion(version) {
-    return this.schema;
+    if (version === this.version) {
+      return this.schema;
+    }
+    try {
+      const compatibleSchema = new Schema({
+        name: this.name,
+        attributes: this.attributes,
+        passphrase: this.passphrase,
+        version,
+        options: {
+          ...this.options,
+          // For older versions, be more lenient with decryption
+          autoDecrypt: true,
+          autoEncrypt: true
+        }
+      });
+      return compatibleSchema;
+    } catch (error) {
+      console.warn(`Failed to create compatible schema for version ${version}, using current schema:`, error.message);
+      return this.schema;
+    }
   }
   /**
    * Create partition references after insert
@@ -10148,7 +10332,7 @@ class Database extends EventEmitter {
     this.version = "1";
     this.s3dbVersion = (() => {
       try {
-        return true ? "4.1.6" : "latest";
+        return true ? "4.1.9" : "latest";
       } catch (e) {
         return "latest";
       }