web-dc-api 0.0.57 → 0.0.59

package/dist/cjs/implements/file/seekableFileStream.js

@@ -0,0 +1,320 @@
+'use strict';
+
+var toBuffer = require('it-to-buffer');
+var dccrypt = require('../../util/dccrypt.js');
+
+/**
+ * 可定位文件流类
+ * 支持随机访问和流式读取加密和未加密的IPFS文件
+ */
+class SeekableFileStream {
+  constructor(options) {
+    Object.defineProperty(this, "position", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: 0
+    });
+    Object.defineProperty(this, "fs", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "cid", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "decryptKey", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "fileInfo", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    Object.defineProperty(this, "encryptChunkSize", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    this.fileInfo = options.fileInfo;
+    this.fs = options.fs;
+    this.cid = options.cid;
+    this.decryptKey = options.decryptKey;
+    this.encryptChunkSize = options.encryptChunkSize;
+  }
+  /**
+   * 获取文件大小
+   */
+  getSize() {
+    return this.fileInfo.size;
+  }
+  /**
+   * 定位到文件指定位置
+   * @param position 目标位置
+   */
+  seek(position) {
+    if (position < 0) {
+      position = 0;
+    } else if (position > this.fileInfo.size) {
+      position = this.fileInfo.size;
+    }
+    this.position = position;
+  }
+  /**
+   * 获取当前位置
+   */
+  getPosition() {
+    return this.position;
+  }
+  /**
+   * 读取指定长度的数据
+   * @param length 要读取的字节数
+   */
+  async read(length) {
+    // 使用局部变量存储当前位置，避免并发问题
+    const startPosition = this.position;
+    if (startPosition >= this.fileInfo.size) {
+      // 已到达文件末尾
+      return new Uint8Array(0);
+    }
+    // 计算实际读取长度（不超过文件末尾）
+    const remainingBytes = this.fileInfo.size - startPosition;
+    const actualLength = Math.min(length, remainingBytes);
+    let result;
+    console.log("**************read actualLength:", actualLength);
+    // 根据是否加密选择不同读取策略
+    if (this.decryptKey) {
+      result = await this.readEncrypted(startPosition, actualLength);
+    } else {
+      result = await this.readPlain(startPosition, actualLength);
+    }
+    console.log("**************read result length:", result.length);
+    // 仅在成功读取后更新位置
+    if (this.position == startPosition) {
+      //表示没有seek过
+      this.position = startPosition + result.length;
+      console.log("**************read this.position:", this.position);
+    }
+    return result;
+  }
+  /**
+   * 读取未加密的数据
+   * @param startPosition 开始位置
+   * @param length 读取长度
+   */
+  async readPlain(startPosition, length) {
+    try {
+      // 计算实际文件中的偏移量（考虑文件头）
+      const actualOffset = startPosition + this.fileInfo.headerSize;
+      // 直接读取指定长度
+      const data = await toBuffer(this.fs.cat(this.cid, {
+        offset: actualOffset,
+        length: length
+      }));
+      return data;
+    } catch (err) {
+      console.error("Error reading plain data:", err);
+      return new Uint8Array(0);
+    }
+  }
+  /**
+  * 读取并解密加密数据，无音视频帧保护
+  * @param startPosition 开始位置
+  * @param length 读取长度
+  */
+  async readEncrypted(startPosition, length) {
+    // 计算块相关参数
+    const blockSize = 3 << 20; // 3MB 原始块大小
+    const blockIndex = Math.floor(startPosition / blockSize);
+    const offsetInBlock = startPosition % blockSize;
+    // 计算需要读取的块数
+    const endPosition = startPosition + length;
+    const endBlockIndex = Math.floor(endPosition / blockSize);
+    const neededBlocks = endBlockIndex - blockIndex + 1;
+    // 分配适当大小的内存
+    const totalSize = Math.min(neededBlocks * blockSize, offsetInBlock + length);
+    let decryptedData = new Uint8Array(totalSize);
+    // 计算文件中的起始位置
+    let readStartPosition = this.fileInfo.headerSize + blockIndex * this.encryptChunkSize;
+    const readTotalSize = neededBlocks * this.encryptChunkSize;
+    try {
+      // 读取加密块
+      const encryptedBlocks = await toBuffer(this.fs.cat(this.cid, {
+        offset: readStartPosition,
+        length: readTotalSize
+      }));
+      // 确保获取到了数据
+      if (encryptedBlocks.length === 0) {
+        console.warn("无法读取加密数据");
+        return new Uint8Array(0);
+      }
+      // 解密所有块并合并
+      let currentOffset = 0;
+      for (let i = 0; i < neededBlocks; i++) {
+        const start = i * this.encryptChunkSize;
+        if (start >= encryptedBlocks.byteLength) {
+          console.warn("*******加密数据块超出范围");
+          break;
+        }
+        const end = Math.min((i + 1) * this.encryptChunkSize, encryptedBlocks.length);
+        const encryptedBlock = encryptedBlocks.slice(start, end);
+        if (encryptedBlock.byteLength === 0) {
+          console.warn("*******加密数据块为空");
+          continue;
+        }
+        // 解密块
+        const decryptedBlock = await dccrypt.decryptContent(encryptedBlock, this.decryptKey);
+        // 合并到结果中
+        const bytesToCopy = Math.min(decryptedBlock.length, decryptedData.length - currentOffset);
+        decryptedData.set(decryptedBlock.subarray(0, bytesToCopy), currentOffset);
+        currentOffset += bytesToCopy;
+      }
+      // 从解密数据中提取请求的范围
+      let result = new Uint8Array(length);
+      result.set(decryptedData.subarray(offsetInBlock, offsetInBlock + length), 0);
+      return result;
+    } catch (err) {
+      console.error("Error reading encrypted data:", err);
+      return new Uint8Array(0);
+    }
+  }
+  /**
+   * 创建标准的可读流
+   * 使用独立缓冲区，不影响随机访问读取
+   */
+  createReadableStream(options) {
+    // 为流创建独立的状态变量，不使用实例变量
+    let streamPosition = options?.start !== undefined ? options.start : this.position;
+    let streamBuffer = new Uint8Array(0);
+    let streamBufferPosition = streamPosition;
+    // 确保位置有效
+    if (streamPosition < 0) {
+      streamPosition = 0;
+    } else if (streamPosition > this.fileInfo.size) {
+      streamPosition = this.fileInfo.size;
+    }
+    const end = options?.end ?? this.fileInfo.size;
+    const chunkSize = 64 * 1024; // 64KB 客户端块
+    const bufferSize = 256 * 1024; // 256KB 预读缓冲区
+    /**
+     * 流式读取专用的预读函数
+     * 使用独立缓冲区
+     */
+    const prefetchStreamData = async size => {
+      const maxSize = Math.min(size, this.fileInfo.size - streamPosition);
+      if (maxSize <= 0) return;
+      try {
+        // 读取数据到独立的流缓冲区
+        let prefetchedData;
+        if (this.decryptKey) {
+          prefetchedData = await this.readEncrypted(streamPosition, maxSize);
+        } else {
+          prefetchedData = await this.readPlain(streamPosition, maxSize);
+        }
+        // 更新流缓冲区和位置
+        streamBuffer = prefetchedData;
+        streamBufferPosition = streamPosition;
+      } catch (err) {
+        console.error("Error prefetching stream data:", err);
+        streamBuffer = new Uint8Array(0);
+      }
+    };
+    /**
+     * 流式读取专用的读取函数
+     * 使用独立缓冲区和位置
+     */
+    const streamRead = async length => {
+      if (streamPosition >= this.fileInfo.size) {
+        return new Uint8Array(0);
+      }
+      const remainingBytes = this.fileInfo.size - streamPosition;
+      const actualLength = Math.min(length, remainingBytes);
+      let result;
+      // 检查流缓冲区中是否有足够数据
+      if (streamPosition >= streamBufferPosition && streamPosition + actualLength <= streamBufferPosition + streamBuffer.length) {
+        // 从流缓冲区获取数据
+        const bufferOffset = streamPosition - streamBufferPosition;
+        result = streamBuffer.slice(bufferOffset, bufferOffset + actualLength);
+      } else {
+        // 缓冲区中没有所需数据，直接读取
+        if (this.decryptKey) {
+          result = await this.readEncrypted(streamPosition, actualLength);
+        } else {
+          result = await this.readPlain(streamPosition, actualLength);
+        }
+      }
+      // 更新流位置
+      streamPosition += result.length;
+      return result;
+    };
+    return new ReadableStream({
+      start: async controller => {
+        // 初始预读
+        try {
+          await prefetchStreamData(bufferSize);
+        } catch (err) {
+          console.warn("Initial stream prefetch failed:", err);
+        }
+      },
+      pull: async controller => {
+        try {
+          if (streamPosition >= end) {
+            controller.close();
+            return;
+          }
+          const bytesToRead = Math.min(chunkSize, end - streamPosition);
+          // 检查流缓冲区中是否有足够数据
+          if (streamPosition >= streamBufferPosition && streamPosition + bytesToRead <= streamBufferPosition + streamBuffer.length) {
+            // 从流缓冲区获取数据
+            const bufferOffset = streamPosition - streamBufferPosition;
+            const chunk = streamBuffer.slice(bufferOffset, bufferOffset + bytesToRead);
+            streamPosition += chunk.length;
+            controller.enqueue(chunk);
+            // 如果接近缓冲区末尾，预读取更多数据
+            const prefetchThreshold = chunkSize * 2;
+            if (streamPosition + prefetchThreshold > streamBufferPosition + streamBuffer.length && streamPosition < end) {
+              prefetchStreamData(bufferSize).catch(err => console.warn("Stream prefetch failed:", err));
+            }
+          } else {
+            // 缓冲区中没有所需数据，直接读取并同时更新缓冲区
+            await prefetchStreamData(Math.max(bufferSize, bytesToRead));
+            // 从新缓冲区获取数据
+            const bufferOffset = streamPosition - streamBufferPosition;
+            if (bufferOffset >= 0 && bufferOffset < streamBuffer.length) {
+              const chunk = streamBuffer.slice(bufferOffset, Math.min(bufferOffset + bytesToRead, streamBuffer.length));
+              streamPosition += chunk.length;
+              controller.enqueue(chunk);
+            } else {
+              // 缓冲区没有预期的数据，直接读取
+              const chunk = await streamRead(bytesToRead);
+              if (chunk.length > 0) {
+                controller.enqueue(chunk);
+              } else {
+                controller.close();
+              }
+            }
+          }
+        } catch (err) {
+          console.error("Stream read error:", err);
+          controller.error(err);
+        }
+      },
+      cancel: () => {
+        // 清理资源
+        streamBuffer = new Uint8Array(0);
+        console.log("ReadableStream cancelled");
+      }
+    });
+  }
+}
+
+exports.SeekableFileStream = SeekableFileStream;

package/dist/cjs/implements/file/streamwriter.js

@@ -0,0 +1,305 @@
+'use strict';
+
+var index$1 = require('../../node_modules/it-pipe/dist/src/index.js');
+var index = require('../../node_modules/it-pushable/dist/src/index.js');
+
+const MaxChunkSize = 4 * 1024 - 5;
+class StreamWriter {
+  constructor(sink) {
+    let options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+    Object.defineProperty(this, "sink", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: sink
+    });
+    Object.defineProperty(this, "options", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: options
+    });
+    Object.defineProperty(this, "p", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: void 0
+    });
+    //private p = pushable({ objectMode: false }) 
+    Object.defineProperty(this, "bytesWritten", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: 0
+    });
+    Object.defineProperty(this, "abortController", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: new AbortController()
+    });
+    // 背压控制相关属性  
+    Object.defineProperty(this, "backpressureHistory", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: []
+    });
+    Object.defineProperty(this, "isBackpressure", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: false
+    });
+    Object.defineProperty(this, "writeQueue", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: []
+    });
+    Object.defineProperty(this, "isProcessingQueue", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: false
+    });
+    // 事件系统  
+    Object.defineProperty(this, "listeners", {
+      enumerable: true,
+      configurable: true,
+      writable: true,
+      value: new Map()
+    });
+    if (options) {
+      this.options = {
+        chunkSize: options.chunkSize ?? MaxChunkSize,
+        bufferSize: options.bufferSize ?? 5 * 1024 * 1024,
+        retries: options.retries ?? 3
+      };
+    } else {
+      this.options = {
+        chunkSize: MaxChunkSize,
+        bufferSize: 5 * 1024 * 1024,
+        retries: 3
+      };
+    }
+    if (this.options.chunkSize && this.options.chunkSize > MaxChunkSize) {
+      this.options.chunkSize = MaxChunkSize;
+    }
+    const basePushable = index.pushable({
+      objectMode: false
+    });
+    // 保留原始方法引用  
+    basePushable._originalPush = basePushable.push.bind(basePushable);
+    basePushable._originalNext = basePushable.next.bind(basePushable);
+    basePushable._queueSize = 0;
+    // 重写 next 方法  
+    Object.defineProperty(basePushable, 'next', {
+      value: async () => {
+        const result = await basePushable._originalNext();
+        if (!result.done && result.value) {
+          basePushable._queueSize -= result.value.byteLength;
+        }
+        return result;
+      },
+      writable: false,
+      configurable: false
+    });
+    // 安全重写 push 方法  
+    Object.defineProperty(basePushable, 'push', {
+      value: chunk => {
+        basePushable._queueSize += chunk.byteLength;
+        return basePushable._originalPush(chunk);
+      },
+      writable: false,
+      configurable: false
+    });
+    this.p = basePushable;
+    this.startPipeline();
+  }
+  get queueSize() {
+    return this.p._queueSize;
+  }
+  // 在 next() 操作时更新队列大小  
+  async safeNext() {
+    const result = await this.p.next();
+    if (!result.done && result.value) {
+      this.p._queueSize -= result.value.byteLength;
+    }
+    return result;
+  }
+  handleError(err) {
+    this.dispatchEvent(new CustomEvent('error', {
+      detail: err
+    }));
+    this.abort(err.message);
+  }
+  startPipeline() {
+    index$1.pipe(this.p, this.createTransform(), this.sink).catch(err => this.handleError(err)); // 正确绑定this  
+  }
+  createTransform() {
+    return async function* (source) {
+      for await (const chunk of source) {
+        yield chunk;
+      }
+    };
+  }
+  async write(data) {
+    if (this.abortController.signal.aborted) return;
+    return new Promise((resolve, reject) => {
+      const task = async () => {
+        try {
+          const buffer = await this.convertToBuffer(data);
+          await this.writeChunks(buffer);
+          resolve();
+        } catch (err) {
+          reject(err);
+        }
+      };
+      this.writeQueue.push(task);
+      this.processQueue();
+    });
+  }
+  async convertToBuffer(data) {
+    if (data instanceof Blob) return data.arrayBuffer();
+    if (typeof data === 'string') return new TextEncoder().encode(data).buffer;
+    return data;
+  }
+  async writeChunks(buffer) {
+    for (let offset = 0; offset < buffer.byteLength; offset += this.options.chunkSize) {
+      const end = Math.min(offset + this.options.chunkSize, buffer.byteLength);
+      const chunk = new Uint8Array(end - offset);
+      chunk.set(new Uint8Array(buffer.slice(offset, end)));
+      await this.retryableWrite(chunk);
+      this.updateProgress(chunk.byteLength);
+    }
+  }
+  async retryableWrite(chunk) {
+    let attempt = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;
+    try {
+      await this.monitorBackpressure();
+      await new Promise((resolve, reject) => {
+        try {
+          this.p.push(chunk);
+        } catch (err) {
+          reject(err);
+        }
+        resolve();
+      });
+    } catch (err) {
+      if (attempt < this.options.retries) {
+        const delay = this.calculateRetryDelay(attempt);
+        await new Promise(r => setTimeout(r, delay));
+        return this.retryableWrite(chunk, attempt + 1);
+      }
+      throw err;
+    }
+  }
+  async monitorBackpressure() {
+    const checkInterval = 50;
+    const historySize = 10;
+    while (true) {
+      const currentSize = this.queueSize; // 空值合并运算符  
+      if (currentSize > 0) {
+        this.backpressureHistory.push(currentSize);
+      }
+      if (this.backpressureHistory.length > historySize) {
+        this.backpressureHistory.shift();
+      }
+      const avg = this.backpressureHistory.reduce((a, b) => a + b, 0) / historySize;
+      const dynamicThreshold = Math.min(this.options.bufferSize * 0.8, avg * 1.5);
+      if (currentSize < dynamicThreshold || currentSize === 0) {
+        if (this.isBackpressure) {
+          this.isBackpressure = false;
+          this.dispatchEvent(new CustomEvent('backpressure', {
+            detail: {
+              currentSize,
+              averageSize: avg,
+              threshold: dynamicThreshold,
+              waitingTime: 0
+            }
+          }));
+        }
+        return;
+      }
+      if (!this.isBackpressure) {
+        this.isBackpressure = true;
+        this.dispatchEvent(new CustomEvent('backpressure', {
+          detail: {
+            currentSize,
+            averageSize: avg,
+            threshold: dynamicThreshold,
+            waitingTime: 0
+          }
+        }));
+      }
+      const waitTime = Math.min(10, checkInterval * Math.pow(2, currentSize / dynamicThreshold));
+      await new Promise(r => setTimeout(r, waitTime));
+      if (this.abortController.signal.aborted) break;
+    }
+  }
+  calculateRetryDelay(attempt) {
+    const baseDelay = 10;
+    const maxDelay = 100;
+    return Math.min(baseDelay * Math.pow(2, attempt) + Math.random() * 100, maxDelay);
+  }
+  async processQueue() {
+    if (this.isProcessingQueue || this.abortController.signal.aborted) return;
+    this.isProcessingQueue = true;
+    while (this.writeQueue.length > 0) {
+      if (this.abortController.signal.aborted) break;
+      await this.monitorBackpressure();
+      const task = this.writeQueue.shift();
+      try {
+        await task();
+      } catch (err) {
+        continue; // 继续处理下一个任务
+      }
+    }
+    this.isProcessingQueue = false;
+  }
+  updateProgress(bytes) {
+    this.bytesWritten += bytes;
+    this.dispatchEvent(new CustomEvent('progress', {
+      detail: {
+        loaded: this.bytesWritten
+      }
+    }));
+  }
+  async end() {
+    this.p.end();
+    await this.sink.return?.();
+    this.cleanup();
+  }
+  abort() {
+    let reason = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : 'User aborted';
+    this.abortController.abort(reason);
+    this.cleanup();
+    this.dispatchEvent(new CustomEvent('abort', {
+      detail: reason
+    }));
+  }
+  cleanup() {
+    this.p.end();
+    this.abortController.abort();
+    this.writeQueue = [];
+  }
+  addEventListener(type, callback) {
+    const handlers = this.listeners.get(type) || [];
+    handlers.push(callback);
+    this.listeners.set(type, handlers);
+  }
+  // 修复后的代码片段  
+  dispatchEvent(event) {
+    const handlers = this.listeners.get(event.type) || [];
+    handlers.forEach(handler => handler(event));
+  }
+  // 明确指定事件类型  
+  dispatchBackpressureEvent(detail) {
+    this.dispatchEvent(new CustomEvent('backpressure', {
+      detail
+    }));
+  }
+}
+
+exports.StreamWriter = StreamWriter;