split-hash 0.3.0 → 0.3.2

package/README.md

@@ -40,6 +40,7 @@ for await (const hash of iter) {
 ```js
 import { SplitHashValidator } from 'split-hash/nodejs'
 import * as crypto from 'crypto'
+import { pipeline } from 'stream'
 
 const KiB = 1024
 
@@ -58,11 +59,13 @@ const createHash = () => {
 const hashList = [/* ... */]
 const validator = new SplitHashValidator(hashList, 512 * KiB, createHash)
 
-const stream = fs.createReadStream('filename.bin')
-stream
-  .pipe(validator)
-  .on('data', /* same as stream */)
-  .on('error', err => console.error('not matched'))
+const stream = pipeline(
+  fs.createReadStream('filename.bin')
+, validator
+, err => {
+    // ...
+  }
+)
 ```
 
 ## API
@@ -82,7 +85,7 @@ function splitHash<T>(
   stream: NodeJS.ReadableStream
 , blockSizeBytes: number
 , createHash: ProgressiveHashFactory<T>
-): AsyncIterable<T>
+): AsyncIterableIterator<T>
 ```
 
 It throws `StreamEncodingError` when the `stream` encoding is set.
@@ -127,5 +130,5 @@ async function* splitHash<T>(
   stream: ReadableStream
 , blockSize: number
 , createHash: ProgressiveHashFactory<T>
-): AsyncIterable<T>
+): AsyncIterableIterator<T>
 ```

package/lib/nodejs/split-hash.d.ts

@@ -1,7 +1,7 @@
 /// <reference types="node" resolution-mode="require"/>
 import { CustomError } from '@blackglory/errors';
 import { ProgressiveHashFactory } from './types.js';
-export declare function splitHash<T>(stream: NodeJS.ReadableStream, blockSizeBytes: number, createHash: ProgressiveHashFactory<T>): AsyncIterable<T>;
+export declare function splitHash<T>(stream: NodeJS.ReadableStream, blockSizeBytes: number, createHash: ProgressiveHashFactory<T>): AsyncIterableIterator<T>;
 export declare class StreamEncodingError extends CustomError {
     constructor();
 }

package/lib/whatwg/split-hash.d.ts

@@ -1,2 +1,2 @@
 import { ProgressiveHashFactory } from './types.js';
-export declare function splitHash<T>(stream: ReadableStream, blockSizeBytes: number, createHash: ProgressiveHashFactory<T>): AsyncIterable<T>;
+export declare function splitHash<T>(stream: ReadableStream, blockSizeBytes: number, createHash: ProgressiveHashFactory<T>): AsyncIterableIterator<T>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "split-hash",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Split the stream based on bytes and get digests from each part.",
   "keywords": [
     "split",
@@ -10,7 +10,8 @@
     "chunk"
   ],
   "files": [
-    "lib"
+    "lib",
+    "src"
   ],
   "type": "module",
   "exports": {
@@ -51,10 +52,10 @@
     "@typescript-eslint/parser": "^5.56.0",
     "cross-env": "^7.0.3",
     "eslint": "^8.36.0",
-    "extra-abort": "^0.3.3",
+    "extra-abort": "^0.3.4",
     "extra-promise": "^6.0.5",
     "husky": "^4.3.0",
-    "iterable-operator": "^4.0.3",
+    "iterable-operator": "^4.0.5",
     "npm-run-all": "^4.1.5",
     "return-style": "^3.0.0",
     "rimraf": "^3.0.2",

package/src/nodejs/index.ts

@@ -0,0 +1,3 @@
+export * from './types.js'
+export * from './split-hash.js'
+export * from './split-hash-validator.js'

package/src/nodejs/split-hash-validator.ts

@@ -0,0 +1,78 @@
+import { Transform, TransformCallback } from 'stream'
+import { CustomError } from '@blackglory/errors'
+import { ProgressiveHashFactory, IProgressiveHash } from './types.js'
+
+export class SplitHashValidator<T> extends Transform {
+  private hash: IProgressiveHash<T> = this.createHash()
+  private accu = 0
+  private digestIndex = 0
+
+  constructor(
+    private digests: T[]
+  , private blockSize: number
+  , private createHash: ProgressiveHashFactory<T>
+  , private equals: (a: T, b: T) => boolean = Object.is
+  ) {
+    super()
+  }
+
+  _transform(
+    chunk: Buffer
+  , encoding: BufferEncoding
+  , callback: TransformCallback
+  ): void {
+    // chunk is always Buffer, encoding is always 'buffer', so there is no need to check
+
+    if (this.accu + chunk.length < this.blockSize) {
+      this.hash.update(chunk)
+      this.accu += chunk.length
+    } else {
+      let offset = 0
+      while (true) {
+        const needed = this.blockSize - this.accu
+        const slice = chunk.slice(offset, offset + needed)
+        if (slice.length === needed) {
+          this.hash.update(slice)
+          const digest = this.hash.digest()
+          if (!this.equals(this.digests[this.digestIndex], digest)) {
+            return callback(new NotMatchedError())
+          }
+          this.digestIndex++
+          // prepare for the next round
+          this.hash = this.createHash()
+          this.accu = 0
+          offset += slice.length
+        } else {
+          // if the length does not match, the remaining data is not long enough, update the remaining data and exit the loop.
+          this.hash.update(slice)
+          this.accu += slice.length
+          break
+        }
+      }
+    }
+
+    callback(null, chunk)
+  }
+
+  _flush(callback: TransformCallback): void {
+    if (this.accu > 0) {
+      const digest = this.hash.digest()
+      if (!this.equals(this.digests[this.digestIndex], digest)) {
+        return callback(new NotMatchedError())
+      }
+      this.digestIndex++
+    }
+
+    if (this.digestIndex !== this.digests.length) {
+      return callback(new NotMatchedError())
+    }
+
+    callback()
+  }
+}
+
+export class NotMatchedError extends CustomError {
+  constructor() {
+    super('hashes do not match')
+  }
+}

package/src/nodejs/split-hash.ts

@@ -0,0 +1,46 @@
+import { CustomError } from '@blackglory/errors'
+import { ProgressiveHashFactory } from './types.js'
+
+export async function* splitHash<T>(
+  stream: NodeJS.ReadableStream
+, blockSizeBytes: number
+, createHash: ProgressiveHashFactory<T>
+): AsyncIterableIterator<T> {
+  let hash = createHash()
+  let accu = 0
+  for await (const chunk of stream) {
+    if (!Buffer.isBuffer(chunk)) throw new StreamEncodingError()
+    if (accu + chunk.length < blockSizeBytes) {
+      hash.update(chunk)
+      accu += chunk.length
+    } else {
+      let offset = 0
+      while (true) {
+        const needed = blockSizeBytes - accu
+        const slice = chunk.slice(offset, offset + needed)
+        if (slice.length === needed) {
+          hash.update(slice)
+          const digest = hash.digest()
+          yield digest
+          // prepare for the next round
+          hash = createHash()
+          accu = 0
+          offset += slice.length
+        } else {
+          // if the length does not match, the remaining data is not long enough, update the remaining data and exit the loop.
+          hash.update(slice)
+          accu += slice.length
+          break
+        }
+      }
+    }
+  }
+  // digest remaining data if it exists
+  if (accu > 0) yield hash.digest()
+}
+
+export class StreamEncodingError extends CustomError {
+  constructor() {
+    super('stream encoding must not be set')
+  }
+}

package/src/nodejs/types.ts

@@ -0,0 +1,6 @@
+export type ProgressiveHashFactory<T> = () => IProgressiveHash<T>
+
+export interface IProgressiveHash<T> {
+  update(buffer: Buffer): void
+  digest(): T
+}

package/src/whatwg/index.ts

@@ -0,0 +1,2 @@
+export * from './types.js'
+export * from './split-hash.js'

package/src/whatwg/split-hash.ts

@@ -0,0 +1,40 @@
+import { toAsyncIterableIterator } from 'extra-stream'
+import { ProgressiveHashFactory } from './types.js'
+
+export async function* splitHash<T>(
+  stream: ReadableStream
+, blockSizeBytes: number
+, createHash: ProgressiveHashFactory<T>
+): AsyncIterableIterator<T> {
+  let hash = createHash()
+  let accu = 0
+  for await (const chunk of toAsyncIterableIterator(stream)) {
+    if (accu + chunk.length < blockSizeBytes) {
+      hash.update(chunk)
+      accu += chunk.length
+    } else {
+      let offset = 0
+      while (true) {
+        const needed = blockSizeBytes - accu
+        const slice = chunk.slice(offset, offset + needed)
+        if (slice.length === needed) {
+          hash.update(slice)
+          const digest = await hash.digest()
+          yield digest
+          // prepare for the next round
+          hash = createHash()
+          accu = 0
+          offset += slice.length
+        } else {
+          // if the length does not match,
+          // the remaining data is not long enough, update the remaining data and exit the loop.
+          hash.update(slice)
+          accu += slice.length
+          break
+        }
+      }
+    }
+  }
+  // digest remaining data if it exists
+  if (accu > 0) yield await hash.digest()
+}

package/src/whatwg/types.ts

@@ -0,0 +1,6 @@
+export type ProgressiveHashFactory<T> = () => IProgressiveHash<T>
+
+export interface IProgressiveHash<T> {
+  update(buffer: Uint8Array): void
+  digest(): Promise<T>
+}