strtok3 6.0.0 → 6.0.4

package/lib/AbstractTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.AbstractTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
 /**
  * Core tokenizer

package/lib/BufferTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.BufferTokenizer = void 0;
 const peek_readable_1 = require("peek-readable");
 class BufferTokenizer {
     /**
@@ -22,7 +23,7 @@ class BufferTokenizer {
     async readBuffer(buffer, options) {
         if (options && options.position) {
             if (options.position < this.position) {
-                throw new Error('`options.position` can be less than `tokenizer.position`');
+                throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
             }
             this.position = options.position;
         }

package/lib/FileTokenizer.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromFile = exports.FileTokenizer = void 0;
 const AbstractTokenizer_1 = require("./AbstractTokenizer");
 const peek_readable_1 = require("peek-readable");
 const fs = require("./FsPromise");
@@ -20,7 +21,7 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
         if (options) {
             if (options.position) {
                 if (options.position < this.position) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
                 }
                 this.position = options.position;
             }
@@ -57,7 +58,7 @@ class FileTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
         if (options) {
             if (options.position) {
                 if (options.position < this.position) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
                 }
                 position = options.position;
             }

package/lib/FsPromise.js

@@ -3,6 +3,7 @@
  * Module convert fs functions to promise based functions
  */
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.readFile = exports.writeFileSync = exports.writeFile = exports.read = exports.open = exports.close = exports.stat = exports.createReadStream = exports.pathExists = void 0;
 const fs = require("fs");
 exports.pathExists = fs.existsSync;
 exports.createReadStream = fs.createReadStream;

package/lib/ReadStreamTokenizer.js

@@ -1,10 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.ReadStreamTokenizer = void 0;
 const AbstractTokenizer_1 = require("./AbstractTokenizer");
 const peek_readable_1 = require("peek-readable");
-const _debug = require("debug");
-const debug = _debug('strtok3:ReadStreamTokenizer');
-const maxBufferSize = 1 * 1000 * 1000;
+// import * as _debug from 'debug';
+// const debug = _debug('strtok3:ReadStreamTokenizer');
+const maxBufferSize = 256000;
 class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
     constructor(stream, fileInfo) {
         super(fileInfo);
@@ -42,7 +43,7 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
                     return this.readBuffer(buffer, options);
                 }
                 else if (skipBytes < 0) {
-                    throw new Error('`options.position` can be less than `tokenizer.position`');
+                    throw new Error('`options.position` must be equal or greater than `tokenizer.position`');
                 }
             }
             if (options.offset) {
@@ -94,14 +95,22 @@ class ReadStreamTokenizer extends AbstractTokenizer_1.AbstractTokenizer {
                 }
             }
         }
-        bytesRead = await this.streamReader.peek(buffer, offset, length);
+        try {
+            bytesRead = await this.streamReader.peek(buffer, offset, length);
+        }
+        catch (err) {
+            if (options && options.mayBeLess && err instanceof peek_readable_1.EndOfStreamError) {
+                return 0;
+            }
+            throw err;
+        }
         if ((!options || !options.mayBeLess) && bytesRead < length) {
             throw new peek_readable_1.EndOfStreamError();
         }
         return bytesRead;
     }
     async ignore(length) {
-        debug(`ignore ${this.position}...${this.position + length - 1}`);
+        // debug(`ignore ${this.position}...${this.position + length - 1}`);
         const bufSize = Math.min(maxBufferSize, length);
         const buf = Buffer.alloc(bufSize);
         let totBytesRead = 0;

package/lib/core.js

@@ -1,9 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromBuffer = exports.fromStream = void 0;
 const ReadStreamTokenizer_1 = require("./ReadStreamTokenizer");
 const BufferTokenizer_1 = require("./BufferTokenizer");
 var peek_readable_1 = require("peek-readable");
-exports.EndOfStreamError = peek_readable_1.EndOfStreamError;
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return peek_readable_1.EndOfStreamError; } });
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property/

package/lib/index.js

@@ -1,12 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.fromStream = void 0;
 const fs = require("./FsPromise");
 const core = require("./core");
 var FileTokenizer_1 = require("./FileTokenizer");
-exports.fromFile = FileTokenizer_1.fromFile;
+Object.defineProperty(exports, "fromFile", { enumerable: true, get: function () { return FileTokenizer_1.fromFile; } });
 var core_1 = require("./core");
-exports.EndOfStreamError = core_1.EndOfStreamError;
-exports.fromBuffer = core_1.fromBuffer;
+Object.defineProperty(exports, "EndOfStreamError", { enumerable: true, get: function () { return core_1.EndOfStreamError; } });
+Object.defineProperty(exports, "fromBuffer", { enumerable: true, get: function () { return core_1.fromBuffer; } });
 /**
  * Construct ReadStreamTokenizer from given Stream.
  * Will set fileSize, if provided given Stream has set the .path property.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "strtok3",
-  "version": "6.0.0",
+  "version": "6.0.4",
   "description": "A promise based streaming tokenizer",
   "author": {
     "name": "Borewit",
@@ -38,32 +38,31 @@
     "url": "https://github.com/Borewit/strtok3/issues"
   },
   "devDependencies": {
-    "@types/chai": "^4.2.5",
-    "@types/mocha": "^5.2.7",
-    "@types/node": "^13.5.0",
-    "@typescript-eslint/eslint-plugin": "^2.17.0",
-    "@typescript-eslint/eslint-plugin-tslint": "^2.17.0",
-    "@typescript-eslint/parser": "^2.17.0",
+    "@types/chai": "^4.2.12",
+    "@types/mocha": "^8.0.0",
+    "@types/node": "^14.0.26",
+    "@typescript-eslint/eslint-plugin": "^2.31.0",
+    "@typescript-eslint/eslint-plugin-tslint": "^2.31.0",
+    "@typescript-eslint/parser": "^2.31.0",
     "chai": "^4.2.0",
-    "coveralls": "^3.0.7",
+    "coveralls": "^3.1.0",
     "del-cli": "^3.0.0",
     "eslint": "^6.8.0",
-    "eslint-plugin-import": "^2.20.0",
-    "eslint-plugin-prefer-arrow": "^1.1.7",
-    "mocha": "^7.0.0",
-    "nyc": "^15.0.0",
-    "remark-cli": "^7.0.1",
-    "remark-preset-lint-recommended": "^3.0.3",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-prefer-arrow": "^1.2.2",
+    "mocha": "^7.2.0",
+    "nyc": "^15.1.0",
+    "remark-cli": "^8.0.1",
+    "remark-preset-lint-recommended": "^4.0.1",
     "source-map-support": "^0.5.16",
     "token-types": "^2.0.0",
-    "ts-node": "^8.6.2",
-    "tslint": "^6.0.0",
-    "typescript": "^3.7.5"
+    "ts-node": "^8.10.2",
+    "tslint": "^6.1.2",
+    "typescript": "^3.9.7"
   },
   "dependencies": {
     "@tokenizer/token": "^0.1.1",
     "@types/debug": "^4.1.5",
-    "debug": "^4.1.1",
     "peek-readable": "^3.1.0"
   },
   "keywords": [