parse5 3.0.1 → 4.0.0

package/lib/index.js

@@ -38,7 +38,29 @@ exports.treeAdapters = {
 
 
 // Streaming
-exports.ParserStream = require('./parser/parser_stream');
-exports.PlainTextConversionStream = require('./parser/plain_text_conversion_stream');
-exports.SerializerStream = require('./serializer/serializer_stream');
-exports.SAXParser = require('./sax');
+// NOTE: streaming API is lazy loadable to enable bundling for platforms
+// that are different from Node.js.
+// See https://github.com/inikulin/parse5/issues/235.
+var streamingAPI = {
+    ParserStream: './parser/parser_stream',
+    PlainTextConversionStream: './parser/plain_text_conversion_stream',
+    SerializerStream: './serializer/serializer_stream',
+    SAXParser: './sax'
+};
+
+Object.keys(streamingAPI).forEach(function (cls) {
+    Object.defineProperty(exports, cls, {
+        get: function () {
+            try {
+                return require(streamingAPI[cls]);
+            }
+
+            catch (e) {
+                throw new Error(
+                    cls + ' is supported only for Node.js.' +
+                    'See https://github.com/inikulin/parse5/issues/235 for the details.'
+                );
+            }
+        }
+    });
+});

package/lib/parser/index.js

@@ -3,11 +3,11 @@
 var Tokenizer = require('../tokenizer'),
     OpenElementStack = require('./open_element_stack'),
     FormattingElementList = require('./formatting_element_list'),
-    locationInfoMixin = require('../location_info/parser_mixin'),
+    LocationInfoParserMixin = require('../extensions/location_info/parser_mixin'),
     defaultTreeAdapter = require('../tree_adapters/default'),
+    mergeOptions = require('../utils/merge_options'),
     doctype = require('../common/doctype'),
     foreignContent = require('../common/foreign_content'),
-    mergeOptions = require('../common/merge_options'),
     UNICODE = require('../common/unicode'),
     HTML = require('../common/html');
 
@@ -310,7 +310,7 @@ var Parser = module.exports = function (options) {
     this.pendingScript = null;
 
     if (this.options.locationInfo)
-        locationInfoMixin.assign(this);
+        new LocationInfoParserMixin(this);
 };
 
 // API
@@ -319,7 +319,7 @@ Parser.prototype.parse = function (html) {
 
     this._bootstrap(document, null);
     this.tokenizer.write(html, true);
-    this._runParsingLoop(null, null);
+    this._runParsingLoop(null);
 
     return document;
 };
@@ -345,7 +345,7 @@ Parser.prototype.parseFragment = function (html, fragmentContext) {
     this._resetInsertionMode();
     this._findFormInFragmentContext();
     this.tokenizer.write(html, true);
-    this._runParsingLoop(null, null);
+    this._runParsingLoop(null);
 
     var rootElement = this.treeAdapter.getFirstChild(documentMock),
         fragment = this.treeAdapter.createDocumentFragment();
@@ -386,7 +386,7 @@ Parser.prototype._bootstrap = function (document, fragmentContext) {
 };
 
 //Parsing loop
-Parser.prototype._runParsingLoop = function (writeCallback, scriptHandler) {
+Parser.prototype._runParsingLoop  = function (scriptHandler) {
     while (!this.stopped) {
         this._setupTokenizerCDATAMode();
 
@@ -411,6 +411,10 @@ Parser.prototype._runParsingLoop = function (writeCallback, scriptHandler) {
         if (scriptHandler && this.pendingScript)
             break;
     }
+};
+
+Parser.prototype.runParsingLoopForCurrentChunk = function (writeCallback, scriptHandler) {
+    this._runParsingLoop(scriptHandler);
 
     if (scriptHandler && this.pendingScript) {
         var script = this.pendingScript;

package/lib/parser/parser_stream.js

@@ -35,12 +35,12 @@ ParserStream.prototype._write = function (chunk, encoding, callback) {
 
 ParserStream.prototype.end = function (chunk, encoding, callback) {
     this.lastChunkWritten = true;
-    WritableStream.prototype.end.call(this, chunk, encoding, callback);
+    WritableStream.prototype.end.call(this, chunk || '', encoding, callback);
 };
 
 //Scriptable parser implementation
 ParserStream.prototype._runParsingLoop = function () {
-    this.parser._runParsingLoop(this.writeCallback, this._scriptHandler);
+    this.parser.runParsingLoopForCurrentChunk(this.writeCallback, this._scriptHandler);
 };
 
 ParserStream.prototype._resume = function () {

package/lib/sax/dev_null_stream.js

@@ -1,14 +1,14 @@
-'use strict';
-
-var WritableStream = require('stream').Writable,
-    util = require('util');
-
-var DevNullStream = module.exports = function () {
-    WritableStream.call(this);
-};
-
-util.inherits(DevNullStream, WritableStream);
-
-DevNullStream.prototype._write = function (chunk, encoding, cb) {
-    cb();
-};
+'use strict';
+
+var WritableStream = require('stream').Writable,
+    util = require('util');
+
+var DevNullStream = module.exports = function () {
+    WritableStream.call(this);
+};
+
+util.inherits(DevNullStream, WritableStream);
+
+DevNullStream.prototype._write = function (chunk, encoding, cb) {
+    cb();
+};

package/lib/sax/index.js

@@ -4,8 +4,9 @@ var TransformStream = require('stream').Transform,
     DevNullStream = require('./dev_null_stream'),
     inherits = require('util').inherits,
     Tokenizer = require('../tokenizer'),
+    LocationInfoTokenizerMixin = require('../extensions/location_info/tokenizer_mixin'),
     ParserFeedbackSimulator = require('./parser_feedback_simulator'),
-    mergeOptions = require('../common/merge_options');
+    mergeOptions = require('../utils/merge_options');
 
 var DEFAULT_OPTIONS = {
     locationInfo: false
@@ -17,6 +18,10 @@ var SAXParser = module.exports = function (options) {
     this.options = mergeOptions(DEFAULT_OPTIONS, options);
 
     this.tokenizer = new Tokenizer(options);
+
+    if (this.options.locationInfo)
+        new LocationInfoTokenizerMixin(this.tokenizer);
+
     this.parserFeedbackSimulator = new ParserFeedbackSimulator(this.tokenizer);
 
     this.pendingText = null;

package/lib/serializer/index.js

@@ -1,8 +1,8 @@
 'use strict';
 
 var defaultTreeAdapter = require('../tree_adapters/default'),
+    mergeOptions = require('../utils/merge_options'),
     doctype = require('../common/doctype'),
-    mergeOptions = require('../common/merge_options'),
     HTML = require('../common/html');
 
 //Aliases

package/lib/tokenizer/index.js

@@ -1,7 +1,6 @@
 'use strict';
 
 var Preprocessor = require('./preprocessor'),
-    locationInfoMixin = require('../location_info/tokenizer_mixin'),
     UNICODE = require('../common/unicode'),
     neTree = require('./named_entity_data');
 
@@ -173,7 +172,7 @@ function findNamedEntityTreeBranch(nodeIx, cp) {
 
 
 //Tokenizer
-var Tokenizer = module.exports = function (options) {
+var Tokenizer = module.exports = function () {
     this.preprocessor = new Preprocessor();
 
     this.tokenQueue = [];
@@ -193,9 +192,6 @@ var Tokenizer = module.exports = function (options) {
     this.currentCharacterToken = null;
     this.currentToken = null;
     this.currentAttr = null;
-
-    if (options && options.locationInfo)
-        locationInfoMixin.assign(this);
 };
 
 //Token types
@@ -210,7 +206,7 @@ Tokenizer.EOF_TOKEN = 'EOF_TOKEN';
 Tokenizer.HIBERNATION_TOKEN = 'HIBERNATION_TOKEN';
 
 //Tokenizer initial states for different modes
-Tokenizer.MODE = Tokenizer.prototype.MODE = {
+Tokenizer.MODE = {
     DATA: DATA_STATE,
     RCDATA: RCDATA_STATE,
     RAWTEXT: RAWTEXT_STATE,

package/lib/tokenizer/preprocessor.js

@@ -32,7 +32,6 @@ var Preprocessor = module.exports = function () {
     this.pos = -1;
     this.lastGapPos = -1;
     this.lastCharPos = -1;
-    this.droppedBufferSize = 0;
 
     this.gapStack = [];
 
@@ -43,16 +42,9 @@ var Preprocessor = module.exports = function () {
     this.bufferWaterline = DEFAULT_BUFFER_WATERLINE;
 };
 
-Object.defineProperty(Preprocessor.prototype, 'sourcePos', {
-    get: function () {
-        return this.droppedBufferSize + this.pos;
-    }
-});
-
 Preprocessor.prototype.dropParsedChunk = function () {
     if (this.pos > this.bufferWaterline) {
         this.lastCharPos -= this.pos;
-        this.droppedBufferSize += this.pos;
         this.html = this.html.substring(this.pos);
         this.pos = 0;
         this.lastGapPos = -1;

package/lib/utils/mixin.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var Mixin = module.exports = function (host) {
+    var originalMethods = {},
+        overriddenMethods = this._getOverriddenMethods(this, originalMethods);
+
+    Object.keys(overriddenMethods).forEach(function (key) {
+        if (typeof overriddenMethods[key] === 'function') {
+            originalMethods[key] = host[key];
+            host[key] = overriddenMethods[key];
+        }
+    });
+};
+
+Mixin.prototype._getOverriddenMethods = function () {
+    throw new Error('Not implemented');
+};
+

package/package.json

@@ -1,63 +1,60 @@
-{
-  "name": "parse5",
-  "description": "HTML parsing/serialization toolset for Node.js. WHATWG HTML Living Standard (aka HTML5)-compliant.",
-  "version": "3.0.1",
-  "author": "Ivan Nikulin <ifaaan@gmail.com> (https://github.com/inikulin)",
-  "contributors": "https://github.com/inikulin/parse5/graphs/contributors",
-  "homepage": "https://github.com/inikulin/parse5",
-  "devDependencies": {
-    "del": "^2.0.2",
-    "gulp": "^3.9.0",
-    "gulp-benchmark": "^1.1.1",
-    "gulp-download": "0.0.1",
-    "gulp-eslint": "^3.0.1",
-    "gulp-install": "^0.6.0",
-    "gulp-mocha": "^2.1.3",
-    "gulp-rename": "^1.2.2",
-    "gulp-typedoc": "^2.0.0",
-    "gulp-typescript": "^3.1.2",
-    "publish-please": "^2.2.0",
-    "through2": "^2.0.0",
-    "typedoc": "^0.5.1",
-    "typescript": "^2.0.6"
-  },
-  "keywords": [
-    "html",
-    "parser",
-    "html5",
-    "WHATWG",
-    "specification",
-    "fast",
-    "html parser",
-    "html5 parser",
-    "htmlparser",
-    "parse5",
-    "serializer",
-    "html serializer",
-    "htmlserializer",
-    "sax",
-    "simple api",
-    "parse",
-    "tokenize",
-    "serialize",
-    "tokenizer"
-  ],
-  "license": "MIT",
-  "main": "./lib/index.js",
-  "types": "./lib/index.d.ts",
-  "repository": {
-    "type": "git",
-    "url": "git://github.com/inikulin/parse5.git"
-  },
-  "scripts": {
-    "test": "gulp test",
-    "publish-please": "publish-please",
-    "prepublish": "publish-please guard"
-  },
-  "files": [
-    "lib"
-  ],
-  "dependencies": {
-    "@types/node": "^6.0.46"
-  }
-}
+{
+  "name": "parse5",
+  "description": "HTML parsing/serialization toolset for Node.js. WHATWG HTML Living Standard (aka HTML5)-compliant.",
+  "version": "4.0.0",
+  "author": "Ivan Nikulin <ifaaan@gmail.com> (https://github.com/inikulin)",
+  "contributors": "https://github.com/inikulin/parse5/graphs/contributors",
+  "homepage": "https://github.com/inikulin/parse5",
+  "devDependencies": {
+    "@types/node": "*",
+    "del": "^2.0.2",
+    "gulp": "^3.9.0",
+    "gulp-benchmark": "^1.1.1",
+    "gulp-download": "0.0.1",
+    "gulp-eslint": "^3.0.1",
+    "gulp-install": "^0.6.0",
+    "gulp-mocha": "^2.1.3",
+    "gulp-rename": "^1.2.2",
+    "gulp-typedoc": "^2.0.0",
+    "gulp-typescript": "^3.1.2",
+    "publish-please": "^2.2.0",
+    "through2": "^2.0.0",
+    "typedoc": "^0.5.1",
+    "typescript": "^2.0.6"
+  },
+  "keywords": [
+    "html",
+    "parser",
+    "html5",
+    "WHATWG",
+    "specification",
+    "fast",
+    "html parser",
+    "html5 parser",
+    "htmlparser",
+    "parse5",
+    "serializer",
+    "html serializer",
+    "htmlserializer",
+    "sax",
+    "simple api",
+    "parse",
+    "tokenize",
+    "serialize",
+    "tokenizer"
+  ],
+  "license": "MIT",
+  "main": "./lib/index.js",
+  "repository": {
+    "type": "git",
+    "url": "git://github.com/inikulin/parse5.git"
+  },
+  "scripts": {
+    "test": "gulp test",
+    "publish-please": "publish-please",
+    "prepublish": "publish-please guard"
+  },
+  "files": [
+    "lib"
+  ]
+}

package/lib/location_info/parser_mixin.js

@@ -1,210 +0,0 @@
-'use strict';
-
-var OpenElementStack = require('../parser/open_element_stack'),
-    Tokenizer = require('../tokenizer'),
-    HTML = require('../common/html');
-
-
-//Aliases
-var $ = HTML.TAG_NAMES;
-
-
-function setEndLocation(element, closingToken, treeAdapter) {
-    var loc = element.__location;
-
-    if (!loc)
-        return;
-
-    if (!loc.startTag) {
-        loc.startTag = {
-            line: loc.line,
-            col: loc.col,
-            startOffset: loc.startOffset,
-            endOffset: loc.endOffset
-        };
-        if (loc.attrs)
-            loc.startTag.attrs = loc.attrs;
-    }
-
-    if (closingToken.location) {
-        var ctLocation = closingToken.location,
-            tn = treeAdapter.getTagName(element),
-        // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing tag and
-        // for cases like <td> <p> </td> - 'p' closes without a closing tag
-            isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN &&
-                              tn === closingToken.tagName;
-
-        if (isClosingEndTag) {
-            loc.endTag = {
-                line: ctLocation.line,
-                col: ctLocation.col,
-                startOffset: ctLocation.startOffset,
-                endOffset: ctLocation.endOffset
-            };
-        }
-
-        if (isClosingEndTag)
-            loc.endOffset = ctLocation.endOffset;
-        else
-            loc.endOffset = ctLocation.startOffset;
-    }
-}
-
-
-exports.assign = function (parser) {
-    //NOTE: obtain Parser proto this way to avoid module circular references
-    var parserProto = Object.getPrototypeOf(parser),
-        treeAdapter = parser.treeAdapter,
-        attachableElementLocation = null,
-        lastFosterParentingLocation = null,
-        currentToken = null;
-
-
-    //NOTE: patch _bootstrap method
-    parser._bootstrap = function (document, fragmentContext) {
-        parserProto._bootstrap.call(this, document, fragmentContext);
-
-        attachableElementLocation = null;
-        lastFosterParentingLocation = null;
-        currentToken = null;
-
-        //OpenElementStack
-        parser.openElements.pop = function () {
-            setEndLocation(this.current, currentToken, treeAdapter);
-            OpenElementStack.prototype.pop.call(this);
-        };
-
-        parser.openElements.popAllUpToHtmlElement = function () {
-            for (var i = this.stackTop; i > 0; i--)
-                setEndLocation(this.items[i], currentToken, treeAdapter);
-
-            OpenElementStack.prototype.popAllUpToHtmlElement.call(this);
-        };
-
-        parser.openElements.remove = function (element) {
-            setEndLocation(element, currentToken, treeAdapter);
-            OpenElementStack.prototype.remove.call(this, element);
-        };
-    };
-
-
-    //Token processing
-    parser._processTokenInForeignContent = function (token) {
-        currentToken = token;
-        parserProto._processTokenInForeignContent.call(this, token);
-    };
-
-    parser._processToken = function (token) {
-        currentToken = token;
-        parserProto._processToken.call(this, token);
-
-        //NOTE: <body> and <html> are never popped from the stack, so we need to updated
-        //their end location explicitly.
-        if (token.type === Tokenizer.END_TAG_TOKEN &&
-            (token.tagName === $.HTML ||
-             token.tagName === $.BODY && this.openElements.hasInScope($.BODY))) {
-            for (var i = this.openElements.stackTop; i >= 0; i--) {
-                var element = this.openElements.items[i];
-
-                if (this.treeAdapter.getTagName(element) === token.tagName) {
-                    setEndLocation(element, token, treeAdapter);
-                    break;
-                }
-            }
-        }
-    };
-
-
-    //Doctype
-    parser._setDocumentType = function (token) {
-        parserProto._setDocumentType.call(this, token);
-
-        var documentChildren = this.treeAdapter.getChildNodes(this.document),
-            cnLength = documentChildren.length;
-
-        for (var i = 0; i < cnLength; i++) {
-            var node = documentChildren[i];
-
-            if (this.treeAdapter.isDocumentTypeNode(node)) {
-                node.__location = token.location;
-                break;
-            }
-        }
-    };
-
-
-    //Elements
-    parser._attachElementToTree = function (element) {
-        //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
-        //So we will use token location stored in this methods for the element.
-        element.__location = attachableElementLocation || null;
-        attachableElementLocation = null;
-        parserProto._attachElementToTree.call(this, element);
-    };
-
-    parser._appendElement = function (token, namespaceURI) {
-        attachableElementLocation = token.location;
-        parserProto._appendElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertElement = function (token, namespaceURI) {
-        attachableElementLocation = token.location;
-        parserProto._insertElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertTemplate = function (token) {
-        attachableElementLocation = token.location;
-        parserProto._insertTemplate.call(this, token);
-
-        var tmplContent = this.treeAdapter.getTemplateContent(this.openElements.current);
-
-        tmplContent.__location = null;
-    };
-
-    parser._insertFakeRootElement = function () {
-        parserProto._insertFakeRootElement.call(this);
-        this.openElements.current.__location = null;
-    };
-
-
-    //Comments
-    parser._appendCommentNode = function (token, parent) {
-        parserProto._appendCommentNode.call(this, token, parent);
-
-        var children = this.treeAdapter.getChildNodes(parent),
-            commentNode = children[children.length - 1];
-
-        commentNode.__location = token.location;
-    };
-
-
-    //Text
-    parser._findFosterParentingLocation = function () {
-        //NOTE: store last foster parenting location, so we will be able to find inserted text
-        //in case of foster parenting
-        lastFosterParentingLocation = parserProto._findFosterParentingLocation.call(this);
-        return lastFosterParentingLocation;
-    };
-
-    parser._insertCharacters = function (token) {
-        parserProto._insertCharacters.call(this, token);
-
-        var hasFosterParent = this._shouldFosterParentOnInsertion(),
-            parent = hasFosterParent && lastFosterParentingLocation.parent ||
-                     this.openElements.currentTmplContent ||
-                     this.openElements.current,
-            siblings = this.treeAdapter.getChildNodes(parent),
-            textNodeIdx = hasFosterParent && lastFosterParentingLocation.beforeElement ?
-            siblings.indexOf(lastFosterParentingLocation.beforeElement) - 1 :
-            siblings.length - 1,
-            textNode = siblings[textNodeIdx];
-
-        //NOTE: if we have location assigned by another token, then just update end position
-        if (textNode.__location)
-            textNode.__location.endOffset = token.location.endOffset;
-
-        else
-            textNode.__location = token.location;
-    };
-};
-