parse5 3.0.1 → 4.0.0

package/LICENSE

@@ -1,19 +1,19 @@
-Copyright (c) 2013-2016 Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+Copyright (c) 2013-2016 Ivan Nikulin (ifaaan@gmail.com, https://github.com/inikulin)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

package/lib/common/unicode.js

@@ -1,47 +1,47 @@
-'use strict';
-
-exports.REPLACEMENT_CHARACTER = '\uFFFD';
-
-exports.CODE_POINTS = {
-    EOF: -1,
-    NULL: 0x00,
-    TABULATION: 0x09,
-    CARRIAGE_RETURN: 0x0D,
-    LINE_FEED: 0x0A,
-    FORM_FEED: 0x0C,
-    SPACE: 0x20,
-    EXCLAMATION_MARK: 0x21,
-    QUOTATION_MARK: 0x22,
-    NUMBER_SIGN: 0x23,
-    AMPERSAND: 0x26,
-    APOSTROPHE: 0x27,
-    HYPHEN_MINUS: 0x2D,
-    SOLIDUS: 0x2F,
-    DIGIT_0: 0x30,
-    DIGIT_9: 0x39,
-    SEMICOLON: 0x3B,
-    LESS_THAN_SIGN: 0x3C,
-    EQUALS_SIGN: 0x3D,
-    GREATER_THAN_SIGN: 0x3E,
-    QUESTION_MARK: 0x3F,
-    LATIN_CAPITAL_A: 0x41,
-    LATIN_CAPITAL_F: 0x46,
-    LATIN_CAPITAL_X: 0x58,
-    LATIN_CAPITAL_Z: 0x5A,
-    GRAVE_ACCENT: 0x60,
-    LATIN_SMALL_A: 0x61,
-    LATIN_SMALL_F: 0x66,
-    LATIN_SMALL_X: 0x78,
-    LATIN_SMALL_Z: 0x7A,
-    REPLACEMENT_CHARACTER: 0xFFFD
-};
-
-exports.CODE_POINT_SEQUENCES = {
-    DASH_DASH_STRING: [0x2D, 0x2D], //--
-    DOCTYPE_STRING: [0x44, 0x4F, 0x43, 0x54, 0x59, 0x50, 0x45], //DOCTYPE
-    CDATA_START_STRING: [0x5B, 0x43, 0x44, 0x41, 0x54, 0x41, 0x5B], //[CDATA[
-    CDATA_END_STRING: [0x5D, 0x5D, 0x3E], //]]>
-    SCRIPT_STRING: [0x73, 0x63, 0x72, 0x69, 0x70, 0x74], //script
-    PUBLIC_STRING: [0x50, 0x55, 0x42, 0x4C, 0x49, 0x43], //PUBLIC
-    SYSTEM_STRING: [0x53, 0x59, 0x53, 0x54, 0x45, 0x4D] //SYSTEM
-};
+'use strict';
+
+exports.REPLACEMENT_CHARACTER = '\uFFFD';
+
+exports.CODE_POINTS = {
+    EOF: -1,
+    NULL: 0x00,
+    TABULATION: 0x09,
+    CARRIAGE_RETURN: 0x0D,
+    LINE_FEED: 0x0A,
+    FORM_FEED: 0x0C,
+    SPACE: 0x20,
+    EXCLAMATION_MARK: 0x21,
+    QUOTATION_MARK: 0x22,
+    NUMBER_SIGN: 0x23,
+    AMPERSAND: 0x26,
+    APOSTROPHE: 0x27,
+    HYPHEN_MINUS: 0x2D,
+    SOLIDUS: 0x2F,
+    DIGIT_0: 0x30,
+    DIGIT_9: 0x39,
+    SEMICOLON: 0x3B,
+    LESS_THAN_SIGN: 0x3C,
+    EQUALS_SIGN: 0x3D,
+    GREATER_THAN_SIGN: 0x3E,
+    QUESTION_MARK: 0x3F,
+    LATIN_CAPITAL_A: 0x41,
+    LATIN_CAPITAL_F: 0x46,
+    LATIN_CAPITAL_X: 0x58,
+    LATIN_CAPITAL_Z: 0x5A,
+    GRAVE_ACCENT: 0x60,
+    LATIN_SMALL_A: 0x61,
+    LATIN_SMALL_F: 0x66,
+    LATIN_SMALL_X: 0x78,
+    LATIN_SMALL_Z: 0x7A,
+    REPLACEMENT_CHARACTER: 0xFFFD
+};
+
+exports.CODE_POINT_SEQUENCES = {
+    DASH_DASH_STRING: [0x2D, 0x2D], //--
+    DOCTYPE_STRING: [0x44, 0x4F, 0x43, 0x54, 0x59, 0x50, 0x45], //DOCTYPE
+    CDATA_START_STRING: [0x5B, 0x43, 0x44, 0x41, 0x54, 0x41, 0x5B], //[CDATA[
+    CDATA_END_STRING: [0x5D, 0x5D, 0x3E], //]]>
+    SCRIPT_STRING: [0x73, 0x63, 0x72, 0x69, 0x70, 0x74], //script
+    PUBLIC_STRING: [0x50, 0x55, 0x42, 0x4C, 0x49, 0x43], //PUBLIC
+    SYSTEM_STRING: [0x53, 0x59, 0x53, 0x54, 0x45, 0x4D] //SYSTEM
+};

package/lib/extensions/location_info/open_element_stack_mixin.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    inherits = require('util').inherits;
+
+var LocationInfoOpenElementStackMixin = module.exports = function (stack, options) {
+    Mixin.call(this, stack);
+
+    this.onItemPop = options.onItemPop;
+};
+
+inherits(LocationInfoOpenElementStackMixin, Mixin);
+
+LocationInfoOpenElementStackMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        pop: function () {
+            mxn.onItemPop(this.current);
+            orig.pop.call(this);
+        },
+
+        popAllUpToHtmlElement: function () {
+            for (var i = this.stackTop; i > 0; i--)
+                mxn.onItemPop(this.items[i]);
+
+            orig.popAllUpToHtmlElement.call(this);
+        },
+
+        remove: function (element) {
+            mxn.onItemPop(this.current);
+            orig.remove.call(this, element);
+        }
+    };
+};
+

package/lib/extensions/location_info/parser_mixin.js

@@ -0,0 +1,213 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    Tokenizer = require('../../tokenizer'),
+    LocationInfoTokenizerMixin = require('./tokenizer_mixin'),
+    PositionTrackingPreprocessorMixin = require('../position_tracking/preprocessor_mixin'),
+    LocationInfoOpenElementStackMixin = require('./open_element_stack_mixin'),
+    HTML = require('../../common/html'),
+    inherits = require('util').inherits;
+
+
+//Aliases
+var $ = HTML.TAG_NAMES;
+
+var LocationInfoParserMixin = module.exports = function (parser) {
+    Mixin.call(this, parser);
+
+    this.parser = parser;
+    this.posTracker = null;
+    this.lastStartTagToken = null;
+    this.lastFosterParentingLocation = null;
+    this.currentToken = null;
+};
+
+inherits(LocationInfoParserMixin, Mixin);
+
+
+LocationInfoParserMixin.prototype._setStartLocation = function (element) {
+    if (this.lastStartTagToken) {
+        element.__location = Object.create(this.lastStartTagToken.location);
+        element.__location.startTag = this.lastStartTagToken.location;
+    }
+    else
+        element.__location = null;
+};
+
+LocationInfoParserMixin.prototype._setEndLocation = function (element, closingToken) {
+    var loc = element.__location;
+
+    if (loc) {
+        if (closingToken.location) {
+            var ctLoc = closingToken.location,
+                tn = this.parser.treeAdapter.getTagName(element);
+
+            // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing
+            // tag and for cases like <td> <p> </td> - 'p' closes without a closing tag.
+            var isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN && tn === closingToken.tagName;
+
+            if (isClosingEndTag) {
+                loc.endTag = Object.create(ctLoc);
+                loc.endOffset = ctLoc.endOffset;
+            }
+
+            else
+                loc.endOffset = ctLoc.startOffset;
+        }
+
+        else if (closingToken.type === Tokenizer.EOF_TOKEN)
+            loc.endOffset = this.posTracker.offset;
+    }
+};
+
+LocationInfoParserMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        _bootstrap: function (document, fragmentContext) {
+            orig._bootstrap.call(this, document, fragmentContext);
+
+            mxn.lastStartTagToken = null;
+            mxn.lastFosterParentingLocation = null;
+            mxn.currentToken = null;
+            mxn.posTracker = new PositionTrackingPreprocessorMixin(this.tokenizer.preprocessor);
+
+            new LocationInfoTokenizerMixin(this.tokenizer);
+
+            new LocationInfoOpenElementStackMixin(this.openElements, {
+                onItemPop: function (element) {
+                    mxn._setEndLocation(element, mxn.currentToken);
+                }
+            });
+        },
+
+        _runParsingLoop: function (scriptHandler) {
+            orig._runParsingLoop.call(this, scriptHandler);
+
+            // NOTE: generate location info for elements
+            // that remains on open element stack
+            for (var i = this.openElements.stackTop; i >= 0; i--)
+                mxn._setEndLocation(this.openElements.items[i], mxn.currentToken);
+        },
+
+
+        //Token processing
+        _processTokenInForeignContent: function (token) {
+            mxn.currentToken = token;
+            orig._processTokenInForeignContent.call(this, token);
+        },
+
+        _processToken: function (token) {
+            mxn.currentToken = token;
+            orig._processToken.call(this, token);
+
+            //NOTE: <body> and <html> are never popped from the stack, so we need to updated
+            //their end location explicitly.
+            var requireExplicitUpdate = token.type === Tokenizer.END_TAG_TOKEN &&
+                                        (token.tagName === $.HTML ||
+                                         token.tagName === $.BODY && this.openElements.hasInScope($.BODY));
+
+            if (requireExplicitUpdate) {
+                for (var i = this.openElements.stackTop; i >= 0; i--) {
+                    var element = this.openElements.items[i];
+
+                    if (this.treeAdapter.getTagName(element) === token.tagName) {
+                        mxn._setEndLocation(element, token);
+                        break;
+                    }
+                }
+            }
+        },
+
+
+        //Doctype
+        _setDocumentType: function (token) {
+            orig._setDocumentType.call(this, token);
+
+            var documentChildren = this.treeAdapter.getChildNodes(this.document),
+                cnLength = documentChildren.length;
+
+            for (var i = 0; i < cnLength; i++) {
+                var node = documentChildren[i];
+
+                if (this.treeAdapter.isDocumentTypeNode(node)) {
+                    node.__location = token.location;
+                    break;
+                }
+            }
+        },
+
+
+        //Elements
+        _attachElementToTree: function (element) {
+            //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
+            //So we will use token location stored in this methods for the element.
+            mxn._setStartLocation(element);
+            mxn.lastStartTagToken = null;
+            orig._attachElementToTree.call(this, element);
+        },
+
+        _appendElement: function (token, namespaceURI) {
+            mxn.lastStartTagToken = token;
+            orig._appendElement.call(this, token, namespaceURI);
+        },
+
+        _insertElement: function (token, namespaceURI) {
+            mxn.lastStartTagToken = token;
+            orig._insertElement.call(this, token, namespaceURI);
+        },
+
+        _insertTemplate: function (token) {
+            mxn.lastStartTagToken = token;
+            orig._insertTemplate.call(this, token);
+
+            var tmplContent = this.treeAdapter.getTemplateContent(this.openElements.current);
+
+            tmplContent.__location = null;
+        },
+
+        _insertFakeRootElement: function () {
+            orig._insertFakeRootElement.call(this);
+            this.openElements.current.__location = null;
+        },
+
+        //Comments
+        _appendCommentNode: function (token, parent) {
+            orig._appendCommentNode.call(this, token, parent);
+
+            var children = this.treeAdapter.getChildNodes(parent),
+                commentNode = children[children.length - 1];
+
+            commentNode.__location = token.location;
+        },
+
+        //Text
+        _findFosterParentingLocation: function () {
+            //NOTE: store last foster parenting location, so we will be able to find inserted text
+            //in case of foster parenting
+            mxn.lastFosterParentingLocation = orig._findFosterParentingLocation.call(this);
+
+            return mxn.lastFosterParentingLocation;
+        },
+
+        _insertCharacters: function (token) {
+            orig._insertCharacters.call(this, token);
+
+            var hasFosterParent = this._shouldFosterParentOnInsertion(),
+                parent = hasFosterParent && mxn.lastFosterParentingLocation.parent ||
+                         this.openElements.currentTmplContent ||
+                         this.openElements.current,
+                siblings = this.treeAdapter.getChildNodes(parent),
+                textNodeIdx = hasFosterParent && mxn.lastFosterParentingLocation.beforeElement ?
+                siblings.indexOf(mxn.lastFosterParentingLocation.beforeElement) - 1 :
+                siblings.length - 1,
+                textNode = siblings[textNodeIdx];
+
+            //NOTE: if we have location assigned by another token, then just update end position
+            if (textNode.__location)
+                textNode.__location.endOffset = token.location.endOffset;
+
+            else
+                textNode.__location = token.location;
+        }
+    };
+};
+

package/lib/extensions/location_info/tokenizer_mixin.js

@@ -0,0 +1,117 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    Tokenizer = require('../../tokenizer'),
+    PositionTrackingPreprocessorMixin = require('../position_tracking/preprocessor_mixin'),
+    inherits = require('util').inherits;
+
+var LocationInfoTokenizerMixin = module.exports = function (tokenizer) {
+    Mixin.call(this, tokenizer);
+
+    this.tokenizer = tokenizer;
+    this.posTracker = new PositionTrackingPreprocessorMixin(tokenizer.preprocessor);
+    this.currentAttrLocation = null;
+    this.currentTokenLocation = null;
+};
+
+inherits(LocationInfoTokenizerMixin, Mixin);
+
+LocationInfoTokenizerMixin.prototype._getCurrentLocation = function () {
+    return {
+        line: this.posTracker.line,
+        col: this.posTracker.col,
+        startOffset: this.posTracker.offset,
+        endOffset: -1
+    };
+};
+
+LocationInfoTokenizerMixin.prototype._attachCurrentAttrLocationInfo = function () {
+    this.currentAttrLocation.endOffset = this.posTracker.offset;
+
+    var currentToken = this.tokenizer.currentToken,
+        currentAttr = this.tokenizer.currentAttr;
+
+    if (!currentToken.location.attrs)
+        currentToken.location.attrs = Object.create(null);
+
+    currentToken.location.attrs[currentAttr.name] = this.currentAttrLocation;
+};
+
+LocationInfoTokenizerMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    var methods = {
+        _createStartTagToken: function () {
+            orig._createStartTagToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createEndTagToken: function () {
+            orig._createEndTagToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createCommentToken: function () {
+            orig._createCommentToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createDoctypeToken: function (initialName) {
+            orig._createDoctypeToken.call(this, initialName);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createCharacterToken: function (type, ch) {
+            orig._createCharacterToken.call(this, type, ch);
+            this.currentCharacterToken.location = mxn.currentTokenLocation;
+        },
+
+        _createAttr: function (attrNameFirstCh) {
+            orig._createAttr.call(this, attrNameFirstCh);
+            mxn.currentAttrLocation = mxn._getCurrentLocation();
+        },
+
+        _leaveAttrName: function (toState) {
+            orig._leaveAttrName.call(this, toState);
+            mxn._attachCurrentAttrLocationInfo();
+        },
+
+        _leaveAttrValue: function (toState) {
+            orig._leaveAttrValue.call(this, toState);
+            mxn._attachCurrentAttrLocationInfo();
+        },
+
+        _emitCurrentToken: function () {
+            //NOTE: if we have pending character token make it's end location equal to the
+            //current token's start location.
+            if (this.currentCharacterToken)
+                this.currentCharacterToken.location.endOffset = this.currentToken.location.startOffset;
+
+            this.currentToken.location.endOffset = mxn.posTracker.offset + 1;
+            orig._emitCurrentToken.call(this);
+        },
+
+        _emitCurrentCharacterToken: function () {
+            //NOTE: if we have character token and it's location wasn't set in the _emitCurrentToken(),
+            //then set it's location at the current preprocessor position.
+            //We don't need to increment preprocessor position, since character token
+            //emission is always forced by the start of the next character token here.
+            //So, we already have advanced position.
+            if (this.currentCharacterToken && this.currentCharacterToken.location.endOffset === -1)
+                this.currentCharacterToken.location.endOffset = mxn.posTracker.offset;
+
+            orig._emitCurrentCharacterToken.call(this);
+        }
+    };
+
+    //NOTE: patch initial states for each mode to obtain token start position
+    Object.keys(Tokenizer.MODE).forEach(function (modeName) {
+        var state = Tokenizer.MODE[modeName];
+
+        methods[state] = function (cp) {
+            mxn.currentTokenLocation = mxn._getCurrentLocation();
+            orig[state].call(this, cp);
+        };
+    });
+
+    return methods;
+};
+

package/lib/extensions/position_tracking/preprocessor_mixin.js

@@ -0,0 +1,72 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    inherits = require('util').inherits,
+    UNICODE = require('../../common/unicode');
+
+//Aliases
+var $ = UNICODE.CODE_POINTS;
+
+var PositionTrackingPreprocessorMixin = module.exports = function (preprocessor) {
+    // NOTE: avoid installing tracker twice
+    if (!preprocessor.__locTracker) {
+        preprocessor.__locTracker = this;
+
+        Mixin.call(this, preprocessor);
+
+        this.preprocessor = preprocessor;
+        this.isEol = false;
+        this.lineStartPos = 0;
+        this.droppedBufferSize = 0;
+
+        this.col = -1;
+        this.line = 1;
+    }
+
+    return preprocessor.__locTracker;
+};
+
+inherits(PositionTrackingPreprocessorMixin, Mixin);
+
+Object.defineProperty(PositionTrackingPreprocessorMixin.prototype, 'offset', {
+    get: function () {
+        return this.droppedBufferSize + this.preprocessor.pos;
+    }
+});
+
+PositionTrackingPreprocessorMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        advance: function () {
+            var cp = orig.advance.call(this);
+
+            //NOTE: LF should be in the last column of the line
+            if (mxn.isEol) {
+                mxn.isEol = false;
+                mxn.line++;
+                mxn.lineStartPos = mxn.offset;
+            }
+
+            if (cp === $.LINE_FEED)
+                mxn.isEol = true;
+
+            mxn.col = mxn.offset - mxn.lineStartPos + 1;
+
+            return cp;
+        },
+
+        retreat: function () {
+            orig.retreat.call(this);
+            mxn.isEol = false;
+
+            mxn.col = mxn.offset - mxn.lineStartPos + 1;
+        },
+
+        dropParsedChunk: function () {
+            var prevPos = this.pos;
+
+            orig.dropParsedChunk.call(this);
+
+            mxn.droppedBufferSize += prevPos - this.pos;
+        }
+    };
+};