parse5 1.4.0 → 1.5.1

package/lib/tokenization/preprocessor.js

@@ -1,115 +1,115 @@
-'use strict';
-
-var UNICODE = require('../common/unicode');
-
-//Aliases
-var $ = UNICODE.CODE_POINTS;
-
-//Utils
-
-//OPTIMIZATION: these utility functions should not be moved out of this module. V8 Crankshaft will not inline
-//this functions if they will be situated in another module due to context switch.
-//Always perform inlining check before modifying this functions ('node --trace-inlining').
-function isReservedCodePoint(cp) {
-    return cp >= 0xD800 && cp <= 0xDFFF || cp > 0x10FFFF;
-}
-
-function isSurrogatePair(cp1, cp2) {
-    return cp1 >= 0xD800 && cp1 <= 0xDBFF && cp2 >= 0xDC00 && cp2 <= 0xDFFF;
-}
-
-function getSurrogatePairCodePoint(cp1, cp2) {
-    return (cp1 - 0xD800) * 0x400 + 0x2400 + cp2;
-}
-
-//Preprocessor
-//NOTE: HTML input preprocessing
-//(see: http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream)
-var Preprocessor = module.exports = function (html) {
-    this.write(html);
-
-    //NOTE: one leading U+FEFF BYTE ORDER MARK character must be ignored if any are present in the input stream.
-    this.pos = this.html.charCodeAt(0) === $.BOM ? 0 : -1;
-
-    this.gapStack = [];
-    this.lastGapPos = -1;
-    this.skipNextNewLine = false;
-};
-
-Preprocessor.prototype.write = function (html) {
-    if (this.html) {
-        this.html = this.html.substring(0, this.pos + 1) +
-                    html +
-                    this.html.substring(this.pos + 1, this.html.length);
-
-    }
-    else
-        this.html = html;
-
-
-    this.lastCharPos = this.html.length - 1;
-};
-
-Preprocessor.prototype.advanceAndPeekCodePoint = function () {
-    this.pos++;
-
-    if (this.pos > this.lastCharPos)
-        return $.EOF;
-
-    var cp = this.html.charCodeAt(this.pos);
-
-    //NOTE: any U+000A LINE FEED (LF) characters that immediately follow a U+000D CARRIAGE RETURN (CR) character
-    //must be ignored.
-    if (this.skipNextNewLine && cp === $.LINE_FEED) {
-        this.skipNextNewLine = false;
-        this._addGap();
-        return this.advanceAndPeekCodePoint();
-    }
-
-    //NOTE: all U+000D CARRIAGE RETURN (CR) characters must be converted to U+000A LINE FEED (LF) characters
-    if (cp === $.CARRIAGE_RETURN) {
-        this.skipNextNewLine = true;
-        return $.LINE_FEED;
-    }
-
-    this.skipNextNewLine = false;
-
-    //OPTIMIZATION: first perform check if the code point in the allowed range that covers most common
-    //HTML input (e.g. ASCII codes) to avoid performance-cost operations for high-range code points.
-    return cp >= 0xD800 ? this._processHighRangeCodePoint(cp) : cp;
-};
-
-Preprocessor.prototype._processHighRangeCodePoint = function (cp) {
-    //NOTE: try to peek a surrogate pair
-    if (this.pos !== this.lastCharPos) {
-        var nextCp = this.html.charCodeAt(this.pos + 1);
-
-        if (isSurrogatePair(cp, nextCp)) {
-            //NOTE: we have a surrogate pair. Peek pair character and recalculate code point.
-            this.pos++;
-            cp = getSurrogatePairCodePoint(cp, nextCp);
-
-            //NOTE: add gap that should be avoided during retreat
-            this._addGap();
-        }
-    }
-
-    if (isReservedCodePoint(cp))
-        cp = $.REPLACEMENT_CHARACTER;
-
-    return cp;
-};
-
-Preprocessor.prototype._addGap = function () {
-    this.gapStack.push(this.lastGapPos);
-    this.lastGapPos = this.pos;
-};
-
-Preprocessor.prototype.retreat = function () {
-    if (this.pos === this.lastGapPos) {
-        this.lastGapPos = this.gapStack.pop();
-        this.pos--;
-    }
-
-    this.pos--;
-};
+'use strict';
+
+var UNICODE = require('../common/unicode');
+
+//Aliases
+var $ = UNICODE.CODE_POINTS;
+
+//Utils
+
+//OPTIMIZATION: these utility functions should not be moved out of this module. V8 Crankshaft will not inline
+//this functions if they will be situated in another module due to context switch.
+//Always perform inlining check before modifying this functions ('node --trace-inlining').
+function isReservedCodePoint(cp) {
+    return cp >= 0xD800 && cp <= 0xDFFF || cp > 0x10FFFF;
+}
+
+function isSurrogatePair(cp1, cp2) {
+    return cp1 >= 0xD800 && cp1 <= 0xDBFF && cp2 >= 0xDC00 && cp2 <= 0xDFFF;
+}
+
+function getSurrogatePairCodePoint(cp1, cp2) {
+    return (cp1 - 0xD800) * 0x400 + 0x2400 + cp2;
+}
+
+//Preprocessor
+//NOTE: HTML input preprocessing
+//(see: http://www.whatwg.org/specs/web-apps/current-work/multipage/parsing.html#preprocessing-the-input-stream)
+var Preprocessor = module.exports = function (html) {
+    this.write(html);
+
+    //NOTE: one leading U+FEFF BYTE ORDER MARK character must be ignored if any are present in the input stream.
+    this.pos = this.html.charCodeAt(0) === $.BOM ? 0 : -1;
+
+    this.gapStack = [];
+    this.lastGapPos = -1;
+    this.skipNextNewLine = false;
+};
+
+Preprocessor.prototype.write = function (html) {
+    if (this.html) {
+        this.html = this.html.substring(0, this.pos + 1) +
+                    html +
+                    this.html.substring(this.pos + 1, this.html.length);
+
+    }
+    else
+        this.html = html;
+
+
+    this.lastCharPos = this.html.length - 1;
+};
+
+Preprocessor.prototype.advanceAndPeekCodePoint = function () {
+    this.pos++;
+
+    if (this.pos > this.lastCharPos)
+        return $.EOF;
+
+    var cp = this.html.charCodeAt(this.pos);
+
+    //NOTE: any U+000A LINE FEED (LF) characters that immediately follow a U+000D CARRIAGE RETURN (CR) character
+    //must be ignored.
+    if (this.skipNextNewLine && cp === $.LINE_FEED) {
+        this.skipNextNewLine = false;
+        this._addGap();
+        return this.advanceAndPeekCodePoint();
+    }
+
+    //NOTE: all U+000D CARRIAGE RETURN (CR) characters must be converted to U+000A LINE FEED (LF) characters
+    if (cp === $.CARRIAGE_RETURN) {
+        this.skipNextNewLine = true;
+        return $.LINE_FEED;
+    }
+
+    this.skipNextNewLine = false;
+
+    //OPTIMIZATION: first perform check if the code point in the allowed range that covers most common
+    //HTML input (e.g. ASCII codes) to avoid performance-cost operations for high-range code points.
+    return cp >= 0xD800 ? this._processHighRangeCodePoint(cp) : cp;
+};
+
+Preprocessor.prototype._processHighRangeCodePoint = function (cp) {
+    //NOTE: try to peek a surrogate pair
+    if (this.pos !== this.lastCharPos) {
+        var nextCp = this.html.charCodeAt(this.pos + 1);
+
+        if (isSurrogatePair(cp, nextCp)) {
+            //NOTE: we have a surrogate pair. Peek pair character and recalculate code point.
+            this.pos++;
+            cp = getSurrogatePairCodePoint(cp, nextCp);
+
+            //NOTE: add gap that should be avoided during retreat
+            this._addGap();
+        }
+    }
+
+    if (isReservedCodePoint(cp))
+        cp = $.REPLACEMENT_CHARACTER;
+
+    return cp;
+};
+
+Preprocessor.prototype._addGap = function () {
+    this.gapStack.push(this.lastGapPos);
+    this.lastGapPos = this.pos;
+};
+
+Preprocessor.prototype.retreat = function () {
+    if (this.pos === this.lastGapPos) {
+        this.lastGapPos = this.gapStack.pop();
+        this.pos--;
+    }
+
+    this.pos--;
+};

package/lib/tree_adapters/htmlparser2.js

@@ -1,5 +1,7 @@
 'use strict';
 
+var Doctype = require('../common/doctype');
+
 //Conversion tables for DOM Level1 structure emulation
 var nodeTypes = {
     element: 1,
@@ -119,18 +121,8 @@ var createTextNode = function (value) {
 
 //Tree mutation
 exports.setDocumentType = function (document, name, publicId, systemId) {
-    var data = '!DOCTYPE';
-
-    if (name)
-        data += ' ' + name;
-
-    if (publicId)
-        data += ' PUBLIC "' + publicId + '"';
-
-    if (systemId)
-        data += ' "' + systemId + '"';
-
-    var doctypeNode = null;
+    var data = Doctype.serializeContent(name, publicId, systemId),
+        doctypeNode = null;
 
     for (var i = 0; i < document.children.length; i++) {
         if (document.children[i].type === 'directive' && document.children[i].name === '!doctype') {

package/lib/tree_construction/location_info_mixin.js

@@ -1,166 +1,197 @@
-'use strict';
-
-var OpenElementStack = require('./open_element_stack'),
-    Tokenizer = require('../tokenization/tokenizer'),
-    HTML = require('../common/html');
-
-
-//Aliases
-var $ = HTML.TAG_NAMES;
-
-//NOTE: patch open elements stack, so we can assign end location for the elements
-function patchOpenElementsStack(stack, parser) {
-    function setEndLocation(element) {
-        if (element.__location)
-            element.__location.end = parser.currentToken.location.end;
-    }
-
-    stack.pop = function () {
-        setEndLocation(this.current);
-        OpenElementStack.prototype.pop.call(this);
-    };
-
-    stack.popAllUpToHtmlElement = function () {
-        for (var i = this.stackTop; i > 0; i--)
-            setEndLocation(this.items[i]);
-
-        OpenElementStack.prototype.popAllUpToHtmlElement.call(this);
-    };
-
-    stack.remove = function (element) {
-        setEndLocation(element);
-        OpenElementStack.prototype.remove.call(this, element);
-    };
-}
-
-exports.assign = function (parser) {
-    //NOTE: obtain Parser proto this way to avoid module circular references
-    var parserProto = Object.getPrototypeOf(parser);
-
-    //NOTE: patch _reset method
-    parser._reset = function (html, document, fragmentContext) {
-        parserProto._reset.call(this, html, document, fragmentContext);
-
-        this.attachableElementLocation = null;
-        this.lastFosterParentingLocation = null;
-        this.currentToken = null;
-
-        patchOpenElementsStack(this.openElements, parser);
-    };
-
-    parser._processTokenInForeignContent = function (token) {
-        this.currentToken = token;
-        parserProto._processTokenInForeignContent.call(this, token);
-    };
-
-    parser._processToken = function (token) {
-        this.currentToken = token;
-        parserProto._processToken.call(this, token);
-
-        //NOTE: <body> and <html> are never popped from the stack, so we need to updated
-        //their end location explicitly.
-        if (token.type === Tokenizer.END_TAG_TOKEN &&
-            (token.tagName === $.HTML ||
-             (token.tagName === $.BODY && this.openElements.hasInScope($.BODY)))) {
-            for (var i = this.openElements.stackTop; i >= 0; i--) {
-                var element = this.openElements.items[i];
-
-                if (this.treeAdapter.getTagName(element) === token.tagName) {
-                    element.__location.end = token.location.end;
-                    break;
-                }
-            }
-        }
-    };
-
-    //Doctype
-    parser._setDocumentType = function (token) {
-        parserProto._setDocumentType.call(this, token);
-
-        var documentChildren = this.treeAdapter.getChildNodes(this.document),
-            cnLength = documentChildren.length;
-
-        for (var i = 0; i < cnLength; i++) {
-            var node = documentChildren[i];
-
-            if (this.treeAdapter.isDocumentTypeNode(node)) {
-                node.__location = token.location;
-                break;
-            }
-        }
-    };
-
-    //Elements
-    parser._attachElementToTree = function (element) {
-        //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
-        //So we will use token location stored in this methods for the element.
-        element.__location = this.attachableElementLocation || null;
-        this.attachableElementLocation = null;
-        parserProto._attachElementToTree.call(this, element);
-    };
-
-    parser._appendElement = function (token, namespaceURI) {
-        this.attachableElementLocation = token.location;
-        parserProto._appendElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertElement = function (token, namespaceURI) {
-        this.attachableElementLocation = token.location;
-        parserProto._insertElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertTemplate = function (token) {
-        this.attachableElementLocation = token.location;
-        parserProto._insertTemplate.call(this, token);
-
-        var tmplContent = this.treeAdapter.getChildNodes(this.openElements.current)[0];
-
-        tmplContent.__location = null;
-    };
-
-    parser._insertFakeRootElement = function () {
-        parserProto._insertFakeRootElement.call(this);
-        this.openElements.current.__location = null;
-    };
-
-    //Comments
-    parser._appendCommentNode = function (token, parent) {
-        parserProto._appendCommentNode.call(this, token, parent);
-
-        var children = this.treeAdapter.getChildNodes(parent),
-            commentNode = children[children.length - 1];
-
-        commentNode.__location = token.location;
-    };
-
-    //Text
-    parser._findFosterParentingLocation = function () {
-        //NOTE: store last foster parenting location, so we will be able to find inserted text
-        //in case of foster parenting
-        this.lastFosterParentingLocation = parserProto._findFosterParentingLocation.call(this);
-        return this.lastFosterParentingLocation;
-    };
-
-    parser._insertCharacters = function (token) {
-        parserProto._insertCharacters.call(this, token);
-
-        var hasFosterParent = this._shouldFosterParentOnInsertion(),
-            parentingLocation = this.lastFosterParentingLocation,
-            parent = (hasFosterParent && parentingLocation.parent) ||
-                     this.openElements.currentTmplContent ||
-                     this.openElements.current,
-            siblings = this.treeAdapter.getChildNodes(parent),
-            textNodeIdx = hasFosterParent && parentingLocation.beforeElement ?
-                          siblings.indexOf(parentingLocation.beforeElement) - 1 :
-                          siblings.length - 1,
-            textNode = siblings[textNodeIdx];
-
-        //NOTE: if we have location assigned by another token, then just update end position
-        if (textNode.__location)
-            textNode.__location.end = token.location.end;
-
-        else
-            textNode.__location = token.location;
-    };
-};
-
+'use strict';
+
+var OpenElementStack = require('./open_element_stack'),
+    Tokenizer = require('../tokenization/tokenizer'),
+    HTML = require('../common/html');
+
+
+//Aliases
+var $ = HTML.TAG_NAMES;
+
+
+function setEndLocation(element, closingToken, treeAdapter) {
+    var loc = element.__location;
+
+    if (!loc)
+        return;
+
+    if (!loc.startTag) {
+        loc.startTag = {
+            start: loc.start,
+            end: loc.end
+        };
+    }
+
+    if (closingToken.location) {
+        var tn = treeAdapter.getTagName(element),
+            // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing tag and
+            // for cases like <td> <p> </td> - 'p' closes without a closing tag
+            isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN &&
+                              tn === closingToken.tagName;
+
+        if (isClosingEndTag) {
+            loc.endTag = {
+                start: closingToken.location.start,
+                end: closingToken.location.end
+            };
+        }
+
+        loc.end = closingToken.location.end;
+    }
+}
+
+//NOTE: patch open elements stack, so we can assign end location for the elements
+function patchOpenElementsStack(stack, parser) {
+    var treeAdapter = parser.treeAdapter;
+
+    stack.pop = function () {
+        setEndLocation(this.current, parser.currentToken, treeAdapter);
+        OpenElementStack.prototype.pop.call(this);
+    };
+
+    stack.popAllUpToHtmlElement = function () {
+        for (var i = this.stackTop; i > 0; i--)
+            setEndLocation(this.items[i], parser.currentToken, treeAdapter);
+
+        OpenElementStack.prototype.popAllUpToHtmlElement.call(this);
+    };
+
+    stack.remove = function (element) {
+        setEndLocation(element, parser.currentToken, treeAdapter);
+        OpenElementStack.prototype.remove.call(this, element);
+    };
+}
+
+exports.assign = function (parser) {
+    //NOTE: obtain Parser proto this way to avoid module circular references
+    var parserProto = Object.getPrototypeOf(parser),
+        treeAdapter = parser.treeAdapter;
+
+
+    //NOTE: patch _reset method
+    parser._reset = function (html, document, fragmentContext) {
+        parserProto._reset.call(this, html, document, fragmentContext);
+
+        this.attachableElementLocation = null;
+        this.lastFosterParentingLocation = null;
+        this.currentToken = null;
+
+        patchOpenElementsStack(this.openElements, parser);
+    };
+
+    parser._processTokenInForeignContent = function (token) {
+        this.currentToken = token;
+        parserProto._processTokenInForeignContent.call(this, token);
+    };
+
+    parser._processToken = function (token) {
+        this.currentToken = token;
+        parserProto._processToken.call(this, token);
+
+        //NOTE: <body> and <html> are never popped from the stack, so we need to updated
+        //their end location explicitly.
+        if (token.type === Tokenizer.END_TAG_TOKEN &&
+            (token.tagName === $.HTML ||
+            (token.tagName === $.BODY && this.openElements.hasInScope($.BODY)))) {
+            for (var i = this.openElements.stackTop; i >= 0; i--) {
+                var element = this.openElements.items[i];
+
+                if (this.treeAdapter.getTagName(element) === token.tagName) {
+                    setEndLocation(element, token, treeAdapter);
+                    break;
+                }
+            }
+        }
+    };
+
+    //Doctype
+    parser._setDocumentType = function (token) {
+        parserProto._setDocumentType.call(this, token);
+
+        var documentChildren = this.treeAdapter.getChildNodes(this.document),
+            cnLength = documentChildren.length;
+
+        for (var i = 0; i < cnLength; i++) {
+            var node = documentChildren[i];
+
+            if (this.treeAdapter.isDocumentTypeNode(node)) {
+                node.__location = token.location;
+                break;
+            }
+        }
+    };
+
+    //Elements
+    parser._attachElementToTree = function (element) {
+        //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
+        //So we will use token location stored in this methods for the element.
+        element.__location = this.attachableElementLocation || null;
+        this.attachableElementLocation = null;
+        parserProto._attachElementToTree.call(this, element);
+    };
+
+    parser._appendElement = function (token, namespaceURI) {
+        this.attachableElementLocation = token.location;
+        parserProto._appendElement.call(this, token, namespaceURI);
+    };
+
+    parser._insertElement = function (token, namespaceURI) {
+        this.attachableElementLocation = token.location;
+        parserProto._insertElement.call(this, token, namespaceURI);
+    };
+
+    parser._insertTemplate = function (token) {
+        this.attachableElementLocation = token.location;
+        parserProto._insertTemplate.call(this, token);
+
+        var tmplContent = this.treeAdapter.getChildNodes(this.openElements.current)[0];
+
+        tmplContent.__location = null;
+    };
+
+    parser._insertFakeRootElement = function () {
+        parserProto._insertFakeRootElement.call(this);
+        this.openElements.current.__location = null;
+    };
+
+    //Comments
+    parser._appendCommentNode = function (token, parent) {
+        parserProto._appendCommentNode.call(this, token, parent);
+
+        var children = this.treeAdapter.getChildNodes(parent),
+            commentNode = children[children.length - 1];
+
+        commentNode.__location = token.location;
+    };
+
+    //Text
+    parser._findFosterParentingLocation = function () {
+        //NOTE: store last foster parenting location, so we will be able to find inserted text
+        //in case of foster parenting
+        this.lastFosterParentingLocation = parserProto._findFosterParentingLocation.call(this);
+        return this.lastFosterParentingLocation;
+    };
+
+    parser._insertCharacters = function (token) {
+        parserProto._insertCharacters.call(this, token);
+
+        var hasFosterParent = this._shouldFosterParentOnInsertion(),
+            parentingLocation = this.lastFosterParentingLocation,
+            parent = (hasFosterParent && parentingLocation.parent) ||
+                     this.openElements.currentTmplContent ||
+                     this.openElements.current,
+            siblings = this.treeAdapter.getChildNodes(parent),
+            textNodeIdx = hasFosterParent && parentingLocation.beforeElement ?
+                          siblings.indexOf(parentingLocation.beforeElement) - 1 :
+                          siblings.length - 1,
+            textNode = siblings[textNodeIdx];
+
+        //NOTE: if we have location assigned by another token, then just update end position
+        if (textNode.__location)
+            textNode.__location.end = token.location.end;
+
+        else
+            textNode.__location = token.location;
+    };
+};
+