parse5 1.0.1 → 1.1.3

package/lib/simple_api/simple_api_parser.js

@@ -0,0 +1,68 @@
+var Tokenizer = require('../tokenization/tokenizer'),
+    TokenizerProxy = require('./tokenizer_proxy');
+
+//Skipping handler
+function skip() {
+    //NOTE: do nothing =)
+}
+
+//SimpleApiParser
+var SimpleApiParser = module.exports = function (handlers) {
+    this.handlers = {
+        doctype: handlers.doctype || skip,
+        startTag: handlers.startTag || skip,
+        endTag: handlers.endTag || skip,
+        text: handlers.text || skip,
+        comment: handlers.comment || skip
+    };
+};
+
+//API
+SimpleApiParser.prototype.parse = function (html) {
+    var token = null;
+
+    this._reset(html);
+
+    do {
+        token = this.tokenizerProxy.getNextToken();
+
+        if (token.type === Tokenizer.CHARACTER_TOKEN ||
+            token.type === Tokenizer.WHITESPACE_CHARACTER_TOKEN ||
+            token.type === Tokenizer.NULL_CHARACTER_TOKEN) {
+            this.pendingText = (this.pendingText || '') + token.chars;
+        }
+
+        else {
+            this._emitPendingText();
+            this._handleToken(token);
+        }
+    } while (token.type !== Tokenizer.EOF_TOKEN)
+};
+
+//Internals
+SimpleApiParser.prototype._handleToken = function (token) {
+    if (token.type === Tokenizer.START_TAG_TOKEN)
+        this.handlers.startTag(token.tagName, token.attrs, token.selfClosing);
+
+    else if (token.type === Tokenizer.END_TAG_TOKEN)
+        this.handlers.endTag(token.tagName);
+
+    else if (token.type === Tokenizer.COMMENT_TOKEN)
+        this.handlers.comment(token.data);
+
+    else if (token.type === Tokenizer.DOCTYPE_TOKEN)
+        this.handlers.doctype(token.name, token.publicId, token.systemId);
+
+};
+
+SimpleApiParser.prototype._reset = function (html) {
+    this.tokenizerProxy = new TokenizerProxy(html);
+    this.pendingText = null;
+};
+
+SimpleApiParser.prototype._emitPendingText = function () {
+    if (this.pendingText !== null) {
+        this.handlers.text(this.pendingText);
+        this.pendingText = null;
+    }
+};

package/lib/simple_api/tokenizer_proxy.js

@@ -0,0 +1,119 @@
+var Tokenizer = require('../tokenization/tokenizer'),
+    ForeignContent = require('../common/foreign_content'),
+    UNICODE = require('../common/unicode'),
+    HTML = require('../common/html');
+
+//Aliases
+var $ = HTML.TAG_NAMES,
+    NS = HTML.NAMESPACES;
+
+
+//Tokenizer proxy
+//NOTE: this proxy simulates adjustment of the Tokenizer which performed by standard parser during tree construction.
+var TokenizerProxy = module.exports = function (html) {
+    this.tokenizer = new Tokenizer(html);
+
+    this.namespaceStack = [];
+    this.namespaceStackTop = -1;
+    this.currentNamespace = null;
+    this.inForeignContent = false;
+};
+
+//API
+TokenizerProxy.prototype.getNextToken = function () {
+    var token = this.tokenizer.getNextToken();
+
+    if (token.type === Tokenizer.START_TAG_TOKEN)
+        this._handleStartTagToken(token);
+
+    else if (token.type === Tokenizer.END_TAG_TOKEN)
+        this._handleEndTagToken(token);
+
+    else if (token.type === Tokenizer.NULL_CHARACTER_TOKEN && this.inForeignContent) {
+        token.type = Tokenizer.CHARACTER_TOKEN;
+        token.chars = UNICODE.REPLACEMENT_CHARACTER;
+    }
+
+    return token;
+};
+
+//Namespace stack mutations
+TokenizerProxy.prototype._enterNamespace = function (namespace) {
+    this.namespaceStackTop++;
+    this.namespaceStack.push(namespace);
+
+    this.inForeignContent = namespace !== NS.HTML;
+    this.currentNamespace = namespace;
+    this.tokenizer.allowCDATA = this.inForeignContent;
+};
+
+TokenizerProxy.prototype._leaveCurrentNamespace = function () {
+    this.namespaceStackTop--;
+    this.namespaceStack.pop();
+
+    this.currentNamespace = this.namespaceStack[this.namespaceStackTop];
+    this.inForeignContent = this.currentNamespace !== NS.HTML;
+    this.tokenizer.allowCDATA = this.inForeignContent;
+};
+
+//Token handlers
+TokenizerProxy.prototype._ensureTokenizerState = function (tn) {
+    if (tn === $.TEXTAREA || tn === $.TITLE)
+        this.tokenizer.state = Tokenizer.RCDATA_STATE;
+
+    else if (tn === $.PLAINTEXT)
+        this.tokenizer.state = Tokenizer.PLAINTEXT_STATE;
+
+    else if (tn === $.SCRIPT)
+        this.tokenizer.state = Tokenizer.SCRIPT_DATA_STATE;
+
+    else if (tn === $.STYLE || tn === $.IFRAME || tn === $.XMP ||
+             tn === $.NOEMBED || tn === $.NOFRAMES || tn === $.NOSCRIPT) {
+        this.tokenizer.state = Tokenizer.RAWTEXT_STATE;
+    }
+};
+
+TokenizerProxy.prototype._handleStartTagToken = function (token) {
+    var tn = token.tagName;
+
+    if (tn === $.SVG)
+        this._enterNamespace(NS.SVG);
+
+    else if (tn === $.MATH)
+        this._enterNamespace(NS.MATHML);
+
+    else {
+        if (this.inForeignContent) {
+            if (ForeignContent.causesExit(token))
+                this._leaveCurrentNamespace();
+
+            else if (ForeignContent.isMathMLTextIntegrationPoint(tn, this.currentNamespace) ||
+                     ForeignContent.isHtmlIntegrationPoint(tn, this.currentNamespace, token.attrs)) {
+                this._enterNamespace(NS.HTML);
+            }
+        }
+
+        else
+            this._ensureTokenizerState(tn);
+    }
+};
+
+TokenizerProxy.prototype._handleEndTagToken = function (token) {
+    var tn = token.tagName;
+
+    if (!this.inForeignContent) {
+        var previousNs = this.namespaceStack[this.namespaceStackTop - 1];
+
+        //NOTE: check for exit from integration point
+        if (ForeignContent.isMathMLTextIntegrationPoint(tn, previousNs) ||
+            ForeignContent.isHtmlIntegrationPoint(tn, previousNs, token.attrs)) {
+            this._leaveCurrentNamespace();
+        }
+
+        else if (tn === $.SCRIPT)
+            this.tokenizer.state = Tokenizer.DATA_STATE;
+    }
+
+    else if ((tn === $.SVG && this.currentNamespace === NS.SVG) || (tn === $.MATH && this.currentNamespace === NS.MATHML))
+        this._leaveCurrentNamespace();
+};

package/lib/{tokenization_stage → tokenization}/preprocessor.js

@@ -1,4 +1,4 @@
-var UNICODE = require('../const/unicode');
+var UNICODE = require('../common/unicode');
 
 //Aliases
 var $ = UNICODE.CODE_POINTS;
@@ -80,4 +80,4 @@ Preprocessor.prototype.retreat = function () {
     }
 
     this.pos--;
-};
+};

package/lib/{tokenization_stage → tokenization}/tokenizer.js

@@ -1,5 +1,5 @@
 var Preprocessor = require('./preprocessor'),
-    UNICODE = require('../const/unicode'),
+    UNICODE = require('../common/unicode'),
     NAMED_ENTITY_TRIE = require('./named_entity_trie');
 
 //Aliases
@@ -171,11 +171,22 @@ Tokenizer.DOCTYPE_TOKEN = 'DOCTYPE_TOKEN';
 Tokenizer.EOF_TOKEN = 'EOF_TOKEN';
 
 //States export
+Tokenizer.DATA_STATE = DATA_STATE;
 Tokenizer.RCDATA_STATE = RCDATA_STATE;
 Tokenizer.RAWTEXT_STATE = RAWTEXT_STATE;
 Tokenizer.SCRIPT_DATA_STATE = SCRIPT_DATA_STATE;
 Tokenizer.PLAINTEXT_STATE = PLAINTEXT_STATE;
 
+//Static
+Tokenizer.getTokenAttr = function (token, attrName) {
+    for (var i = token.attrs.length - 1; i >= 0; i--) {
+        if (token.attrs[i].name === attrName)
+            return token.attrs[i].value;
+    }
+
+    return null;
+};
+
 //Get token
 Tokenizer.prototype.getNextToken = function () {
     while (!this.tokenQueue.length)
@@ -310,14 +321,7 @@ Tokenizer.prototype._createAttr = function (attrNameFirstCh) {
 };
 
 Tokenizer.prototype._isDuplicateAttr = function () {
-    var attrs = this.currentToken.attrs;
-
-    for (var i = 0; i < attrs.length; i++) {
-        if (attrs[i].name === this.currentAttr.name)
-            return true;
-    }
-
-    return false;
+    return Tokenizer.getTokenAttr(this.currentToken, this.currentAttr.name) !== null;
 };
 
 Tokenizer.prototype._leaveAttrName = function (toState) {
@@ -2293,4 +2297,4 @@ _[CDATA_SECTION_STATE] = function cdataSectionState(cp) {
             cp = this._consume();
         }
     }
-};
+};

package/lib/tree_adapters/default.js

@@ -195,4 +195,4 @@ exports.isDocumentTypeNode = function (node) {
 
 exports.isElementNode = function (node) {
     return !!node.tagName;
-};
+};

package/lib/tree_adapters/htmlparser2.js

@@ -264,4 +264,4 @@ exports.isDocumentTypeNode = function (node) {
 
 exports.isElementNode = function (node) {
     return !!node.attribs;
-};
+};

package/lib/tree_construction/doctype.js

@@ -0,0 +1,107 @@
+//Const
+var VALID_DOCTYPE_NAME = 'html',
+    QUIRKS_MODE_SYSTEM_ID = 'http://www.ibm.com/data/dtd/v11/ibmxhtml1-transitional.dtd',
+    QUIRKS_MODE_PUBLIC_ID_PREFIXES = [
+        "+//silmaril//dtd html pro v0r11 19970101//en",
+        "-//advasoft ltd//dtd html 3.0 aswedit + extensions//en",
+        "-//as//dtd html 3.0 aswedit + extensions//en",
+        "-//ietf//dtd html 2.0 level 1//en",
+        "-//ietf//dtd html 2.0 level 2//en",
+        "-//ietf//dtd html 2.0 strict level 1//en",
+        "-//ietf//dtd html 2.0 strict level 2//en",
+        "-//ietf//dtd html 2.0 strict//en",
+        "-//ietf//dtd html 2.0//en",
+        "-//ietf//dtd html 2.1e//en",
+        "-//ietf//dtd html 3.0//en",
+        "-//ietf//dtd html 3.0//en//",
+        "-//ietf//dtd html 3.2 final//en",
+        "-//ietf//dtd html 3.2//en",
+        "-//ietf//dtd html 3//en",
+        "-//ietf//dtd html level 0//en",
+        "-//ietf//dtd html level 0//en//2.0",
+        "-//ietf//dtd html level 1//en",
+        "-//ietf//dtd html level 1//en//2.0",
+        "-//ietf//dtd html level 2//en",
+        "-//ietf//dtd html level 2//en//2.0",
+        "-//ietf//dtd html level 3//en",
+        "-//ietf//dtd html level 3//en//3.0",
+        "-//ietf//dtd html strict level 0//en",
+        "-//ietf//dtd html strict level 0//en//2.0",
+        "-//ietf//dtd html strict level 1//en",
+        "-//ietf//dtd html strict level 1//en//2.0",
+        "-//ietf//dtd html strict level 2//en",
+        "-//ietf//dtd html strict level 2//en//2.0",
+        "-//ietf//dtd html strict level 3//en",
+        "-//ietf//dtd html strict level 3//en//3.0",
+        "-//ietf//dtd html strict//en",
+        "-//ietf//dtd html strict//en//2.0",
+        "-//ietf//dtd html strict//en//3.0",
+        "-//ietf//dtd html//en",
+        "-//ietf//dtd html//en//2.0",
+        "-//ietf//dtd html//en//3.0",
+        "-//metrius//dtd metrius presentational//en",
+        "-//microsoft//dtd internet explorer 2.0 html strict//en",
+        "-//microsoft//dtd internet explorer 2.0 html//en",
+        "-//microsoft//dtd internet explorer 2.0 tables//en",
+        "-//microsoft//dtd internet explorer 3.0 html strict//en",
+        "-//microsoft//dtd internet explorer 3.0 html//en",
+        "-//microsoft//dtd internet explorer 3.0 tables//en",
+        "-//netscape comm. corp.//dtd html//en",
+        "-//netscape comm. corp.//dtd strict html//en",
+        "-//o'reilly and associates//dtd html 2.0//en",
+        "-//o'reilly and associates//dtd html extended 1.0//en",
+        "-//spyglass//dtd html 2.0 extended//en",
+        "-//sq//dtd html 2.0 hotmetal + extensions//en",
+        "-//sun microsystems corp.//dtd hotjava html//en",
+        "-//sun microsystems corp.//dtd hotjava strict html//en",
+        "-//w3c//dtd html 3 1995-03-24//en",
+        "-//w3c//dtd html 3.2 draft//en",
+        "-//w3c//dtd html 3.2 final//en",
+        "-//w3c//dtd html 3.2//en",
+        "-//w3c//dtd html 3.2s draft//en",
+        "-//w3c//dtd html 4.0 frameset//en",
+        "-//w3c//dtd html 4.0 transitional//en",
+        "-//w3c//dtd html experimental 19960712//en",
+        "-//w3c//dtd html experimental 970421//en",
+        "-//w3c//dtd w3 html//en",
+        "-//w3o//dtd w3 html 3.0//en",
+        "-//w3o//dtd w3 html 3.0//en//",
+        "-//webtechs//dtd mozilla html 2.0//en",
+        "-//webtechs//dtd mozilla html//en"
+    ],
+    QUIRKS_MODE_NO_SYSTEM_ID_PUBLIC_ID_PREFIXES = [
+        '-//w3c//dtd html 4.01 frameset//',
+        '-//w3c//dtd html 4.01 transitional//'
+    ],
+    QUIRKS_MODE_PUBLIC_IDS = [
+        '-//w3o//dtd w3 html strict 3.0//en//',
+        '-/w3c/dtd html 4.0 transitional/en',
+        'html'
+    ];
+
+exports.isQuirks = function (name, publicId, systemId) {
+    if (name !== VALID_DOCTYPE_NAME)
+        return true;
+
+    if (systemId && systemId.toLowerCase() === QUIRKS_MODE_SYSTEM_ID)
+        return true;
+
+    if (publicId !== null) {
+        publicId = publicId.toLowerCase();
+
+        if (QUIRKS_MODE_PUBLIC_IDS.indexOf(publicId) > -1)
+            return true;
+
+        var prefixes = QUIRKS_MODE_PUBLIC_ID_PREFIXES;
+
+        if (systemId === null)
+            prefixes = prefixes.concat(QUIRKS_MODE_NO_SYSTEM_ID_PUBLIC_ID_PREFIXES);
+
+        for (var i = 0; i < prefixes.length; i++) {
+            if (publicId.indexOf(prefixes[i]) === 0)
+                return true;
+        }
+    }
+
+    return false;
+};

package/lib/{tree_construction_stage → tree_construction}/open_element_stack.js

@@ -1,4 +1,4 @@
-var HTML = require('../const/html');
+var HTML = require('../common/html');
 
 //Aliases
 var $ = HTML.TAG_NAMES,