parse5 2.2.3 → 3.0.3

package/lib/parser/parser_stream.js

@@ -0,0 +1,76 @@
+'use strict';
+
+var WritableStream = require('stream').Writable,
+    inherits = require('util').inherits,
+    Parser = require('./index');
+
+var ParserStream = module.exports = function (options) {
+    WritableStream.call(this);
+
+    this.parser = new Parser(options);
+
+    this.lastChunkWritten = false;
+    this.writeCallback = null;
+    this.pausedByScript = false;
+
+    this.document = this.parser.treeAdapter.createDocument();
+
+    this.pendingHtmlInsertions = [];
+
+    this._resume = this._resume.bind(this);
+    this._documentWrite = this._documentWrite.bind(this);
+    this._scriptHandler = this._scriptHandler.bind(this);
+
+    this.parser._bootstrap(this.document, null);
+};
+
+inherits(ParserStream, WritableStream);
+
+//WritableStream implementation
+ParserStream.prototype._write = function (chunk, encoding, callback) {
+    this.writeCallback = callback;
+    this.parser.tokenizer.write(chunk.toString('utf8'), this.lastChunkWritten);
+    this._runParsingLoop();
+};
+
+ParserStream.prototype.end = function (chunk, encoding, callback) {
+    this.lastChunkWritten = true;
+    WritableStream.prototype.end.call(this, chunk || '', encoding, callback);
+};
+
+//Scriptable parser implementation
+ParserStream.prototype._runParsingLoop = function () {
+    this.parser.runParsingLoopForCurrentChunk(this.writeCallback, this._scriptHandler);
+};
+
+ParserStream.prototype._resume = function () {
+    if (!this.pausedByScript)
+        throw new Error('Parser was already resumed');
+
+    while (this.pendingHtmlInsertions.length) {
+        var html = this.pendingHtmlInsertions.pop();
+
+        this.parser.tokenizer.insertHtmlAtCurrentPos(html);
+    }
+
+    this.pausedByScript = false;
+
+    //NOTE: keep parsing if we don't wait for the next input chunk
+    if (this.parser.tokenizer.active)
+        this._runParsingLoop();
+};
+
+ParserStream.prototype._documentWrite = function (html) {
+    if (!this.parser.stopped)
+        this.pendingHtmlInsertions.push(html);
+};
+
+ParserStream.prototype._scriptHandler = function (scriptElement) {
+    if (this.listeners('script').length) {
+        this.pausedByScript = true;
+        this.emit('script', scriptElement, this._documentWrite, this._resume);
+    }
+    else
+        this._runParsingLoop();
+};
+

package/lib/parser/plain_text_conversion_stream.js

@@ -0,0 +1,20 @@
+'use strict';
+
+var ParserStream = require('./parser_stream'),
+    inherits = require('util').inherits,
+    $ = require('../common/html').TAG_NAMES;
+
+var PlainTextConversionStream = module.exports = function (options) {
+    ParserStream.call(this, options);
+
+    // NOTE: see https://html.spec.whatwg.org/#read-text
+    this.parser._insertFakeElement($.HTML);
+    this.parser._insertFakeElement($.HEAD);
+    this.parser.openElements.pop();
+    this.parser._insertFakeElement($.BODY);
+    this.parser._insertFakeElement($.PRE);
+    this.parser.treeAdapter.insertText(this.parser.openElements.current, '\n');
+    this.parser.switchToPlaintextParsing();
+};
+
+inherits(PlainTextConversionStream, ParserStream);

package/lib/sax/dev_null_stream.js

@@ -1,14 +1,14 @@
-'use strict';
-
-var WritableStream = require('stream').Writable,
-    util = require('util');
-
-var DevNullStream = module.exports = function () {
-    WritableStream.call(this);
-};
-
-util.inherits(DevNullStream, WritableStream);
-
-DevNullStream.prototype._write = function (chunk, encoding, cb) {
-    cb();
-};
+'use strict';
+
+var WritableStream = require('stream').Writable,
+    util = require('util');
+
+var DevNullStream = module.exports = function () {
+    WritableStream.call(this);
+};
+
+util.inherits(DevNullStream, WritableStream);
+
+DevNullStream.prototype._write = function (chunk, encoding, cb) {
+    cb();
+};

package/lib/sax/index.js

@@ -4,55 +4,24 @@ var TransformStream = require('stream').Transform,
     DevNullStream = require('./dev_null_stream'),
     inherits = require('util').inherits,
     Tokenizer = require('../tokenizer'),
+    LocationInfoTokenizerMixin = require('../extensions/location_info/tokenizer_mixin'),
     ParserFeedbackSimulator = require('./parser_feedback_simulator'),
-    mergeOptions = require('../common/merge_options');
-
-/**
- * @typedef {Object} SAXParserOptions
- *
- * @property {Boolean} [locationInfo=false] - Enables source code location information for the tokens.
- * When enabled, each token event handler will receive {@link LocationInfo} (or {@link StartTagLocationInfo})
- * object as its last argument.
- */
+    mergeOptions = require('../utils/merge_options');
+
 var DEFAULT_OPTIONS = {
     locationInfo: false
 };
 
-/**
- * Streaming [SAX]{@link https://en.wikipedia.org/wiki/Simple_API_for_XML}-style HTML parser.
- * A [transform stream](https://nodejs.org/api/stream.html#stream_class_stream_transform)
- * (which means you can pipe *through* it, see example).
- * @class SAXParser
- * @memberof parse5
- * @instance
- * @extends stream.Transform
- * @param {SAXParserOptions} options - Parsing options.
- * @example
- * var parse5 = require('parse5');
- * var http = require('http');
- * var fs = require('fs');
- *
- * var file = fs.createWriteStream('/home/google.com.html');
- * var parser = new parse5.SAXParser();
- *
- * parser.on('text', function(text) {
- *  // Handle page text content
- *  ...
- * });
- *
- * http.get('http://google.com', function(res) {
- *  // SAXParser is the Transform stream, which means you can pipe
- *  // through it. So, you can analyze page content and, e.g., save it
- *  // to the file at the same time:
- *  res.pipe(parser).pipe(file);
- * });
- */
 var SAXParser = module.exports = function (options) {
     TransformStream.call(this);
 
     this.options = mergeOptions(DEFAULT_OPTIONS, options);
 
     this.tokenizer = new Tokenizer(options);
+
+    if (this.options.locationInfo)
+        new LocationInfoTokenizerMixin(this.tokenizer);
+
     this.parserFeedbackSimulator = new ParserFeedbackSimulator(this.tokenizer);
 
     this.pendingText = null;
@@ -90,33 +59,6 @@ SAXParser.prototype.end = function (chunk, encoding, callback) {
     TransformStream.prototype.end.call(this, chunk, encoding, callback);
 };
 
-/**
- * Stops parsing. Useful if you want the parser to stop consuming CPU time once you've obtained the desired info
- * from the input stream. Doesn't prevent piping, so that data will flow through the parser as usual.
- *
- * @function stop
- * @memberof parse5#SAXParser
- * @instance
- * @example
- * var parse5 = require('parse5');
- * var http = require('http');
- * var fs = require('fs');
- *
- * var file = fs.createWriteStream('/home/google.com.html');
- * var parser = new parse5.SAXParser();
- *
- * parser.on('doctype', function(name, publicId, systemId) {
- *  // Process doctype info ans stop parsing
- *  ...
- *  parser.stop();
- * });
- *
- * http.get('http://google.com', function(res) {
- *  // Despite the fact that parser.stop() was called whole
- *  // content of the page will be written to the file
- *  res.pipe(parser).pipe(file);
- * });
- */
 SAXParser.prototype.stop = function () {
     this.stopped = true;
 };
@@ -156,74 +98,20 @@ SAXParser.prototype._handleToken = function (token) {
         this.currentTokenLocation = token.location;
 
     if (token.type === Tokenizer.START_TAG_TOKEN)
-        /**
-         * Raised when the parser encounters a start tag.
-         * @event startTag
-         * @memberof parse5#SAXParser
-         * @instance
-         * @type {Function}
-         * @param {String} name - Tag name.
-         * @param {Array} attrs - List of attributes in the `{ name: String, value: String, prefix?: String }` form.
-         * @param {Boolean} selfClosing - Indicates if the tag is self-closing.
-         * @param {StartTagLocationInfo} [location] - Start tag source code location info.
-         * Available if location info is enabled in {@link SAXParserOptions}.
-         */
         this.emit('startTag', token.tagName, token.attrs, token.selfClosing, this.currentTokenLocation);
 
     else if (token.type === Tokenizer.END_TAG_TOKEN)
-        /**
-         * Raised then parser encounters an end tag.
-         * @event endTag
-         * @memberof parse5#SAXParser
-         * @instance
-         * @type {Function}
-         * @param {String} name - Tag name.
-         * @param {LocationInfo} [location] - End tag source code location info.
-         * Available if location info is enabled in {@link SAXParserOptions}.
-         */
         this.emit('endTag', token.tagName, this.currentTokenLocation);
 
     else if (token.type === Tokenizer.COMMENT_TOKEN)
-        /**
-         * Raised then parser encounters a comment.
-         * @event comment
-         * @memberof parse5#SAXParser
-         * @instance
-         * @type {Function}
-         * @param {String} text - Comment text.
-         * @param {LocationInfo} [location] - Comment source code location info.
-         * Available if location info is enabled in {@link SAXParserOptions}.
-         */
         this.emit('comment', token.data, this.currentTokenLocation);
 
     else if (token.type === Tokenizer.DOCTYPE_TOKEN)
-        /**
-         * Raised then parser encounters a [document type declaration]{@link https://en.wikipedia.org/wiki/Document_type_declaration}.
-         * @event doctype
-         * @memberof parse5#SAXParser
-         * @instance
-         * @type {Function}
-         * @param {String} name - Document type name.
-         * @param {String} publicId - Document type public identifier.
-         * @param {String} systemId - Document type system identifier.
-         * @param {LocationInfo} [location] - Document type declaration source code location info.
-         * Available if location info is enabled in {@link SAXParserOptions}.
-         */
         this.emit('doctype', token.name, token.publicId, token.systemId, this.currentTokenLocation);
 };
 
 SAXParser.prototype._emitPendingText = function () {
     if (this.pendingText !== null) {
-        /**
-         * Raised then parser encounters text content.
-         * @event text
-         * @memberof parse5#SAXParser
-         * @instance
-         * @type {Function}
-         * @param {String} text - Text content.
-         * @param {LocationInfo} [location] - Text content code location info.
-         * Available if location info is enabled in {@link SAXParserOptions}.
-         */
         this.emit('text', this.pendingText, this.currentTokenLocation);
         this.pendingText = null;
     }

package/lib/serializer/index.js

@@ -1,180 +1,162 @@
-'use strict';
-
-var defaultTreeAdapter = require('../tree_adapters/default'),
-    doctype = require('../common/doctype'),
-    mergeOptions = require('../common/merge_options'),
-    HTML = require('../common/html');
-
-//Aliases
-var $ = HTML.TAG_NAMES,
-    NS = HTML.NAMESPACES;
-
-//Default serializer options
-/**
- * @typedef {Object} SerializerOptions
- *
- * @property {TreeAdapter} [treeAdapter=parse5.treeAdapters.default] - Specifies input tree format.
- */
-var DEFAULT_OPTIONS = {
-    treeAdapter: defaultTreeAdapter
-};
-
-//Escaping regexes
-var AMP_REGEX = /&/g,
-    NBSP_REGEX = /\u00a0/g,
-    DOUBLE_QUOTE_REGEX = /"/g,
-    LT_REGEX = /</g,
-    GT_REGEX = />/g;
-
-//Serializer
-var Serializer = module.exports = function (node, options) {
-    this.options = mergeOptions(DEFAULT_OPTIONS, options);
-    this.treeAdapter = this.options.treeAdapter;
-
-    this.html = '';
-    this.startNode = node;
-};
-
-// NOTE: exported as static method for the testing purposes
-Serializer.escapeString = function (str, attrMode) {
-    str = str
-        .replace(AMP_REGEX, '&amp;')
-        .replace(NBSP_REGEX, '&nbsp;');
-
-    if (attrMode)
-        str = str.replace(DOUBLE_QUOTE_REGEX, '&quot;');
-
-    else {
-        str = str
-            .replace(LT_REGEX, '&lt;')
-            .replace(GT_REGEX, '&gt;');
-    }
-
-    return str;
-};
-
-
-//API
-Serializer.prototype.serialize = function () {
-    this._serializeChildNodes(this.startNode);
-
-    return this.html;
-};
-
-
-//Internals
-Serializer.prototype._serializeChildNodes = function (parentNode) {
-    var childNodes = this.treeAdapter.getChildNodes(parentNode);
-
-    if (childNodes) {
-        for (var i = 0, cnLength = childNodes.length; i < cnLength; i++) {
-            var currentNode = childNodes[i];
-
-            if (this.treeAdapter.isElementNode(currentNode))
-                this._serializeElement(currentNode);
-
-            else if (this.treeAdapter.isTextNode(currentNode))
-                this._serializeTextNode(currentNode);
-
-            else if (this.treeAdapter.isCommentNode(currentNode))
-                this._serializeCommentNode(currentNode);
-
-            else if (this.treeAdapter.isDocumentTypeNode(currentNode))
-                this._serializeDocumentTypeNode(currentNode);
-        }
-    }
-};
-
-Serializer.prototype._serializeElement = function (node) {
-    var tn = this.treeAdapter.getTagName(node),
-        ns = this.treeAdapter.getNamespaceURI(node);
-
-    this.html += '<' + tn;
-    this._serializeAttributes(node);
-    this.html += '>';
-
-    if (tn !== $.AREA && tn !== $.BASE && tn !== $.BASEFONT && tn !== $.BGSOUND && tn !== $.BR && tn !== $.BR &&
-        tn !== $.COL && tn !== $.EMBED && tn !== $.FRAME && tn !== $.HR && tn !== $.IMG && tn !== $.INPUT &&
-        tn !== $.KEYGEN && tn !== $.LINK && tn !== $.MENUITEM && tn !== $.META && tn !== $.PARAM && tn !== $.SOURCE &&
-        tn !== $.TRACK && tn !== $.WBR) {
-
-        if (tn === $.PRE || tn === $.TEXTAREA || tn === $.LISTING) {
-            var firstChild = this.treeAdapter.getFirstChild(node);
-
-            if (firstChild && this.treeAdapter.isTextNode(firstChild)) {
-                var content = this.treeAdapter.getTextNodeContent(firstChild);
-
-                if (content[0] === '\n')
-                    this.html += '\n';
-            }
-        }
-
-        var childNodesHolder = tn === $.TEMPLATE && ns === NS.HTML ?
-            this.treeAdapter.getTemplateContent(node) :
-            node;
-
-        this._serializeChildNodes(childNodesHolder);
-        this.html += '</' + tn + '>';
-    }
-};
-
-Serializer.prototype._serializeAttributes = function (node) {
-    var attrs = this.treeAdapter.getAttrList(node);
-
-    for (var i = 0, attrsLength = attrs.length; i < attrsLength; i++) {
-        var attr = attrs[i],
-            value = Serializer.escapeString(attr.value, true);
-
-        this.html += ' ';
-
-        if (!attr.namespace)
-            this.html += attr.name;
-
-        else if (attr.namespace === NS.XML)
-            this.html += 'xml:' + attr.name;
-
-        else if (attr.namespace === NS.XMLNS) {
-            if (attr.name !== 'xmlns')
-                this.html += 'xmlns:';
-
-            this.html += attr.name;
-        }
-
-        else if (attr.namespace === NS.XLINK)
-            this.html += 'xlink:' + attr.name;
-
-        else
-            this.html += attr.namespace + ':' + attr.name;
-
-        this.html += '="' + value + '"';
-    }
-};
-
-Serializer.prototype._serializeTextNode = function (node) {
-    var content = this.treeAdapter.getTextNodeContent(node),
-        parent = this.treeAdapter.getParentNode(node),
-        parentTn = void 0;
-
-    if (parent && this.treeAdapter.isElementNode(parent))
-        parentTn = this.treeAdapter.getTagName(parent);
-
-    if (parentTn === $.STYLE || parentTn === $.SCRIPT || parentTn === $.XMP || parentTn === $.IFRAME ||
-        parentTn === $.NOEMBED || parentTn === $.NOFRAMES || parentTn === $.PLAINTEXT || parentTn === $.NOSCRIPT)
-
-        this.html += content;
-
-    else
-        this.html += Serializer.escapeString(content, false);
-};
-
-Serializer.prototype._serializeCommentNode = function (node) {
-    this.html += '<!--' + this.treeAdapter.getCommentNodeContent(node) + '-->';
-};
-
-Serializer.prototype._serializeDocumentTypeNode = function (node) {
-    var name = this.treeAdapter.getDocumentTypeNodeName(node),
-        publicId = this.treeAdapter.getDocumentTypeNodePublicId(node),
-        systemId = this.treeAdapter.getDocumentTypeNodeSystemId(node);
-
-    this.html += '<' + doctype.serializeContent(name, publicId, systemId) + '>';
-};
+'use strict';
+
+var defaultTreeAdapter = require('../tree_adapters/default'),
+    mergeOptions = require('../utils/merge_options'),
+    doctype = require('../common/doctype'),
+    HTML = require('../common/html');
+
+//Aliases
+var $ = HTML.TAG_NAMES,
+    NS = HTML.NAMESPACES;
+
+//Default serializer options
+var DEFAULT_OPTIONS = {
+    treeAdapter: defaultTreeAdapter
+};
+
+//Escaping regexes
+var AMP_REGEX = /&/g,
+    NBSP_REGEX = /\u00a0/g,
+    DOUBLE_QUOTE_REGEX = /"/g,
+    LT_REGEX = /</g,
+    GT_REGEX = />/g;
+
+//Serializer
+var Serializer = module.exports = function (node, options) {
+    this.options = mergeOptions(DEFAULT_OPTIONS, options);
+    this.treeAdapter = this.options.treeAdapter;
+
+    this.html = '';
+    this.startNode = node;
+};
+
+// NOTE: exported as static method for the testing purposes
+Serializer.escapeString = function (str, attrMode) {
+    str = str
+        .replace(AMP_REGEX, '&amp;')
+        .replace(NBSP_REGEX, '&nbsp;');
+
+    if (attrMode)
+        str = str.replace(DOUBLE_QUOTE_REGEX, '&quot;');
+
+    else {
+        str = str
+            .replace(LT_REGEX, '&lt;')
+            .replace(GT_REGEX, '&gt;');
+    }
+
+    return str;
+};
+
+
+//API
+Serializer.prototype.serialize = function () {
+    this._serializeChildNodes(this.startNode);
+
+    return this.html;
+};
+
+
+//Internals
+Serializer.prototype._serializeChildNodes = function (parentNode) {
+    var childNodes = this.treeAdapter.getChildNodes(parentNode);
+
+    if (childNodes) {
+        for (var i = 0, cnLength = childNodes.length; i < cnLength; i++) {
+            var currentNode = childNodes[i];
+
+            if (this.treeAdapter.isElementNode(currentNode))
+                this._serializeElement(currentNode);
+
+            else if (this.treeAdapter.isTextNode(currentNode))
+                this._serializeTextNode(currentNode);
+
+            else if (this.treeAdapter.isCommentNode(currentNode))
+                this._serializeCommentNode(currentNode);
+
+            else if (this.treeAdapter.isDocumentTypeNode(currentNode))
+                this._serializeDocumentTypeNode(currentNode);
+        }
+    }
+};
+
+Serializer.prototype._serializeElement = function (node) {
+    var tn = this.treeAdapter.getTagName(node),
+        ns = this.treeAdapter.getNamespaceURI(node);
+
+    this.html += '<' + tn;
+    this._serializeAttributes(node);
+    this.html += '>';
+
+    if (tn !== $.AREA && tn !== $.BASE && tn !== $.BASEFONT && tn !== $.BGSOUND && tn !== $.BR && tn !== $.BR &&
+        tn !== $.COL && tn !== $.EMBED && tn !== $.FRAME && tn !== $.HR && tn !== $.IMG && tn !== $.INPUT &&
+        tn !== $.KEYGEN && tn !== $.LINK && tn !== $.MENUITEM && tn !== $.META && tn !== $.PARAM && tn !== $.SOURCE &&
+        tn !== $.TRACK && tn !== $.WBR) {
+
+        var childNodesHolder = tn === $.TEMPLATE && ns === NS.HTML ?
+            this.treeAdapter.getTemplateContent(node) :
+            node;
+
+        this._serializeChildNodes(childNodesHolder);
+        this.html += '</' + tn + '>';
+    }
+};
+
+Serializer.prototype._serializeAttributes = function (node) {
+    var attrs = this.treeAdapter.getAttrList(node);
+
+    for (var i = 0, attrsLength = attrs.length; i < attrsLength; i++) {
+        var attr = attrs[i],
+            value = Serializer.escapeString(attr.value, true);
+
+        this.html += ' ';
+
+        if (!attr.namespace)
+            this.html += attr.name;
+
+        else if (attr.namespace === NS.XML)
+            this.html += 'xml:' + attr.name;
+
+        else if (attr.namespace === NS.XMLNS) {
+            if (attr.name !== 'xmlns')
+                this.html += 'xmlns:';
+
+            this.html += attr.name;
+        }
+
+        else if (attr.namespace === NS.XLINK)
+            this.html += 'xlink:' + attr.name;
+
+        else
+            this.html += attr.namespace + ':' + attr.name;
+
+        this.html += '="' + value + '"';
+    }
+};
+
+Serializer.prototype._serializeTextNode = function (node) {
+    var content = this.treeAdapter.getTextNodeContent(node),
+        parent = this.treeAdapter.getParentNode(node),
+        parentTn = void 0;
+
+    if (parent && this.treeAdapter.isElementNode(parent))
+        parentTn = this.treeAdapter.getTagName(parent);
+
+    if (parentTn === $.STYLE || parentTn === $.SCRIPT || parentTn === $.XMP || parentTn === $.IFRAME ||
+        parentTn === $.NOEMBED || parentTn === $.NOFRAMES || parentTn === $.PLAINTEXT || parentTn === $.NOSCRIPT)
+
+        this.html += content;
+
+    else
+        this.html += Serializer.escapeString(content, false);
+};
+
+Serializer.prototype._serializeCommentNode = function (node) {
+    this.html += '<!--' + this.treeAdapter.getCommentNodeContent(node) + '-->';
+};
+
+Serializer.prototype._serializeDocumentTypeNode = function (node) {
+    var name = this.treeAdapter.getDocumentTypeNodeName(node);
+
+    this.html += '<' + doctype.serializeContent(name, null, null) + '>';
+};