parse5 3.0.2 → 3.0.3

package/lib/extensions/location_info/open_element_stack_mixin.js

@@ -0,0 +1,34 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    inherits = require('util').inherits;
+
+var LocationInfoOpenElementStackMixin = module.exports = function (stack, options) {
+    Mixin.call(this, stack);
+
+    this.onItemPop = options.onItemPop;
+};
+
+inherits(LocationInfoOpenElementStackMixin, Mixin);
+
+LocationInfoOpenElementStackMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        pop: function () {
+            mxn.onItemPop(this.current);
+            orig.pop.call(this);
+        },
+
+        popAllUpToHtmlElement: function () {
+            for (var i = this.stackTop; i > 0; i--)
+                mxn.onItemPop(this.items[i]);
+
+            orig.popAllUpToHtmlElement.call(this);
+        },
+
+        remove: function (element) {
+            mxn.onItemPop(this.current);
+            orig.remove.call(this, element);
+        }
+    };
+};
+

package/lib/extensions/location_info/parser_mixin.js

@@ -0,0 +1,213 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    Tokenizer = require('../../tokenizer'),
+    LocationInfoTokenizerMixin = require('./tokenizer_mixin'),
+    PositionTrackingPreprocessorMixin = require('../position_tracking/preprocessor_mixin'),
+    LocationInfoOpenElementStackMixin = require('./open_element_stack_mixin'),
+    HTML = require('../../common/html'),
+    inherits = require('util').inherits;
+
+
+//Aliases
+var $ = HTML.TAG_NAMES;
+
+var LocationInfoParserMixin = module.exports = function (parser) {
+    Mixin.call(this, parser);
+
+    this.parser = parser;
+    this.posTracker = null;
+    this.lastStartTagToken = null;
+    this.lastFosterParentingLocation = null;
+    this.currentToken = null;
+};
+
+inherits(LocationInfoParserMixin, Mixin);
+
+
+LocationInfoParserMixin.prototype._setStartLocation = function (element) {
+    if (this.lastStartTagToken) {
+        element.__location = Object.create(this.lastStartTagToken.location);
+        element.__location.startTag = this.lastStartTagToken.location;
+    }
+    else
+        element.__location = null;
+};
+
+LocationInfoParserMixin.prototype._setEndLocation = function (element, closingToken) {
+    var loc = element.__location;
+
+    if (loc) {
+        if (closingToken.location) {
+            var ctLoc = closingToken.location,
+                tn = this.parser.treeAdapter.getTagName(element);
+
+            // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing
+            // tag and for cases like <td> <p> </td> - 'p' closes without a closing tag.
+            var isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN && tn === closingToken.tagName;
+
+            if (isClosingEndTag) {
+                loc.endTag = Object.create(ctLoc);
+                loc.endOffset = ctLoc.endOffset;
+            }
+
+            else
+                loc.endOffset = ctLoc.startOffset;
+        }
+
+        else if (closingToken.type === Tokenizer.EOF_TOKEN)
+            loc.endOffset = this.posTracker.offset;
+    }
+};
+
+LocationInfoParserMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        _bootstrap: function (document, fragmentContext) {
+            orig._bootstrap.call(this, document, fragmentContext);
+
+            mxn.lastStartTagToken = null;
+            mxn.lastFosterParentingLocation = null;
+            mxn.currentToken = null;
+            mxn.posTracker = new PositionTrackingPreprocessorMixin(this.tokenizer.preprocessor);
+
+            new LocationInfoTokenizerMixin(this.tokenizer);
+
+            new LocationInfoOpenElementStackMixin(this.openElements, {
+                onItemPop: function (element) {
+                    mxn._setEndLocation(element, mxn.currentToken);
+                }
+            });
+        },
+
+        _runParsingLoop: function (scriptHandler) {
+            orig._runParsingLoop.call(this, scriptHandler);
+
+            // NOTE: generate location info for elements
+            // that remains on open element stack
+            for (var i = this.openElements.stackTop; i >= 0; i--)
+                mxn._setEndLocation(this.openElements.items[i], mxn.currentToken);
+        },
+
+
+        //Token processing
+        _processTokenInForeignContent: function (token) {
+            mxn.currentToken = token;
+            orig._processTokenInForeignContent.call(this, token);
+        },
+
+        _processToken: function (token) {
+            mxn.currentToken = token;
+            orig._processToken.call(this, token);
+
+            //NOTE: <body> and <html> are never popped from the stack, so we need to updated
+            //their end location explicitly.
+            var requireExplicitUpdate = token.type === Tokenizer.END_TAG_TOKEN &&
+                                        (token.tagName === $.HTML ||
+                                         token.tagName === $.BODY && this.openElements.hasInScope($.BODY));
+
+            if (requireExplicitUpdate) {
+                for (var i = this.openElements.stackTop; i >= 0; i--) {
+                    var element = this.openElements.items[i];
+
+                    if (this.treeAdapter.getTagName(element) === token.tagName) {
+                        mxn._setEndLocation(element, token);
+                        break;
+                    }
+                }
+            }
+        },
+
+
+        //Doctype
+        _setDocumentType: function (token) {
+            orig._setDocumentType.call(this, token);
+
+            var documentChildren = this.treeAdapter.getChildNodes(this.document),
+                cnLength = documentChildren.length;
+
+            for (var i = 0; i < cnLength; i++) {
+                var node = documentChildren[i];
+
+                if (this.treeAdapter.isDocumentTypeNode(node)) {
+                    node.__location = token.location;
+                    break;
+                }
+            }
+        },
+
+
+        //Elements
+        _attachElementToTree: function (element) {
+            //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
+            //So we will use token location stored in this methods for the element.
+            mxn._setStartLocation(element);
+            mxn.lastStartTagToken = null;
+            orig._attachElementToTree.call(this, element);
+        },
+
+        _appendElement: function (token, namespaceURI) {
+            mxn.lastStartTagToken = token;
+            orig._appendElement.call(this, token, namespaceURI);
+        },
+
+        _insertElement: function (token, namespaceURI) {
+            mxn.lastStartTagToken = token;
+            orig._insertElement.call(this, token, namespaceURI);
+        },
+
+        _insertTemplate: function (token) {
+            mxn.lastStartTagToken = token;
+            orig._insertTemplate.call(this, token);
+
+            var tmplContent = this.treeAdapter.getTemplateContent(this.openElements.current);
+
+            tmplContent.__location = null;
+        },
+
+        _insertFakeRootElement: function () {
+            orig._insertFakeRootElement.call(this);
+            this.openElements.current.__location = null;
+        },
+
+        //Comments
+        _appendCommentNode: function (token, parent) {
+            orig._appendCommentNode.call(this, token, parent);
+
+            var children = this.treeAdapter.getChildNodes(parent),
+                commentNode = children[children.length - 1];
+
+            commentNode.__location = token.location;
+        },
+
+        //Text
+        _findFosterParentingLocation: function () {
+            //NOTE: store last foster parenting location, so we will be able to find inserted text
+            //in case of foster parenting
+            mxn.lastFosterParentingLocation = orig._findFosterParentingLocation.call(this);
+
+            return mxn.lastFosterParentingLocation;
+        },
+
+        _insertCharacters: function (token) {
+            orig._insertCharacters.call(this, token);
+
+            var hasFosterParent = this._shouldFosterParentOnInsertion(),
+                parent = hasFosterParent && mxn.lastFosterParentingLocation.parent ||
+                         this.openElements.currentTmplContent ||
+                         this.openElements.current,
+                siblings = this.treeAdapter.getChildNodes(parent),
+                textNodeIdx = hasFosterParent && mxn.lastFosterParentingLocation.beforeElement ?
+                siblings.indexOf(mxn.lastFosterParentingLocation.beforeElement) - 1 :
+                siblings.length - 1,
+                textNode = siblings[textNodeIdx];
+
+            //NOTE: if we have location assigned by another token, then just update end position
+            if (textNode.__location)
+                textNode.__location.endOffset = token.location.endOffset;
+
+            else
+                textNode.__location = token.location;
+        }
+    };
+};
+

package/lib/extensions/location_info/tokenizer_mixin.js

@@ -0,0 +1,117 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    Tokenizer = require('../../tokenizer'),
+    PositionTrackingPreprocessorMixin = require('../position_tracking/preprocessor_mixin'),
+    inherits = require('util').inherits;
+
+var LocationInfoTokenizerMixin = module.exports = function (tokenizer) {
+    Mixin.call(this, tokenizer);
+
+    this.tokenizer = tokenizer;
+    this.posTracker = new PositionTrackingPreprocessorMixin(tokenizer.preprocessor);
+    this.currentAttrLocation = null;
+    this.currentTokenLocation = null;
+};
+
+inherits(LocationInfoTokenizerMixin, Mixin);
+
+LocationInfoTokenizerMixin.prototype._getCurrentLocation = function () {
+    return {
+        line: this.posTracker.line,
+        col: this.posTracker.col,
+        startOffset: this.posTracker.offset,
+        endOffset: -1
+    };
+};
+
+LocationInfoTokenizerMixin.prototype._attachCurrentAttrLocationInfo = function () {
+    this.currentAttrLocation.endOffset = this.posTracker.offset;
+
+    var currentToken = this.tokenizer.currentToken,
+        currentAttr = this.tokenizer.currentAttr;
+
+    if (!currentToken.location.attrs)
+        currentToken.location.attrs = Object.create(null);
+
+    currentToken.location.attrs[currentAttr.name] = this.currentAttrLocation;
+};
+
+LocationInfoTokenizerMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    var methods = {
+        _createStartTagToken: function () {
+            orig._createStartTagToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createEndTagToken: function () {
+            orig._createEndTagToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createCommentToken: function () {
+            orig._createCommentToken.call(this);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createDoctypeToken: function (initialName) {
+            orig._createDoctypeToken.call(this, initialName);
+            this.currentToken.location = mxn.currentTokenLocation;
+        },
+
+        _createCharacterToken: function (type, ch) {
+            orig._createCharacterToken.call(this, type, ch);
+            this.currentCharacterToken.location = mxn.currentTokenLocation;
+        },
+
+        _createAttr: function (attrNameFirstCh) {
+            orig._createAttr.call(this, attrNameFirstCh);
+            mxn.currentAttrLocation = mxn._getCurrentLocation();
+        },
+
+        _leaveAttrName: function (toState) {
+            orig._leaveAttrName.call(this, toState);
+            mxn._attachCurrentAttrLocationInfo();
+        },
+
+        _leaveAttrValue: function (toState) {
+            orig._leaveAttrValue.call(this, toState);
+            mxn._attachCurrentAttrLocationInfo();
+        },
+
+        _emitCurrentToken: function () {
+            //NOTE: if we have pending character token make it's end location equal to the
+            //current token's start location.
+            if (this.currentCharacterToken)
+                this.currentCharacterToken.location.endOffset = this.currentToken.location.startOffset;
+
+            this.currentToken.location.endOffset = mxn.posTracker.offset + 1;
+            orig._emitCurrentToken.call(this);
+        },
+
+        _emitCurrentCharacterToken: function () {
+            //NOTE: if we have character token and it's location wasn't set in the _emitCurrentToken(),
+            //then set it's location at the current preprocessor position.
+            //We don't need to increment preprocessor position, since character token
+            //emission is always forced by the start of the next character token here.
+            //So, we already have advanced position.
+            if (this.currentCharacterToken && this.currentCharacterToken.location.endOffset === -1)
+                this.currentCharacterToken.location.endOffset = mxn.posTracker.offset;
+
+            orig._emitCurrentCharacterToken.call(this);
+        }
+    };
+
+    //NOTE: patch initial states for each mode to obtain token start position
+    Object.keys(Tokenizer.MODE).forEach(function (modeName) {
+        var state = Tokenizer.MODE[modeName];
+
+        methods[state] = function (cp) {
+            mxn.currentTokenLocation = mxn._getCurrentLocation();
+            orig[state].call(this, cp);
+        };
+    });
+
+    return methods;
+};
+

package/lib/extensions/position_tracking/preprocessor_mixin.js

@@ -0,0 +1,72 @@
+'use strict';
+
+var Mixin = require('../../utils/mixin'),
+    inherits = require('util').inherits,
+    UNICODE = require('../../common/unicode');
+
+//Aliases
+var $ = UNICODE.CODE_POINTS;
+
+var PositionTrackingPreprocessorMixin = module.exports = function (preprocessor) {
+    // NOTE: avoid installing tracker twice
+    if (!preprocessor.__locTracker) {
+        preprocessor.__locTracker = this;
+
+        Mixin.call(this, preprocessor);
+
+        this.preprocessor = preprocessor;
+        this.isEol = false;
+        this.lineStartPos = 0;
+        this.droppedBufferSize = 0;
+
+        this.col = -1;
+        this.line = 1;
+    }
+
+    return preprocessor.__locTracker;
+};
+
+inherits(PositionTrackingPreprocessorMixin, Mixin);
+
+Object.defineProperty(PositionTrackingPreprocessorMixin.prototype, 'offset', {
+    get: function () {
+        return this.droppedBufferSize + this.preprocessor.pos;
+    }
+});
+
+PositionTrackingPreprocessorMixin.prototype._getOverriddenMethods = function (mxn, orig) {
+    return {
+        advance: function () {
+            var cp = orig.advance.call(this);
+
+            //NOTE: LF should be in the last column of the line
+            if (mxn.isEol) {
+                mxn.isEol = false;
+                mxn.line++;
+                mxn.lineStartPos = mxn.offset;
+            }
+
+            if (cp === $.LINE_FEED)
+                mxn.isEol = true;
+
+            mxn.col = mxn.offset - mxn.lineStartPos + 1;
+
+            return cp;
+        },
+
+        retreat: function () {
+            orig.retreat.call(this);
+            mxn.isEol = false;
+
+            mxn.col = mxn.offset - mxn.lineStartPos + 1;
+        },
+
+        dropParsedChunk: function () {
+            var prevPos = this.pos;
+
+            orig.dropParsedChunk.call(this);
+
+            mxn.droppedBufferSize += prevPos - this.pos;
+        }
+    };
+};

package/lib/index.d.ts

@@ -54,9 +54,9 @@ declare namespace MarkupData {
 declare namespace Options {
     export interface ParserOptions {
         /**
-         * Enables source code location information for the nodes. When enabled, each node (except root node) has the `__location` property.
-         * In case the node is not an empty element, `__location` will be {@link MarkupData.ElementLocation} object, otherwise it's {@link MarkupData.Location}.
-         * If the element was implicitly created by the parser it's `__location` property will be null.
+         * Enables source code location information. When enabled, each node (except the root node) will have a `__location` property.
+         * If the node is not an empty element, `__location` will be a {@link MarkupData.ElementLocation} object, otherwise it will be {@link MarkupData.Location}.
+         * If the element was implicitly created by the parser (as part of [tree correction](https://html.spec.whatwg.org/multipage/syntax.html#an-introduction-to-error-handling-and-strange-cases-in-the-parser)), its `__location` property will be `undefined`.
          *
          * **Default:** `false`
          */

package/lib/parser/index.js

@@ -3,11 +3,11 @@
 var Tokenizer = require('../tokenizer'),
     OpenElementStack = require('./open_element_stack'),
     FormattingElementList = require('./formatting_element_list'),
-    locationInfoMixin = require('../location_info/parser_mixin'),
+    LocationInfoParserMixin = require('../extensions/location_info/parser_mixin'),
     defaultTreeAdapter = require('../tree_adapters/default'),
+    mergeOptions = require('../utils/merge_options'),
     doctype = require('../common/doctype'),
     foreignContent = require('../common/foreign_content'),
-    mergeOptions = require('../common/merge_options'),
     UNICODE = require('../common/unicode'),
     HTML = require('../common/html');
 
@@ -310,7 +310,7 @@ var Parser = module.exports = function (options) {
     this.pendingScript = null;
 
     if (this.options.locationInfo)
-        locationInfoMixin.assign(this);
+        new LocationInfoParserMixin(this);
 };
 
 // API

package/lib/parser/parser_stream.js

@@ -35,7 +35,7 @@ ParserStream.prototype._write = function (chunk, encoding, callback) {
 
 ParserStream.prototype.end = function (chunk, encoding, callback) {
     this.lastChunkWritten = true;
-    WritableStream.prototype.end.call(this, chunk, encoding, callback);
+    WritableStream.prototype.end.call(this, chunk || '', encoding, callback);
 };
 
 //Scriptable parser implementation

package/lib/sax/index.js

@@ -4,8 +4,9 @@ var TransformStream = require('stream').Transform,
     DevNullStream = require('./dev_null_stream'),
     inherits = require('util').inherits,
     Tokenizer = require('../tokenizer'),
+    LocationInfoTokenizerMixin = require('../extensions/location_info/tokenizer_mixin'),
     ParserFeedbackSimulator = require('./parser_feedback_simulator'),
-    mergeOptions = require('../common/merge_options');
+    mergeOptions = require('../utils/merge_options');
 
 var DEFAULT_OPTIONS = {
     locationInfo: false
@@ -17,6 +18,10 @@ var SAXParser = module.exports = function (options) {
     this.options = mergeOptions(DEFAULT_OPTIONS, options);
 
     this.tokenizer = new Tokenizer(options);
+
+    if (this.options.locationInfo)
+        new LocationInfoTokenizerMixin(this.tokenizer);
+
     this.parserFeedbackSimulator = new ParserFeedbackSimulator(this.tokenizer);
 
     this.pendingText = null;

package/lib/serializer/index.js

@@ -1,8 +1,8 @@
 'use strict';
 
 var defaultTreeAdapter = require('../tree_adapters/default'),
+    mergeOptions = require('../utils/merge_options'),
     doctype = require('../common/doctype'),
-    mergeOptions = require('../common/merge_options'),
     HTML = require('../common/html');
 
 //Aliases

package/lib/tokenizer/index.js

@@ -1,7 +1,6 @@
 'use strict';
 
 var Preprocessor = require('./preprocessor'),
-    locationInfoMixin = require('../location_info/tokenizer_mixin'),
     UNICODE = require('../common/unicode'),
     neTree = require('./named_entity_data');
 
@@ -173,7 +172,7 @@ function findNamedEntityTreeBranch(nodeIx, cp) {
 
 
 //Tokenizer
-var Tokenizer = module.exports = function (options) {
+var Tokenizer = module.exports = function () {
     this.preprocessor = new Preprocessor();
 
     this.tokenQueue = [];
@@ -193,9 +192,6 @@ var Tokenizer = module.exports = function (options) {
     this.currentCharacterToken = null;
     this.currentToken = null;
     this.currentAttr = null;
-
-    if (options && options.locationInfo)
-        locationInfoMixin.assign(this);
 };
 
 //Token types
@@ -210,7 +206,7 @@ Tokenizer.EOF_TOKEN = 'EOF_TOKEN';
 Tokenizer.HIBERNATION_TOKEN = 'HIBERNATION_TOKEN';
 
 //Tokenizer initial states for different modes
-Tokenizer.MODE = Tokenizer.prototype.MODE = {
+Tokenizer.MODE = {
     DATA: DATA_STATE,
     RCDATA: RCDATA_STATE,
     RAWTEXT: RAWTEXT_STATE,

package/lib/tokenizer/preprocessor.js

@@ -32,7 +32,6 @@ var Preprocessor = module.exports = function () {
     this.pos = -1;
     this.lastGapPos = -1;
     this.lastCharPos = -1;
-    this.droppedBufferSize = 0;
 
     this.gapStack = [];
 
@@ -43,16 +42,9 @@ var Preprocessor = module.exports = function () {
     this.bufferWaterline = DEFAULT_BUFFER_WATERLINE;
 };
 
-Object.defineProperty(Preprocessor.prototype, 'sourcePos', {
-    get: function () {
-        return this.droppedBufferSize + this.pos;
-    }
-});
-
 Preprocessor.prototype.dropParsedChunk = function () {
     if (this.pos > this.bufferWaterline) {
         this.lastCharPos -= this.pos;
-        this.droppedBufferSize += this.pos;
         this.html = this.html.substring(this.pos);
         this.pos = 0;
         this.lastGapPos = -1;

package/lib/utils/mixin.js

@@ -0,0 +1,18 @@
+'use strict';
+
+var Mixin = module.exports = function (host) {
+    var originalMethods = {},
+        overriddenMethods = this._getOverriddenMethods(this, originalMethods);
+
+    Object.keys(overriddenMethods).forEach(function (key) {
+        if (typeof overriddenMethods[key] === 'function') {
+            originalMethods[key] = host[key];
+            host[key] = overriddenMethods[key];
+        }
+    });
+};
+
+Mixin.prototype._getOverriddenMethods = function () {
+    throw new Error('Not implemented');
+};
+

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "parse5",
   "description": "HTML parsing/serialization toolset for Node.js. WHATWG HTML Living Standard (aka HTML5)-compliant.",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "author": "Ivan Nikulin <ifaaan@gmail.com> (https://github.com/inikulin)",
   "contributors": "https://github.com/inikulin/parse5/graphs/contributors",
   "homepage": "https://github.com/inikulin/parse5",
@@ -58,6 +58,6 @@
     "lib"
   ],
   "dependencies": {
-    "@types/node": "^6.0.46"
+    "@types/node": "*"
   }
 }

package/lib/location_info/parser_mixin.js

@@ -1,220 +0,0 @@
-'use strict';
-
-var OpenElementStack = require('../parser/open_element_stack'),
-    Tokenizer = require('../tokenizer'),
-    HTML = require('../common/html');
-
-
-//Aliases
-var $ = HTML.TAG_NAMES;
-
-exports.assign = function (parser) {
-    //NOTE: obtain Parser proto this way to avoid module circular references
-    var parserProto = Object.getPrototypeOf(parser),
-        treeAdapter = parser.treeAdapter,
-        attachableElementLocation = null,
-        lastFosterParentingLocation = null,
-        currentToken = null;
-
-    function setEndLocation(element, closingToken) {
-        var loc = element.__location;
-
-        if (!loc)
-            return;
-
-        if (!loc.startTag) {
-            loc.startTag = {
-                line: loc.line,
-                col: loc.col,
-                startOffset: loc.startOffset,
-                endOffset: loc.endOffset
-            };
-
-            if (loc.attrs)
-                loc.startTag.attrs = loc.attrs;
-        }
-
-        if (closingToken.location) {
-            var ctLocation = closingToken.location,
-                tn = treeAdapter.getTagName(element),
-            // NOTE: For cases like <p> <p> </p> - First 'p' closes without a closing tag and
-            // for cases like <td> <p> </td> - 'p' closes without a closing tag
-                isClosingEndTag = closingToken.type === Tokenizer.END_TAG_TOKEN &&
-                                  tn === closingToken.tagName;
-
-            if (isClosingEndTag) {
-                loc.endTag = {
-                    line: ctLocation.line,
-                    col: ctLocation.col,
-                    startOffset: ctLocation.startOffset,
-                    endOffset: ctLocation.endOffset
-                };
-            }
-
-            if (isClosingEndTag)
-                loc.endOffset = ctLocation.endOffset;
-            else
-                loc.endOffset = ctLocation.startOffset;
-        }
-
-        else if (closingToken.type === Tokenizer.EOF_TOKEN)
-            loc.endOffset = parser.tokenizer.preprocessor.sourcePos;
-    }
-
-    //NOTE: patch _bootstrap method
-    parser._bootstrap = function (document, fragmentContext) {
-        parserProto._bootstrap.call(this, document, fragmentContext);
-
-        attachableElementLocation = null;
-        lastFosterParentingLocation = null;
-        currentToken = null;
-
-        //OpenElementStack
-        parser.openElements.pop = function () {
-            setEndLocation(this.current, currentToken);
-            OpenElementStack.prototype.pop.call(this);
-        };
-
-        parser.openElements.popAllUpToHtmlElement = function () {
-            for (var i = this.stackTop; i > 0; i--)
-                setEndLocation(this.items[i], currentToken);
-
-            OpenElementStack.prototype.popAllUpToHtmlElement.call(this);
-        };
-
-        parser.openElements.remove = function (element) {
-            setEndLocation(element, currentToken);
-            OpenElementStack.prototype.remove.call(this, element);
-        };
-    };
-
-    parser._runParsingLoop = function (scriptHandler) {
-        parserProto._runParsingLoop.call(this, scriptHandler);
-
-        // NOTE: generate location info for elements
-        // that remains on open element stack
-        for (var i = parser.openElements.stackTop; i >= 0; i--)
-            setEndLocation(parser.openElements.items[i], currentToken);
-    };
-
-
-    //Token processing
-    parser._processTokenInForeignContent = function (token) {
-        currentToken = token;
-        parserProto._processTokenInForeignContent.call(this, token);
-    };
-
-    parser._processToken = function (token) {
-        currentToken = token;
-        parserProto._processToken.call(this, token);
-
-        //NOTE: <body> and <html> are never popped from the stack, so we need to updated
-        //their end location explicitly.
-        if (token.type === Tokenizer.END_TAG_TOKEN &&
-            (token.tagName === $.HTML ||
-             token.tagName === $.BODY && this.openElements.hasInScope($.BODY))) {
-            for (var i = this.openElements.stackTop; i >= 0; i--) {
-                var element = this.openElements.items[i];
-
-                if (this.treeAdapter.getTagName(element) === token.tagName) {
-                    setEndLocation(element, token);
-                    break;
-                }
-            }
-        }
-    };
-
-
-    //Doctype
-    parser._setDocumentType = function (token) {
-        parserProto._setDocumentType.call(this, token);
-
-        var documentChildren = this.treeAdapter.getChildNodes(this.document),
-            cnLength = documentChildren.length;
-
-        for (var i = 0; i < cnLength; i++) {
-            var node = documentChildren[i];
-
-            if (this.treeAdapter.isDocumentTypeNode(node)) {
-                node.__location = token.location;
-                break;
-            }
-        }
-    };
-
-
-    //Elements
-    parser._attachElementToTree = function (element) {
-        //NOTE: _attachElementToTree is called from _appendElement, _insertElement and _insertTemplate methods.
-        //So we will use token location stored in this methods for the element.
-        element.__location = attachableElementLocation || null;
-        attachableElementLocation = null;
-        parserProto._attachElementToTree.call(this, element);
-    };
-
-    parser._appendElement = function (token, namespaceURI) {
-        attachableElementLocation = token.location;
-        parserProto._appendElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertElement = function (token, namespaceURI) {
-        attachableElementLocation = token.location;
-        parserProto._insertElement.call(this, token, namespaceURI);
-    };
-
-    parser._insertTemplate = function (token) {
-        attachableElementLocation = token.location;
-        parserProto._insertTemplate.call(this, token);
-
-        var tmplContent = this.treeAdapter.getTemplateContent(this.openElements.current);
-
-        tmplContent.__location = null;
-    };
-
-    parser._insertFakeRootElement = function () {
-        parserProto._insertFakeRootElement.call(this);
-        this.openElements.current.__location = null;
-    };
-
-
-    //Comments
-    parser._appendCommentNode = function (token, parent) {
-        parserProto._appendCommentNode.call(this, token, parent);
-
-        var children = this.treeAdapter.getChildNodes(parent),
-            commentNode = children[children.length - 1];
-
-        commentNode.__location = token.location;
-    };
-
-
-    //Text
-    parser._findFosterParentingLocation = function () {
-        //NOTE: store last foster parenting location, so we will be able to find inserted text
-        //in case of foster parenting
-        lastFosterParentingLocation = parserProto._findFosterParentingLocation.call(this);
-        return lastFosterParentingLocation;
-    };
-
-    parser._insertCharacters = function (token) {
-        parserProto._insertCharacters.call(this, token);
-
-        var hasFosterParent = this._shouldFosterParentOnInsertion(),
-            parent = hasFosterParent && lastFosterParentingLocation.parent ||
-                     this.openElements.currentTmplContent ||
-                     this.openElements.current,
-            siblings = this.treeAdapter.getChildNodes(parent),
-            textNodeIdx = hasFosterParent && lastFosterParentingLocation.beforeElement ?
-            siblings.indexOf(lastFosterParentingLocation.beforeElement) - 1 :
-            siblings.length - 1,
-            textNode = siblings[textNodeIdx];
-
-        //NOTE: if we have location assigned by another token, then just update end position
-        if (textNode.__location)
-            textNode.__location.endOffset = token.location.endOffset;
-
-        else
-            textNode.__location = token.location;
-    };
-};
-

package/lib/location_info/tokenizer_mixin.js

@@ -1,148 +0,0 @@
-'use strict';
-
-var UNICODE = require('../common/unicode');
-
-//Aliases
-var $ = UNICODE.CODE_POINTS;
-
-
-exports.assign = function (tokenizer) {
-    //NOTE: obtain Tokenizer proto this way to avoid module circular references
-    var tokenizerProto = Object.getPrototypeOf(tokenizer),
-        tokenStartOffset = -1,
-        tokenCol = -1,
-        tokenLine = 1,
-        isEol = false,
-        lineStartPos = 0,
-        col = -1,
-        line = 1;
-
-    function attachLocationInfo(token) {
-        token.location = {
-            line: tokenLine,
-            col: tokenCol,
-            startOffset: tokenStartOffset,
-            endOffset: -1
-        };
-    }
-
-    //NOTE: patch consumption method to track line/col information
-    tokenizer._consume = function () {
-        var cp = tokenizerProto._consume.call(this);
-
-        //NOTE: LF should be in the last column of the line
-        if (isEol) {
-            isEol = false;
-            line++;
-            lineStartPos = this.preprocessor.sourcePos;
-        }
-
-        if (cp === $.LINE_FEED)
-            isEol = true;
-
-        col = this.preprocessor.sourcePos - lineStartPos + 1;
-
-        return cp;
-    };
-
-    tokenizer._unconsume = function () {
-        tokenizerProto._unconsume.call(this);
-        isEol = false;
-
-        col = this.preprocessor.sourcePos - lineStartPos + 1;
-    };
-
-    //NOTE: patch token creation methods and attach location objects
-    tokenizer._createStartTagToken = function () {
-        tokenizerProto._createStartTagToken.call(this);
-        attachLocationInfo(this.currentToken);
-    };
-
-    tokenizer._createEndTagToken = function () {
-        tokenizerProto._createEndTagToken.call(this);
-        attachLocationInfo(this.currentToken);
-    };
-
-    tokenizer._createCommentToken = function () {
-        tokenizerProto._createCommentToken.call(this);
-        attachLocationInfo(this.currentToken);
-    };
-
-    tokenizer._createDoctypeToken = function (initialName) {
-        tokenizerProto._createDoctypeToken.call(this, initialName);
-        attachLocationInfo(this.currentToken);
-    };
-
-    tokenizer._createCharacterToken = function (type, ch) {
-        tokenizerProto._createCharacterToken.call(this, type, ch);
-        attachLocationInfo(this.currentCharacterToken);
-    };
-
-    tokenizer._createAttr = function (attrNameFirstCh) {
-        tokenizerProto._createAttr.call(this, attrNameFirstCh);
-        this.currentAttrLocation = {
-            line: line,
-            col: col,
-            startOffset: this.preprocessor.sourcePos,
-            endOffset: -1
-        };
-    };
-
-    tokenizer._leaveAttrName = function (toState) {
-        tokenizerProto._leaveAttrName.call(this, toState);
-        this._attachCurrentAttrLocationInfo();
-    };
-
-    tokenizer._leaveAttrValue = function (toState) {
-        tokenizerProto._leaveAttrValue.call(this, toState);
-        this._attachCurrentAttrLocationInfo();
-    };
-
-    tokenizer._attachCurrentAttrLocationInfo = function () {
-        this.currentAttrLocation.endOffset = this.preprocessor.sourcePos;
-
-        if (!this.currentToken.location.attrs)
-            this.currentToken.location.attrs = Object.create(null);
-
-        this.currentToken.location.attrs[this.currentAttr.name] = this.currentAttrLocation;
-    };
-
-    //NOTE: patch token emission methods to determine end location
-    tokenizer._emitCurrentToken = function () {
-        //NOTE: if we have pending character token make it's end location equal to the
-        //current token's start location.
-        if (this.currentCharacterToken)
-            this.currentCharacterToken.location.endOffset = this.currentToken.location.startOffset;
-
-        this.currentToken.location.endOffset = this.preprocessor.sourcePos + 1;
-        tokenizerProto._emitCurrentToken.call(this);
-    };
-
-    tokenizer._emitCurrentCharacterToken = function () {
-        //NOTE: if we have character token and it's location wasn't set in the _emitCurrentToken(),
-        //then set it's location at the current preprocessor position.
-        //We don't need to increment preprocessor position, since character token
-        //emission is always forced by the start of the next character token here.
-        //So, we already have advanced position.
-        if (this.currentCharacterToken && this.currentCharacterToken.location.endOffset === -1)
-            this.currentCharacterToken.location.endOffset = this.preprocessor.sourcePos;
-
-        tokenizerProto._emitCurrentCharacterToken.call(this);
-    };
-
-    //NOTE: patch initial states for each mode to obtain token start position
-    Object.keys(tokenizerProto.MODE)
-
-        .map(function (modeName) {
-            return tokenizerProto.MODE[modeName];
-        })
-
-        .forEach(function (state) {
-            tokenizer[state] = function (cp) {
-                tokenStartOffset = this.preprocessor.sourcePos;
-                tokenLine = line;
-                tokenCol = col;
-                tokenizerProto[state].call(this, cp);
-            };
-        });
-};