rapydscript-ng 0.7.24 → 0.8.1

package/src/tokenizer.pyj

@@ -2,97 +2,97 @@
 # License: BSD Copyright: 2016, Kovid Goyal <kovid at kovidgoyal.net>
 from __python__ import hash_literals
 
-from unicode_aliases import ALIAS_MAP
-from utils import make_predicate, characters
 from ast import AST_Token
 from errors import SyntaxError
 from string_interpolation import interpolate, quoted_string
+from unicode_aliases import ALIAS_MAP
+from utils import make_predicate, characters
 
 RE_HEX_NUMBER = /^0x[0-9a-f]+$/i
 RE_OCT_NUMBER = /^0[0-7]+$/
 RE_DEC_NUMBER = /^\d*\.?\d*(?:e[+-]?\d*(?:\d\.?|\.?\d)\d*)?$/i
 
-OPERATOR_CHARS = make_predicate(characters("+-*&%=<>!?|~^@"))
+OPERATOR_CHARS = make_predicate(characters('+-*&%=<>!?|~^@'))
 
-ASCII_CONTROL_CHARS = {'a':7, 'b':8, 'f': 12, 'n': 10, 'r': 13, 't': 9, 'v': 11}
+ASCII_CONTROL_CHARS = {'a': 7, 'b': 8, 'f': 12, 'n': 10, 'r': 13, 't': 9, 'v': 11}
 HEX_PAT = /[a-fA-F0-9]/
 NAME_PAT = /[a-zA-Z ]/
 
 OPERATORS = make_predicate([
-    "in",
-    "instanceof",
-    "typeof",
-    "new",
-    "void",
-    "del",
-    "+",
-    "-",
-    "not",
-    "~",
-    "&",
-    "|",
-    "^",
-    "**",
-    "*",
-    "//",
-    "/",
-    "%",
-    ">>",
-    "<<",
-    ">>>",
-    "<",
-    ">",
-    "<=",
-    ">=",
-    "==",
-    "is",
-    "!=",
-    "=",
-    "+=",
-    "-=",
-    "//=",
-    "/=",
-    "*=",
-    "%=",
-    ">>=",
-    "<<=",
-    ">>>=",
-    "|=",
-    "^=",
-    "&=",
-    "and",
-    "or",
-    "@",
-    "->"
+    'in',
+    'instanceof',
+    'typeof',
+    'new',
+    'void',
+    'del',
+    '+',
+    '-',
+    'not',
+    '~',
+    '&',
+    '|',
+    '^',
+    '**',
+    '*',
+    '//',
+    '/',
+    '%',
+    '>>',
+    '<<',
+    '>>>',
+    '<',
+    '>',
+    '<=',
+    '>=',
+    '==',
+    'is',
+    '!=',
+    '=',
+    '+=',
+    '-=',
+    '//=',
+    '/=',
+    '*=',
+    '%=',
+    '>>=',
+    '<<=',
+    '>>>=',
+    '|=',
+    '^=',
+    '&=',
+    'and',
+    'or',
+    '@',
+    '->'
 ])
 
 OP_MAP = {
-    'or': "||",
-    'and': "&&",
-    'not': "!",
-    'del': "delete",
-    'None': "null",
-    'is': "===",
+    'or': '||',
+    'and': '&&',
+    'not': '!',
+    'del': 'delete',
+    'None': 'null',
+    'is': '===',
 }
 
-WHITESPACE_CHARS = make_predicate(characters(" \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000"))
+WHITESPACE_CHARS = make_predicate(characters(' \u00a0\n\r\t\f\u000b\u200b\u180e\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a\u202f\u205f\u3000'))
 
-PUNC_BEFORE_EXPRESSION = make_predicate(characters("[{(,.;:"))
+PUNC_BEFORE_EXPRESSION = make_predicate(characters('[{(,.;:'))
 
-PUNC_CHARS = make_predicate(characters("[]{}(),;:?"))
+PUNC_CHARS = make_predicate(characters('[]{}(),;:?'))
 
-KEYWORDS = "as assert break class continue def del do elif else except finally for from global if import in is new nonlocal pass raise return yield try while with or and not"
+KEYWORDS = 'as assert async await break class continue def del do elif else except finally for from global if import in is new nonlocal pass raise return yield try while with or and not'
 
-KEYWORDS_ATOM = "False None True"
+KEYWORDS_ATOM = 'False None True'
 
 # see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Lexical_grammar
-RESERVED_WORDS = ("break case class catch const continue debugger default delete do else export extends"
-" finally for function if import in instanceof new return super switch this throw try typeof var void"
-" while with yield enum implements static private package let public protected interface await null true false" )
+RESERVED_WORDS = ('break case class catch const continue debugger default delete do else export extends'
+' finally for function if import in instanceof new return super switch this throw try typeof var void'
+' while with yield enum implements static private package let public protected interface null true false')
 
-KEYWORDS_BEFORE_EXPRESSION = "return yield new del raise elif else if"
+KEYWORDS_BEFORE_EXPRESSION = 'return yield new del raise elif else if await'
 
-ALL_KEYWORDS = KEYWORDS + " " + KEYWORDS_ATOM
+ALL_KEYWORDS = KEYWORDS + ' ' + KEYWORDS_ATOM
 
 KEYWORDS = make_predicate(KEYWORDS)
 RESERVED_WORDS = make_predicate(RESERVED_WORDS)
@@ -100,37 +100,47 @@ KEYWORDS_BEFORE_EXPRESSION = make_predicate(KEYWORDS_BEFORE_EXPRESSION)
 KEYWORDS_ATOM = make_predicate(KEYWORDS_ATOM)
 IDENTIFIER_PAT = /^[a-z_$][_a-z0-9$]*$/i
 
+
 def is_string_modifier(val):
     for ch in val:
         if 'vrufVRUF'.indexOf(ch) is -1:
             return False
     return True
 
+
 def is_letter(code):
     return code >= 97 and code <= 122 or code >= 65 and code <= 90 or code >= 170 and UNICODE.letter.test(String.fromCharCode(code))
 
+
 def is_digit(code):
     return code >= 48 and code <= 57
 
+
 def is_alphanumeric_char(code):
     return is_digit(code) or is_letter(code)
 
+
 def is_unicode_combining_mark(ch):
     return UNICODE.non_spacing_mark.test(ch) or UNICODE.space_combining_mark.test(ch)
 
+
 def is_unicode_connector_punctuation(ch):
     return UNICODE.connector_punctuation.test(ch)
 
+
 def is_identifier(name):
     return not RESERVED_WORDS[name] and not KEYWORDS[name] and not KEYWORDS_ATOM[name] and IDENTIFIER_PAT.test(name)
 
+
 def is_identifier_start(code):
     return code is 36 or code is 95 or is_letter(code)
 
+
 def is_identifier_char(ch):
     code = ch.charCodeAt(0)
     return is_identifier_start(code) or is_digit(code) or code is 8204 or code is 8205 or is_unicode_combining_mark(ch) or is_unicode_connector_punctuation(ch)
 
+
 def parse_js_number(num):
     if RE_HEX_NUMBER.test(num):
         return parseInt(num.substr(2), 16)
@@ -141,10 +151,10 @@ def parse_js_number(num):
 
 # regexps adapted from http://xregexp.com/plugins/#unicode
 UNICODE = {  # {{{
-    'letter': RegExp("[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376\\u0377\\u037A-\\u037D\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u0523\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05D0-\\u05EA\\u05F0-\\u05F2\\u0621-\\u064A\\u066E\\u066F\\u0671-\\u06D3\\u06D5\\u06E5\\u06E6\\u06EE\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4\\u07F5\\u07FA\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971\\u0972\\u097B-\\u097F\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC\\u09DD\\u09DF-\\u09E1\\u09F0\\u09F1\\u0A05-\\u0A0A\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A33\\u0A35\\u0A36\\u0A38\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0\\u0AE1\\u0B05-\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C33\\u0C35-\\u0C39\\u0C3D\\u0C58\\u0C59\\u0C60\\u0C61\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDE\\u0CE0\\u0CE1\\u0D05-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D28\\u0D2A-\\u0D39\\u0D3D\\u0D60\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32\\u0E33\\u0E40-\\u0E46\\u0E81\\u0E82\\u0E84\\u0E87\\u0E88\\u0E8A\\u0E8D\\u0E94-\\u0E97\\u0E99-\\u0E9F\\u0EA1-\\u0EA3\\u0EA5\\u0EA7\\u0EAA\\u0EAB\\u0EAD-\\u0EB0\\u0EB2\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC\\u0EDD\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8B\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10D0-\\u10FA\\u10FC\\u1100-\\u1159\\u115F-\\u11A2\\u11A8-\\u11F9\\u1200-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F4\\u1401-\\u166C\\u166F-\\u1676\\u1681-\\u169A\\u16A0-\\u16EA\\u1700-\\u170C\\u170E-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1877\\u1880-\\u18A8\\u18AA\\u1900-\\u191C\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19A9\\u19C1-\\u19C7\\u1A00-\\u1A16\\u1B05-\\u1B33\\u1B45-\\u1B4B\\u1B83-\\u1BA0\\u1BAE\\u1BAF\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u2094\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183\\u2184\\u2C00-\\u2C2E\\u2C30-\\u2C5E\\u2C60-\\u2C6F\\u2C71-\\u2C7D\\u2C80-\\u2CE4\\u2D00-\\u2D25\\u2D30-\\u2D65\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005\\u3006\\u3031-\\u3035\\u303B\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312D\\u3131-\\u318E\\u31A0-\\u31B7\\u31F0-\\u31FF\\u3400\\u4DB5\\u4E00\\u9FC3\\uA000-\\uA48C\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A\\uA62B\\uA640-\\uA65F\\uA662-\\uA66E\\uA67F-\\uA697\\uA717-\\uA71F\\uA722-\\uA788\\uA78B\\uA78C\\uA7FB-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA90A-\\uA925\\uA930-\\uA946\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAC00\\uD7A3\\uF900-\\uFA2D\\uFA30-\\uFA6A\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC]"),
-    'non_spacing_mark': RegExp("[\\u0300-\\u036F\\u0483-\\u0487\\u0591-\\u05BD\\u05BF\\u05C1\\u05C2\\u05C4\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065E\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0900-\\u0902\\u093C\\u0941-\\u0948\\u094D\\u0951-\\u0955\\u0962\\u0963\\u0981\\u09BC\\u09C1-\\u09C4\\u09CD\\u09E2\\u09E3\\u0A01\\u0A02\\u0A3C\\u0A41\\u0A42\\u0A47\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70\\u0A71\\u0A75\\u0A81\\u0A82\\u0ABC\\u0AC1-\\u0AC5\\u0AC7\\u0AC8\\u0ACD\\u0AE2\\u0AE3\\u0B01\\u0B3C\\u0B3F\\u0B41-\\u0B44\\u0B4D\\u0B56\\u0B62\\u0B63\\u0B82\\u0BC0\\u0BCD\\u0C3E-\\u0C40\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55\\u0C56\\u0C62\\u0C63\\u0CBC\\u0CBF\\u0CC6\\u0CCC\\u0CCD\\u0CE2\\u0CE3\\u0D41-\\u0D44\\u0D4D\\u0D62\\u0D63\\u0DCA\\u0DD2-\\u0DD4\\u0DD6\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EB9\\u0EBB\\u0EBC\\u0EC8-\\u0ECD\\u0F18\\u0F19\\u0F35\\u0F37\\u0F39\\u0F71-\\u0F7E\\u0F80-\\u0F84\\u0F86\\u0F87\\u0F90-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102D-\\u1030\\u1032-\\u1037\\u1039\\u103A\\u103D\\u103E\\u1058\\u1059\\u105E-\\u1060\\u1071-\\u1074\\u1082\\u1085\\u1086\\u108D\\u109D\\u135F\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17B7-\\u17BD\\u17C6\\u17C9-\\u17D3\\u17DD\\u180B-\\u180D\\u18A9\\u1920-\\u1922\\u1927\\u1928\\u1932\\u1939-\\u193B\\u1A17\\u1A18\\u1A56\\u1A58-\\u1A5E\\u1A60\\u1A62\\u1A65-\\u1A6C\\u1A73-\\u1A7C\\u1A7F\\u1B00-\\u1B03\\u1B34\\u1B36-\\u1B3A\\u1B3C\\u1B42\\u1B6B-\\u1B73\\u1B80\\u1B81\\u1BA2-\\u1BA5\\u1BA8\\u1BA9\\u1C2C-\\u1C33\\u1C36\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE0\\u1CE2-\\u1CE8\\u1CED\\u1DC0-\\u1DE6\\u1DFD-\\u1DFF\\u20D0-\\u20DC\\u20E1\\u20E5-\\u20F0\\u2CEF-\\u2CF1\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099\\u309A\\uA66F\\uA67C\\uA67D\\uA6F0\\uA6F1\\uA802\\uA806\\uA80B\\uA825\\uA826\\uA8C4\\uA8E0-\\uA8F1\\uA926-\\uA92D\\uA947-\\uA951\\uA980-\\uA982\\uA9B3\\uA9B6-\\uA9B9\\uA9BC\\uAA29-\\uAA2E\\uAA31\\uAA32\\uAA35\\uAA36\\uAA43\\uAA4C\\uAAB0\\uAAB2-\\uAAB4\\uAAB7\\uAAB8\\uAABE\\uAABF\\uAAC1\\uABE5\\uABE8\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE26]"),
-    'space_combining_mark': RegExp("[\\u0903\\u093E-\\u0940\\u0949-\\u094C\\u094E\\u0982\\u0983\\u09BE-\\u09C0\\u09C7\\u09C8\\u09CB\\u09CC\\u09D7\\u0A03\\u0A3E-\\u0A40\\u0A83\\u0ABE-\\u0AC0\\u0AC9\\u0ACB\\u0ACC\\u0B02\\u0B03\\u0B3E\\u0B40\\u0B47\\u0B48\\u0B4B\\u0B4C\\u0B57\\u0BBE\\u0BBF\\u0BC1\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCC\\u0BD7\\u0C01-\\u0C03\\u0C41-\\u0C44\\u0C82\\u0C83\\u0CBE\\u0CC0-\\u0CC4\\u0CC7\\u0CC8\\u0CCA\\u0CCB\\u0CD5\\u0CD6\\u0D02\\u0D03\\u0D3E-\\u0D40\\u0D46-\\u0D48\\u0D4A-\\u0D4C\\u0D57\\u0D82\\u0D83\\u0DCF-\\u0DD1\\u0DD8-\\u0DDF\\u0DF2\\u0DF3\\u0F3E\\u0F3F\\u0F7F\\u102B\\u102C\\u1031\\u1038\\u103B\\u103C\\u1056\\u1057\\u1062-\\u1064\\u1067-\\u106D\\u1083\\u1084\\u1087-\\u108C\\u108F\\u109A-\\u109C\\u17B6\\u17BE-\\u17C5\\u17C7\\u17C8\\u1923-\\u1926\\u1929-\\u192B\\u1930\\u1931\\u1933-\\u1938\\u19B0-\\u19C0\\u19C8\\u19C9\\u1A19-\\u1A1B\\u1A55\\u1A57\\u1A61\\u1A63\\u1A64\\u1A6D-\\u1A72\\u1B04\\u1B35\\u1B3B\\u1B3D-\\u1B41\\u1B43\\u1B44\\u1B82\\u1BA1\\u1BA6\\u1BA7\\u1BAA\\u1C24-\\u1C2B\\u1C34\\u1C35\\u1CE1\\u1CF2\\uA823\\uA824\\uA827\\uA880\\uA881\\uA8B4-\\uA8C3\\uA952\\uA953\\uA983\\uA9B4\\uA9B5\\uA9BA\\uA9BB\\uA9BD-\\uA9C0\\uAA2F\\uAA30\\uAA33\\uAA34\\uAA4D\\uAA7B\\uABE3\\uABE4\\uABE6\\uABE7\\uABE9\\uABEA\\uABEC]"),
-    'connector_punctuation': RegExp("[\\u005F\\u203F\\u2040\\u2054\\uFE33\\uFE34\\uFE4D-\\uFE4F\\uFF3F]")
+    'letter': RegExp('[\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376\\u0377\\u037A-\\u037D\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u0523\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05D0-\\u05EA\\u05F0-\\u05F2\\u0621-\\u064A\\u066E\\u066F\\u0671-\\u06D3\\u06D5\\u06E5\\u06E6\\u06EE\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4\\u07F5\\u07FA\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971\\u0972\\u097B-\\u097F\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC\\u09DD\\u09DF-\\u09E1\\u09F0\\u09F1\\u0A05-\\u0A0A\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A33\\u0A35\\u0A36\\u0A38\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0\\u0AE1\\u0B05-\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C33\\u0C35-\\u0C39\\u0C3D\\u0C58\\u0C59\\u0C60\\u0C61\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDE\\u0CE0\\u0CE1\\u0D05-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D28\\u0D2A-\\u0D39\\u0D3D\\u0D60\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32\\u0E33\\u0E40-\\u0E46\\u0E81\\u0E82\\u0E84\\u0E87\\u0E88\\u0E8A\\u0E8D\\u0E94-\\u0E97\\u0E99-\\u0E9F\\u0EA1-\\u0EA3\\u0EA5\\u0EA7\\u0EAA\\u0EAB\\u0EAD-\\u0EB0\\u0EB2\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC\\u0EDD\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8B\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10D0-\\u10FA\\u10FC\\u1100-\\u1159\\u115F-\\u11A2\\u11A8-\\u11F9\\u1200-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F4\\u1401-\\u166C\\u166F-\\u1676\\u1681-\\u169A\\u16A0-\\u16EA\\u1700-\\u170C\\u170E-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1877\\u1880-\\u18A8\\u18AA\\u1900-\\u191C\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19A9\\u19C1-\\u19C7\\u1A00-\\u1A16\\u1B05-\\u1B33\\u1B45-\\u1B4B\\u1B83-\\u1BA0\\u1BAE\\u1BAF\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u2094\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2183\\u2184\\u2C00-\\u2C2E\\u2C30-\\u2C5E\\u2C60-\\u2C6F\\u2C71-\\u2C7D\\u2C80-\\u2CE4\\u2D00-\\u2D25\\u2D30-\\u2D65\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005\\u3006\\u3031-\\u3035\\u303B\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312D\\u3131-\\u318E\\u31A0-\\u31B7\\u31F0-\\u31FF\\u3400\\u4DB5\\u4E00\\u9FC3\\uA000-\\uA48C\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A\\uA62B\\uA640-\\uA65F\\uA662-\\uA66E\\uA67F-\\uA697\\uA717-\\uA71F\\uA722-\\uA788\\uA78B\\uA78C\\uA7FB-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA90A-\\uA925\\uA930-\\uA946\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAC00\\uD7A3\\uF900-\\uFA2D\\uFA30-\\uFA6A\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC]'),
+    'non_spacing_mark': RegExp('[\\u0300-\\u036F\\u0483-\\u0487\\u0591-\\u05BD\\u05BF\\u05C1\\u05C2\\u05C4\\u05C5\\u05C7\\u0610-\\u061A\\u064B-\\u065E\\u0670\\u06D6-\\u06DC\\u06DF-\\u06E4\\u06E7\\u06E8\\u06EA-\\u06ED\\u0711\\u0730-\\u074A\\u07A6-\\u07B0\\u07EB-\\u07F3\\u0816-\\u0819\\u081B-\\u0823\\u0825-\\u0827\\u0829-\\u082D\\u0900-\\u0902\\u093C\\u0941-\\u0948\\u094D\\u0951-\\u0955\\u0962\\u0963\\u0981\\u09BC\\u09C1-\\u09C4\\u09CD\\u09E2\\u09E3\\u0A01\\u0A02\\u0A3C\\u0A41\\u0A42\\u0A47\\u0A48\\u0A4B-\\u0A4D\\u0A51\\u0A70\\u0A71\\u0A75\\u0A81\\u0A82\\u0ABC\\u0AC1-\\u0AC5\\u0AC7\\u0AC8\\u0ACD\\u0AE2\\u0AE3\\u0B01\\u0B3C\\u0B3F\\u0B41-\\u0B44\\u0B4D\\u0B56\\u0B62\\u0B63\\u0B82\\u0BC0\\u0BCD\\u0C3E-\\u0C40\\u0C46-\\u0C48\\u0C4A-\\u0C4D\\u0C55\\u0C56\\u0C62\\u0C63\\u0CBC\\u0CBF\\u0CC6\\u0CCC\\u0CCD\\u0CE2\\u0CE3\\u0D41-\\u0D44\\u0D4D\\u0D62\\u0D63\\u0DCA\\u0DD2-\\u0DD4\\u0DD6\\u0E31\\u0E34-\\u0E3A\\u0E47-\\u0E4E\\u0EB1\\u0EB4-\\u0EB9\\u0EBB\\u0EBC\\u0EC8-\\u0ECD\\u0F18\\u0F19\\u0F35\\u0F37\\u0F39\\u0F71-\\u0F7E\\u0F80-\\u0F84\\u0F86\\u0F87\\u0F90-\\u0F97\\u0F99-\\u0FBC\\u0FC6\\u102D-\\u1030\\u1032-\\u1037\\u1039\\u103A\\u103D\\u103E\\u1058\\u1059\\u105E-\\u1060\\u1071-\\u1074\\u1082\\u1085\\u1086\\u108D\\u109D\\u135F\\u1712-\\u1714\\u1732-\\u1734\\u1752\\u1753\\u1772\\u1773\\u17B7-\\u17BD\\u17C6\\u17C9-\\u17D3\\u17DD\\u180B-\\u180D\\u18A9\\u1920-\\u1922\\u1927\\u1928\\u1932\\u1939-\\u193B\\u1A17\\u1A18\\u1A56\\u1A58-\\u1A5E\\u1A60\\u1A62\\u1A65-\\u1A6C\\u1A73-\\u1A7C\\u1A7F\\u1B00-\\u1B03\\u1B34\\u1B36-\\u1B3A\\u1B3C\\u1B42\\u1B6B-\\u1B73\\u1B80\\u1B81\\u1BA2-\\u1BA5\\u1BA8\\u1BA9\\u1C2C-\\u1C33\\u1C36\\u1C37\\u1CD0-\\u1CD2\\u1CD4-\\u1CE0\\u1CE2-\\u1CE8\\u1CED\\u1DC0-\\u1DE6\\u1DFD-\\u1DFF\\u20D0-\\u20DC\\u20E1\\u20E5-\\u20F0\\u2CEF-\\u2CF1\\u2DE0-\\u2DFF\\u302A-\\u302F\\u3099\\u309A\\uA66F\\uA67C\\uA67D\\uA6F0\\uA6F1\\uA802\\uA806\\uA80B\\uA825\\uA826\\uA8C4\\uA8E0-\\uA8F1\\uA926-\\uA92D\\uA947-\\uA951\\uA980-\\uA982\\uA9B3\\uA9B6-\\uA9B9\\uA9BC\\uAA29-\\uAA2E\\uAA31\\uAA32\\uAA35\\uAA36\\uAA43\\uAA4C\\uAAB0\\uAAB2-\\uAAB4\\uAAB7\\uAAB8\\uAABE\\uAABF\\uAAC1\\uABE5\\uABE8\\uABED\\uFB1E\\uFE00-\\uFE0F\\uFE20-\\uFE26]'),
+    'space_combining_mark': RegExp('[\\u0903\\u093E-\\u0940\\u0949-\\u094C\\u094E\\u0982\\u0983\\u09BE-\\u09C0\\u09C7\\u09C8\\u09CB\\u09CC\\u09D7\\u0A03\\u0A3E-\\u0A40\\u0A83\\u0ABE-\\u0AC0\\u0AC9\\u0ACB\\u0ACC\\u0B02\\u0B03\\u0B3E\\u0B40\\u0B47\\u0B48\\u0B4B\\u0B4C\\u0B57\\u0BBE\\u0BBF\\u0BC1\\u0BC2\\u0BC6-\\u0BC8\\u0BCA-\\u0BCC\\u0BD7\\u0C01-\\u0C03\\u0C41-\\u0C44\\u0C82\\u0C83\\u0CBE\\u0CC0-\\u0CC4\\u0CC7\\u0CC8\\u0CCA\\u0CCB\\u0CD5\\u0CD6\\u0D02\\u0D03\\u0D3E-\\u0D40\\u0D46-\\u0D48\\u0D4A-\\u0D4C\\u0D57\\u0D82\\u0D83\\u0DCF-\\u0DD1\\u0DD8-\\u0DDF\\u0DF2\\u0DF3\\u0F3E\\u0F3F\\u0F7F\\u102B\\u102C\\u1031\\u1038\\u103B\\u103C\\u1056\\u1057\\u1062-\\u1064\\u1067-\\u106D\\u1083\\u1084\\u1087-\\u108C\\u108F\\u109A-\\u109C\\u17B6\\u17BE-\\u17C5\\u17C7\\u17C8\\u1923-\\u1926\\u1929-\\u192B\\u1930\\u1931\\u1933-\\u1938\\u19B0-\\u19C0\\u19C8\\u19C9\\u1A19-\\u1A1B\\u1A55\\u1A57\\u1A61\\u1A63\\u1A64\\u1A6D-\\u1A72\\u1B04\\u1B35\\u1B3B\\u1B3D-\\u1B41\\u1B43\\u1B44\\u1B82\\u1BA1\\u1BA6\\u1BA7\\u1BAA\\u1C24-\\u1C2B\\u1C34\\u1C35\\u1CE1\\u1CF2\\uA823\\uA824\\uA827\\uA880\\uA881\\uA8B4-\\uA8C3\\uA952\\uA953\\uA983\\uA9B4\\uA9B5\\uA9BA\\uA9BB\\uA9BD-\\uA9C0\\uAA2F\\uAA30\\uAA33\\uAA34\\uAA4D\\uAA7B\\uABE3\\uABE4\\uABE6\\uABE7\\uABE9\\uABEA\\uABEC]'),
+    'connector_punctuation': RegExp('[\\u005F\\u203F\\u2040\\u2054\\uFE33\\uFE34\\uFE4D-\\uFE4F\\uFF3F]')
 }  # }}}
 
 
@@ -153,10 +163,13 @@ def is_token(token, type, val):
 
 EX_EOF = {}
 
-def tokenizer(raw_text, filename):
+
+def tokenizer(raw_text, filename, recover_errors):
     S = {
-        'text': raw_text.replace(/\r\n?|[\n\u2028\u2029]/g, "\n").replace(/\uFEFF/g, ""),
+        'text': raw_text.replace(/\r\n?|[\n\u2028\u2029]/g, '\n').replace(/\uFEFF/g, ''),
         'filename': filename,
+        'recover_errors': not not recover_errors,  # LSP error-recovery mode (see parse())
+        'recovered_errors': v'[]',  # collected errors when recover_errors is on
         'pos': 0,
         'tokpos': 0,
         'line': 1,
@@ -170,11 +183,12 @@ def tokenizer(raw_text, filename):
         'newblock': False,
         'endblock': False,
         'indentation_matters': v'[ true ]',
-        'cached_whitespace': "",
+        'cached_whitespace': '',
         'prev': undefined,
         'index_or_slice': v'[ false ]',
         'expecting_object_literal_key': False,  # This is set by the parser when it is expecting an object literal key
     }
+
     def peek():
         return S.text.charAt(S.pos)
 
@@ -187,7 +201,7 @@ def tokenizer(raw_text, filename):
         if signal_eof and not ch:
             raise EX_EOF
 
-        if ch is "\n":
+        if ch is '\n':
             S.newline_before = S.newline_before or not in_string
             S.line += 1
             S.col = 0
@@ -207,15 +221,15 @@ def tokenizer(raw_text, filename):
         S.tokpos = S.pos
 
     def token(type, value, is_comment, keep_newline):
-        S.regex_allowed = (type is "operator"
-            or type is "keyword" and KEYWORDS_BEFORE_EXPRESSION[value]
-            or type is "punc" and PUNC_BEFORE_EXPRESSION[value])
+        S.regex_allowed = (type is 'operator'
+            or type is 'keyword' and KEYWORDS_BEFORE_EXPRESSION[value]
+            or type is 'punc' and PUNC_BEFORE_EXPRESSION[value])
 
-        if type is "operator" and value is "is" and S.text.substr(S.pos).trimLeft().substr(0, 4).trimRight() is "not":
+        if type is 'operator' and value is 'is' and S.text.substr(S.pos).trimLeft().substr(0, 4).trimRight() is 'not':
             next_token()
-            value = "!=="
+            value = '!=='
 
-        if type is "operator" and OP_MAP[value]:
+        if type is 'operator' and OP_MAP[value]:
             value = OP_MAP[value]
 
         ret = {
@@ -239,29 +253,29 @@ def tokenizer(raw_text, filename):
         if not keep_newline:
             S.newline_before = False
 
-        if type is "punc":
+        if type is 'punc':
             #            if (value is ":" && peek() is "\n") {
-            if value is ":" and not S.index_or_slice[-1]
+            if value is ':' and not S.index_or_slice[-1]
             and not S.expecting_object_literal_key
-            and (not S.text.substring(S.pos + 1, find("\n")).trim() or not S.text.substring(S.pos + 1, find("#")).trim()):
+            and (not S.text.substring(S.pos + 1, find('\n')).trim() or not S.text.substring(S.pos + 1, find('#')).trim()):
                 S.newblock = True
                 S.indentation_matters.push(True)
 
-            if value is "[":
+            if value is '[':
                 if S.prev and (
-                    S.prev.type is "name" or
+                    S.prev.type is 'name' or
                     (S.prev.type is 'punc' and ')]'.indexOf(S.prev.value) is not -1)
                 ):
                     S.index_or_slice.push(True)
                 else:
                     S.index_or_slice.push(False)
                 S.indentation_matters.push(False)
-            elif value is "{" or value is "(":
+            elif value is '{' or value is '(':
                 S.indentation_matters.push(False)
-            elif value is "]":
+            elif value is ']':
                 S.index_or_slice.pop()
                 S.indentation_matters.pop()
-            elif value is "}" or value is ")":
+            elif value is '}' or value is ')':
                 S.indentation_matters.pop()
         S.prev = AST_Token(ret)
         return S.prev
@@ -269,16 +283,16 @@ def tokenizer(raw_text, filename):
     # this will transform leading whitespace to block tokens unless
     # part of array/hash, and skip non-leading whitespace
     def parse_whitespace():
-        leading_whitespace = ""
+        leading_whitespace = ''
         whitespace_exists = False
         while WHITESPACE_CHARS[peek()]:
             whitespace_exists = True
             ch = next()
-            if ch is "\n":
-                leading_whitespace = ""
+            if ch is '\n':
+                leading_whitespace = ''
             else:
                 leading_whitespace += ch
-        if peek() is not "#":
+        if peek() is not '#':
             if not whitespace_exists:
                 leading_whitespace = S.cached_whitespace
             else:
@@ -287,7 +301,7 @@ def tokenizer(raw_text, filename):
                 return test_indent_token(leading_whitespace)
 
     def test_indent_token(leading_whitespace):
-        most_recent = S.whitespace_before[-1] or ""
+        most_recent = S.whitespace_before[-1] or ''
         S.endblock = False
         if S.indentation_matters[-1] and leading_whitespace is not most_recent:
             if S.newblock and leading_whitespace and leading_whitespace.indexOf(most_recent) is 0:
@@ -302,11 +316,11 @@ def tokenizer(raw_text, filename):
                 return -1
             else:
                 # indent mismatch, inconsistent indentation
-                parse_error("Inconsistent indentation")
+                parse_error('Inconsistent indentation')
         return 0
 
     def read_while(pred):
-        ret = ""
+        ret = ''
         i = 0
         ch = ''
         while (ch = peek()) and pred(ch, i):
@@ -320,16 +334,16 @@ def tokenizer(raw_text, filename):
     def read_num(prefix):
         has_e = False
         has_x = False
-        has_dot = prefix is "."
+        has_dot = prefix is '.'
         if not prefix and peek() is '0' and S.text.charAt(S.pos + 1) is 'b':
             next(), next()
-            num = read_while(def(ch): return ch is '0' or ch is '1';)
+            num = read_while(def (ch): return ch is '0' or ch is '1';)
             valid = parseInt(num, 2)
             if isNaN(valid):
                 parse_error('Invalid syntax for a binary number')
             return token('num', valid)
         seen = v'[]'
-        num = read_while(def(ch, i):
+        num = read_while(def (ch, i):
             nonlocal has_dot, has_e, has_x
             seen.push(ch)
             if ch is 'x' or ch is 'X':
@@ -347,11 +361,11 @@ def tokenizer(raw_text, filename):
             elif ch is '-':
                 if i is 0 and not prefix:
                     return True
-                if has_e and seen[i-1].toLowerCase() is 'e':
+                if has_e and seen[i - 1].toLowerCase() is 'e':
                     return True
                 return False
             elif ch is '+':
-                if has_e and seen[i-1].toLowerCase() is 'e':
+                if has_e and seen[i - 1].toLowerCase() is 'e':
                     return True
                 return False
             elif ch is '.':
@@ -363,9 +377,9 @@ def tokenizer(raw_text, filename):
 
         valid = parse_js_number(num)
         if not isNaN(valid):
-            return token("num", valid)
+            return token('num', valid)
         else:
-            parse_error("Invalid syntax: " + num)
+            parse_error('Invalid syntax: ' + num)
 
     def read_hex_digits(count):
         ans = ''
@@ -415,7 +429,7 @@ def tokenizer(raw_text, filename):
                 if code <= 0xFFFF:
                     return String.fromCharCode(code)
                 code -= 0x10000
-                return String.fromCharCode(0xD800+(code>>10), 0xDC00+(code&0x3FF))
+                return String.fromCharCode(0xD800 + (code >> 10), 0xDC00 + (code & 0x3FF))
             return '\\U' + code
         if q is 'N' and peek() is '{':
             next()
@@ -430,11 +444,11 @@ def tokenizer(raw_text, filename):
             if code <= 0xFFFF:
                 return String.fromCharCode(code)
             code -= 0x10000
-            return String.fromCharCode(0xD800+(code>>10), 0xDC00+(code&0x3FF))
+            return String.fromCharCode(0xD800 + (code >> 10), 0xDC00 + (code & 0x3FF))
         return '\\' + q
 
     def with_eof_error(eof_error, cont):
-        return def():
+        return def ():
             try:
                 return cont.apply(None, arguments)
             except as ex:
@@ -443,10 +457,10 @@ def tokenizer(raw_text, filename):
                 else:
                     raise
 
-    read_string = with_eof_error("Unterminated string constant", def(is_raw_literal, is_js_literal):
+    read_string = with_eof_error('Unterminated string constant', def (is_raw_literal, is_js_literal):
         quote = next()
         tok_type = 'js' if is_js_literal else 'string'
-        ret = ""
+        ret = ''
         is_multiline = False
         if peek() is quote:
             # two quotes in a row
@@ -457,18 +471,15 @@ def tokenizer(raw_text, filename):
                 is_multiline = True
             else:
                 return token(tok_type, '')
-
         while True:
             ch = next(True, True)
             if not ch:
                 break
-            if ch is "\n" and not is_multiline:
-                parse_error("End of line while scanning string literal")
-
-            if ch is "\\":
+            if ch is '\n' and not is_multiline:
+                parse_error('End of line while scanning string literal')
+            if ch is '\\':
                 ret += ('\\' + next(True)) if is_raw_literal else read_escape_sequence()
                 continue
-
             if ch is quote:
                 if not is_multiline:
                     break
@@ -483,10 +494,13 @@ def tokenizer(raw_text, filename):
         return token(tok_type, ret)
     )
 
-    def handle_interpolated_string(string, start_tok):
+    def handle_interpolated_string(string, start_tok, is_fstring=True):
         def raise_error(err):
             raise new SyntaxError(err, filename, start_tok.line, start_tok.col, start_tok.pos, False)
-        parts = v'[interpolate(string, raise_error)]'
+        if is_fstring:
+            parts = v'[interpolate(string, raise_error)]'
+        else:
+            parts = v'[quoted_string(string)]'
         # Look ahead for consecutive string literals to concatenate (e.g. f'a'f'b' or f'a''b')
         while True:
             # Skip horizontal whitespace (spaces and tabs, not newlines)
@@ -530,7 +544,7 @@ def tokenizer(raw_text, filename):
     def read_line_comment(shebang):
         if not shebang:
             next()
-        i = find("\n")
+        i = find('\n')
 
         if i is -1:
             ret = S.text.substr(S.pos)
@@ -539,13 +553,13 @@ def tokenizer(raw_text, filename):
             ret = S.text.substring(S.pos, i)
             S.pos = i
 
-        return token("shebang" if shebang else "comment1", ret, True)
+        return token('shebang' if shebang else 'comment1', ret, True)
 
     def read_name():
-        name = ch = ""
+        name = ch = ''
         while (ch = peek()) is not None:
-            if ch is "\\":
-                if S.text.charAt(S.pos + 1) is "\n":
+            if ch is '\\':
+                if S.text.charAt(S.pos + 1) is '\n':
                     S.pos += 2
                     continue
                 break
@@ -555,21 +569,20 @@ def tokenizer(raw_text, filename):
                 break
         return name
 
-    read_regexp = with_eof_error("Unterminated regular expression", def():
+    read_regexp = with_eof_error('Unterminated regular expression', def ():
         prev_backslash = False
         regexp = ch = ''
         in_class = False
         verbose_regexp = False
         in_comment = False
-
         if peek() is '/':
             next(True)
             if peek() is '/':
-                verbose_regexp  = True
+                verbose_regexp = True
                 next(True)
-            else: # empty regexp (//)
+            else:  # empty regexp (//)
                 mods = read_name()
-                return token("regexp", RegExp(regexp, mods))
+                return token('regexp', RegExp(regexp, mods))
         while True:
             ch = next(True)
             if not ch:
@@ -579,15 +592,15 @@ def tokenizer(raw_text, filename):
                     in_comment = False
                 continue
             if prev_backslash:
-                regexp += "\\" + ch
+                regexp += '\\' + ch
                 prev_backslash = False
-            elif ch is "[":
+            elif ch is '[':
                 in_class = True
                 regexp += ch
-            elif ch is "]" and in_class:
+            elif ch is ']' and in_class:
                 in_class = False
                 regexp += ch
-            elif ch is "/" and not in_class:
+            elif ch is '/' and not in_class:
                 if verbose_regexp:
                     if peek() is not '/':
                         regexp += '\\/'
@@ -598,7 +611,7 @@ def tokenizer(raw_text, filename):
                         continue
                     next(True)
                 break
-            elif ch is "\\":
+            elif ch is '\\':
                 prev_backslash = True
             elif verbose_regexp and not in_class and ' \n\r\t'.indexOf(ch) is not -1:
                 pass
@@ -606,9 +619,8 @@ def tokenizer(raw_text, filename):
                 in_comment = True
             else:
                 regexp += ch
-
         mods = read_name()
-        return token("regexp", RegExp(regexp, mods))
+        return token('regexp', RegExp(regexp, mods))
     )
 
     def read_operator(prefix):
@@ -627,60 +639,78 @@ def tokenizer(raw_text, filename):
             # pretend that this is an operator as the tokenizer only allows
             # one character punctuation.
             return token('punc', op)
-        return token("operator", op)
+        return token('operator', op)
 
     def handle_slash():
         next()
-        return read_regexp("") if S.regex_allowed else read_operator("/")
+        return read_regexp('') if S.regex_allowed else read_operator('/')
 
     def handle_dot():
         next()
-        return read_num(".") if is_digit(peek().charCodeAt(0)) else token("punc", ".")
+        return read_num('.') if is_digit(peek().charCodeAt(0)) else token('punc', '.')
 
     def read_word():
         word = read_name()
-        return token("atom", word) if KEYWORDS_ATOM[word] else (token("name", word) if not KEYWORDS[word] else (token("operator", word) if OPERATORS[word] and prevChar() is not "." else token("keyword", word)))
+        return token(
+            'atom',
+            word
+        ) if KEYWORDS_ATOM[word] else (token('name', word) if not KEYWORDS[word] else (token('operator', word) if OPERATORS[word] and prevChar() is not '.' else token('keyword', word)))
 
     def next_token():
-
         indent = parse_whitespace()
         # if indent is 1:
         #     return token("punc", "{")
         if indent is -1:
-            return token("punc", "}", False, True)
+            return token('punc', '}', False, True)
 
         start_token()
         ch = peek()
         if not ch:
-            return token("eof")
+            return token('eof')
 
         code = ch.charCodeAt(0)
         tmp_ = code
-        if tmp_ is 34 or tmp_ is 39:    # double-quote (") or single quote (')
-            return read_string(False)
-        elif tmp_ is 35:                # pound-sign (#)
+        if tmp_ is 34 or tmp_ is 39:  # double-quote (") or single quote (')
+            stok = read_string(False)
+            # Look ahead to detect a following f-string (e.g. "a" f"b"), which would
+            # otherwise be parsed as a call expression "a"(...) after the tokenizer
+            # injects the f-string result as a parenthesised expression.
+            j = S.pos
+            while j < S.text.length and (S.text.charAt(j) is ' ' or S.text.charAt(j) is '\t'):
+                j += 1
+            if j < S.text.length and is_identifier_start(S.text.charAt(j).charCodeAt(0)):
+                k = j
+                while k < S.text.length and is_identifier_char(S.text.charAt(k)):
+                    k += 1
+                potential_mod = S.text.substring(j, k)
+                if is_string_modifier(potential_mod) and k < S.text.length and '\'"'.indexOf(S.text.charAt(k)) is not -1:
+                    mods_check = potential_mod.toLowerCase()
+                    if mods_check.indexOf('f') is not -1 and mods_check.indexOf('v') is -1:
+                        return handle_interpolated_string(stok.value, stok, False)
+            return stok
+        elif tmp_ is 35:  # pound-sign (#)
             if S.pos is 0 and S.text.charAt(1) is '!':
-                #shebang
+                # shebang
                 return read_line_comment(True)
             regex_allowed = S.regex_allowed
             S.comments_before.push(read_line_comment())
             S.regex_allowed = regex_allowed
             return next_token()
-        elif tmp_ is 46:                # dot (.)
+        elif tmp_ is 46:  # dot (.)
             return handle_dot()
-        elif tmp_ is 47:                # slash (/)
+        elif tmp_ is 47:  # slash (/)
             return handle_slash()
 
         if is_digit(code):
             return read_num()
 
         if PUNC_CHARS[ch]:
-            return token("punc", next())
+            return token('punc', next())
 
         if OPERATOR_CHARS[ch]:
             return read_operator()
 
-        if code is 92 and S.text.charAt(S.pos + 1) is "\n":
+        if code is 92 and S.text.charAt(S.pos + 1) is '\n':
             # backslash will consume the newline character that follows
             next()
             # backslash
@@ -703,9 +733,19 @@ def tokenizer(raw_text, filename):
                 tok.type = stok.type
             return tok
 
-        parse_error("Unexpected character «" + ch + "»")
+        if S.recover_errors:
+            # In error-recovery mode (used by the LSP) an unrecognized character
+            # must not raise, because re-tokenizing from the same position would
+            # loop forever (parse_error is raised before the character is
+            # consumed). Record it and skip the single offending character so the
+            # tokenizer always makes forward progress.
+            S.recovered_errors.push({'message': 'Unexpected character «' + ch + '»',
+                                     'line': S.tokline, 'col': S.tokcol, 'pos': S.tokpos, 'is_eof': False})
+            next()
+            return next_token()
+        parse_error('Unexpected character «' + ch + '»')
 
-    next_token.context = def(nc):
+    next_token.context = def (nc):
         nonlocal S
         if nc:
             S = nc