lizard 1.17.13__py2.py3-none-any.whl → 1.17.15__py2.py3-none-any.whl

lizard_languages/javascript.py

@@ -6,152 +6,14 @@ from .code_reader import CodeReader
 from .clike import CCppCommentsMixin
 from .js_style_regex_expression import js_style_regex_expression
 from .js_style_language_states import JavaScriptStyleLanguageStates
+from .typescript import TypeScriptReader, JSTokenizer
 
 
-class JavaScriptReader(CodeReader, CCppCommentsMixin):
+class JavaScriptReader(TypeScriptReader):
     # pylint: disable=R0903
 
-    ext = ['js', 'jsx']
+    ext = ['js']
     language_names = ['javascript', 'js']
 
-    @staticmethod
-    @js_style_regex_expression
-    def generate_tokens(source_code, addition='', token_class=None):
-        addition = addition +\
-            r"|(?:\$\w+)" + \
-            r"|(?:\<\/\w+\>)" + \
-            r"|`.*?`"
-        js_tokenizer = JSTokenizer()
-        for token in CodeReader.generate_tokens(
-                source_code, addition, token_class):
-            for tok in js_tokenizer(token):
-                yield tok
-
     def __init__(self, context):
         super(JavaScriptReader, self).__init__(context)
-        self.parallel_states = [JavaScriptStyleLanguageStates(context)]
-
-
-class Tokenizer(object):
-    def __init__(self):
-        self.sub_tokenizer = None
-        self._ended = False
-
-    def __call__(self, token):
-        if self.sub_tokenizer:
-            for tok in self.sub_tokenizer(token):
-                yield tok
-            if self.sub_tokenizer._ended:
-                self.sub_tokenizer = None
-            return
-        for tok in self.process_token(token):
-            yield tok
-
-    def stop(self):
-        self._ended = True
-
-    def process_token(self, token):
-        pass
-
-
-class JSTokenizer(Tokenizer):
-    def __init__(self):
-        super(JSTokenizer, self).__init__()
-        self.depth = 1
-
-    def process_token(self, token):
-        if token == "<":
-            self.sub_tokenizer = XMLTagWithAttrTokenizer()
-            return
-        if token == "{":
-            self.depth += 1
-        elif token == "}":
-            self.depth -= 1
-            if self.depth == 0:
-                self.stop()
-                # {} in JSX is not listed as token
-                # otherwise it will be regarded
-                # as JS object
-                return
-        yield token
-
-
-class XMLTagWithAttrTokenizer(Tokenizer):
-    def __init__(self):
-        super(XMLTagWithAttrTokenizer, self).__init__()
-        self.tag = None
-        self.state = self._global_state
-        self.cache = ['<']
-
-    def process_token(self, token):
-        self.cache.append(token)
-        if not token.isspace():
-            result = self.state(token)
-            if result is not None:
-                return result
-        return ()
-
-    def abort(self):
-        self.stop()
-        return self.cache
-
-    def flush(self):
-        tmp, self.cache = self.cache, []
-        return [''.join(tmp)]
-
-    def _global_state(self, token):
-        if not isidentifier(token):
-            return self.abort()
-        self.tag = token
-        self.state = self._after_tag
-
-    def _after_tag(self, token):
-        if token == '>':
-            self.state = self._body
-        elif token == "/":
-            self.state = self._expecting_self_closing
-        elif isidentifier(token):
-            self.state = self._expecting_equal_sign
-        else:
-            return self.abort()
-
-    def _expecting_self_closing(self, token):
-        if token == ">":
-            self.stop()
-            return self.flush()
-        return self.abort()
-
-    def _expecting_equal_sign(self, token):
-        if token == '=':
-            self.state = self._expecting_value
-        else:
-            return self.abort()
-
-    def _expecting_value(self, token):
-        if token[0] in "'\"":
-            self.state = self._after_tag
-        elif token == "{":
-            self.cache.append("}")
-            self.sub_tokenizer = JSTokenizer()
-            self.state = self._after_tag
-
-    def _body(self, token):
-        if token == "<":
-            self.sub_tokenizer = XMLTagWithAttrTokenizer()
-            self.cache.pop()
-            return self.flush()
-
-        if token.startswith("</"):
-            self.stop()
-            return self.flush()
-
-        if token == '{':
-            self.sub_tokenizer = JSTokenizer()
-            return self.flush()
-
-
-def isidentifier(token):
-    try:
-        return token.isidentifier()
-    except AttributeError:
-        return token.encode(encoding='UTF-8')[0].isalpha()

lizard_languages/js_style_language_states.py

@@ -11,8 +11,18 @@ class JavaScriptStyleLanguageStates(CodeStateMachine):  # pylint: disable=R0903
         self.last_tokens = ''
         self.function_name = ''
         self.started_function = None
+        self.as_object = False
 
     def _state_global(self, token):
+        if self.as_object:
+            if token == ':':
+                self.function_name = self.last_tokens
+                return
+            elif token == '(':
+                self._function(self.last_tokens)
+                self.next(self._function, token)
+                return
+
         if token in '.':
             self._state = self._field
             self.last_tokens += token
@@ -29,11 +39,11 @@ class JavaScriptStyleLanguageStates(CodeStateMachine):  # pylint: disable=R0903
             self.function_name = self.last_tokens
         elif token == "(":
             self.sub_state(
-                JavaScriptStyleLanguageStates(self.context))
+                self.__class__(self.context))
         elif token in '{':
             if self.started_function:
                 self.sub_state(
-                    JavaScriptStyleLanguageStates(self.context),
+                    self.__class__(self.context),
                     self._pop_function_from_stack)
             else:
                 self.read_object()
@@ -46,7 +56,12 @@ class JavaScriptStyleLanguageStates(CodeStateMachine):  # pylint: disable=R0903
         self.last_tokens = token
 
     def read_object(self):
-        self.sub_state(ES6ObjectStates(self.context))
+        def callback():
+            self.next(self._state_global)
+
+        object_reader = self.__class__(self.context)
+        object_reader.as_object = True
+        self.sub_state(object_reader, callback)
 
     def statemachine_before_return(self):
         self._pop_function_from_stack()
@@ -63,16 +78,14 @@ class JavaScriptStyleLanguageStates(CodeStateMachine):  # pylint: disable=R0903
             return
 
         self.sub_state(
-            JavaScriptStyleLanguageStates(self.context),
-            callback)
+            self.__class__(self.context), callback)
 
     def _expecting_statement_or_block(self, token):
         def callback():
             self.next(self._state_global)
         if token == "{":
             self.sub_state(
-                JavaScriptStyleLanguageStates(self.context),
-                callback)
+                self.__class__(self.context), callback)
         else:
             self.next(self._state_global, token)
 
@@ -116,17 +129,3 @@ class JavaScriptStyleLanguageStates(CodeStateMachine):  # pylint: disable=R0903
         if token != '{':
             self.started_function = None
         self.next(self._state_global, token)
-
-
-class ES6ObjectStates(JavaScriptStyleLanguageStates):  # pylint: disable=R0903
-    def __init__(self, context):
-        super(ES6ObjectStates, self).__init__(context)
-
-    def _state_global(self, token):
-        if token == ':':
-            self.function_name = self.last_tokens
-        elif token == '(':
-            self._function(self.last_tokens)
-            self.next(self._function, token)
-        else:
-            super(ES6ObjectStates, self)._state_global(token)

lizard_languages/js_style_regex_expression.py

@@ -6,18 +6,48 @@ import re
 
 
 def js_style_regex_expression(func):
-    def generate_tokens_with_regex(source_code, _=""):
+    def generate_tokens_with_regex(source_code, addition='', token_class=None):
         regx_regx = r"\/(\S*?[^\s\\]\/)+?(igm)*"
         regx_pattern = re.compile(regx_regx)
-        word_pattern = re.compile(r'\w+')
-        tokens = func(source_code, r"|"+regx_regx)
-        leading_by_word = False
-        for token in tokens:
-            if leading_by_word and regx_pattern.match(token):
-                for subtoken in func(token, _):
-                    yield subtoken
+        tokens = list(func(source_code, addition, token_class))
+        result = []
+        i = 0
+        while i < len(tokens):
+            token = tokens[i]
+            if token == '/':
+                # Check if this could be a regex pattern
+                is_regex = False
+                if i == 0:
+                    is_regex = True
+                elif i > 0:
+                    prev_token = tokens[i-1].strip()
+                    if prev_token and prev_token[-1] in '=,({[?:!&|;':
+                        is_regex = True
+
+                if is_regex:
+                    # This is likely a regex pattern start
+                    regex_tokens = [token]
+                    i += 1
+                    while i < len(tokens) and not tokens[i].endswith('/'):
+                        regex_tokens.append(tokens[i])
+                        i += 1
+                    if i < len(tokens):
+                        regex_tokens.append(tokens[i])
+                        i += 1
+                        # Check for regex flags
+                        if i < len(tokens) and re.match(r'^[igm]+$', tokens[i]):
+                            regex_tokens.append(tokens[i])
+                            i += 1
+                    combined = ''.join(regex_tokens)
+                    if regx_pattern.match(combined):
+                        result.append(combined)
+                    else:
+                        result.extend(regex_tokens)
+                else:
+                    # This is a division operator
+                    result.append(token)
             else:
-                yield token
-            if not token.isspace():
-                leading_by_word = word_pattern.match(token)
+                result.append(token)
+            i += 1
+        return result
     return generate_tokens_with_regex

lizard_languages/jsx.py

@@ -0,0 +1,127 @@
+'''
+Language parser for JSX
+'''
+
+from .javascript import JavaScriptReader
+from .typescript import JSTokenizer, Tokenizer
+from .code_reader import CodeReader
+from .js_style_regex_expression import js_style_regex_expression
+
+
+class JSXMixin:
+    '''Base mixin class for JSX/TSX shared functionality'''
+    @staticmethod
+    @js_style_regex_expression
+    def generate_tokens(source_code, addition='', token_class=None):
+        addition = addition +\
+            r"|(?:\$\w+)" + \
+            r"|(?:\<\/\w+\>)" + \
+            r"|`.*?`"
+        js_tokenizer = JSTokenizer()
+        for token in CodeReader.generate_tokens(
+                source_code, addition, token_class):
+            for tok in js_tokenizer(token):
+                yield tok
+
+    def _expecting_func_opening_bracket(self, token):
+        if token == '<':
+            self.next(self._expecting_jsx)
+            return
+        super()._expecting_func_opening_bracket(token)
+
+    def _expecting_jsx(self, token):
+        if token == '>':
+            self.next(self._expecting_func_opening_bracket)
+
+
+class JSXReader(JavaScriptReader, JSXMixin):
+    # pylint: disable=R0903
+
+    ext = ['jsx']
+    language_names = ['jsx']
+
+    @staticmethod
+    @js_style_regex_expression
+    def generate_tokens(source_code, addition='', token_class=None):
+        return JSXMixin.generate_tokens(source_code, addition, token_class)
+
+
+class XMLTagWithAttrTokenizer(Tokenizer):
+    def __init__(self):
+        super(XMLTagWithAttrTokenizer, self).__init__()
+        self.tag = None
+        self.state = self._global_state
+        self.cache = ['<']
+
+    def process_token(self, token):
+        self.cache.append(token)
+        if not token.isspace():
+            result = self.state(token)
+            if result is not None:
+                return result
+        return ()
+
+    def abort(self):
+        self.stop()
+        return self.cache
+
+    def flush(self):
+        tmp, self.cache = self.cache, []
+        return [''.join(tmp)]
+
+    def _global_state(self, token):
+        if not isidentifier(token):
+            return self.abort()
+        self.tag = token
+        self.state = self._after_tag
+
+    def _after_tag(self, token):
+        if token == '>':
+            self.state = self._body
+        elif token == "/":
+            self.state = self._expecting_self_closing
+        elif isidentifier(token):
+            self.state = self._expecting_equal_sign
+        else:
+            return self.abort()
+
+    def _expecting_self_closing(self, token):
+        if token == ">":
+            self.stop()
+            return self.flush()
+        return self.abort()
+
+    def _expecting_equal_sign(self, token):
+        if token == '=':
+            self.state = self._expecting_value
+        else:
+            return self.abort()
+
+    def _expecting_value(self, token):
+        if token[0] in "'\"":
+            self.state = self._after_tag
+        elif token == "{":
+            self.cache.append("}")
+            self.sub_tokenizer = JSTokenizer()
+            self.state = self._after_tag
+
+    def _body(self, token):
+        if token == "<":
+            self.sub_tokenizer = XMLTagWithAttrTokenizer()
+            self.cache.pop()
+            return self.flush()
+
+        if token.startswith("</"):
+            self.stop()
+            return self.flush()
+
+        if token == '{':
+            self.sub_tokenizer = JSTokenizer()
+            return self.flush()
+
+
+def isidentifier(token):
+    try:
+        return token.isidentifier()
+    except AttributeError:
+        return token.encode(encoding='UTF-8')[0].isalpha()

lizard_languages/python.py

@@ -13,7 +13,7 @@ class PythonIndents:  # pylint: disable=R0902
         self.indents = [0]
         self.context = context
 
-    def set_nesting(self, spaces, token = ""):
+    def set_nesting(self, spaces, token=""):
         while self.indents[-1] > spaces and (not token.startswith(")")):
             self.indents.pop()
             self.context.pop_nesting()
@@ -29,8 +29,10 @@ class PythonReader(CodeReader, ScriptLanguageMixIn):
 
     ext = ['py']
     language_names = ['python']
-    _conditions = set(['if', 'for', 'while', 'and', 'or',
-                      'elif', 'except', 'finally'])
+    _conditions = set([
+        'if', 'for', 'while', 'and', 'or',
+        'elif', 'except', 'finally'
+    ])
 
     def __init__(self, context):
         super(PythonReader, self).__init__(context)
@@ -39,8 +41,9 @@ class PythonReader(CodeReader, ScriptLanguageMixIn):
     @staticmethod
     def generate_tokens(source_code, addition='', token_class=None):
         return ScriptLanguageMixIn.generate_common_tokens(
-                source_code,
-                r"|\'\'\'.*?\'\'\'" + r'|\"\"\".*?\"\"\"', token_class)
+            source_code,
+            r"|\'\'\'.*?\'\'\'" + r'|\"\"\".*?\"\"\"',
+            token_class)
 
     def preprocess(self, tokens):
         indents = PythonIndents(self.context)
@@ -54,7 +57,8 @@ class PythonReader(CodeReader, ScriptLanguageMixIn):
                     else:
                         if not token.startswith('#'):
                             current_function = self.context.current_function
-                            if current_function.name == '*global*' or current_function.long_name.endswith(')'):
+                            if (current_function.name == '*global*' or
+                                    current_function.long_name.endswith(')')):
                                 indents.set_nesting(current_leading_spaces, token)
                         reading_leading_space = False
             else:

lizard_languages/script_language.py

@@ -4,7 +4,7 @@ Common behaviours of script languages
 from .code_reader import CodeReader
 
 
-class ScriptLanguageMixIn(object):
+class ScriptLanguageMixIn:
     # pylint: disable=R0903
 
     @staticmethod

lizard_languages/tsx.py

@@ -0,0 +1,24 @@
+'''
+Language parser for TSX
+'''
+
+from .typescript import TypeScriptReader, TypeScriptStates, JSTokenizer
+from .jsx import JSXMixin
+from .code_reader import CodeReader
+from .js_style_regex_expression import js_style_regex_expression
+
+
+class TSXReader(TypeScriptReader, JSXMixin):
+    # pylint: disable=R0903
+
+    ext = ['tsx']
+    language_names = ['tsx']
+
+    @staticmethod
+    @js_style_regex_expression
+    def generate_tokens(source_code, addition='', token_class=None):
+        return JSXMixin.generate_tokens(source_code, addition, token_class)
+
+    def __init__(self, context):
+        super(TSXReader, self).__init__(context)
+        # No need for parallel states since JSX handling is in the mixin

lizard_languages/typescript.py

@@ -3,12 +3,54 @@ Language parser for JavaScript
 '''
 
 import re
-from .code_reader import CodeReader
+from .code_reader import CodeReader, CodeStateMachine
 from .clike import CCppCommentsMixin
 from .js_style_language_states import JavaScriptStyleLanguageStates
 from .js_style_regex_expression import js_style_regex_expression
 
 
+class Tokenizer(object):
+    def __init__(self):
+        self.sub_tokenizer = None
+        self._ended = False
+
+    def __call__(self, token):
+        if self.sub_tokenizer:
+            for tok in self.sub_tokenizer(token):
+                yield tok
+            if self.sub_tokenizer._ended:
+                self.sub_tokenizer = None
+            return
+        for tok in self.process_token(token):
+            yield tok
+
+    def stop(self):
+        self._ended = True
+
+    def process_token(self, token):
+        pass
+
+
+class JSTokenizer(Tokenizer):
+    def __init__(self):
+        super().__init__()
+        self.depth = 1
+
+    def process_token(self, token):
+        if token == "<":
+            from .jsx import XMLTagWithAttrTokenizer  # Import only when needed
+            self.sub_tokenizer = XMLTagWithAttrTokenizer()
+            return
+        if token == "{":
+            self.depth += 1
+        elif token == "}":
+            self.depth -= 1
+            if self.depth == 0:
+                self.stop()
+                return
+        yield token
+
+
 class TypeScriptReader(CodeReader, CCppCommentsMixin):
     # pylint: disable=R0903
 
@@ -18,32 +60,65 @@ class TypeScriptReader(CodeReader, CCppCommentsMixin):
                        'catch', 'case'])
 
     def __init__(self, context):
-        super(TypeScriptReader, self).__init__(context)
+        super().__init__(context)
         self.parallel_states = [TypeScriptStates(context)]
 
     @staticmethod
     @js_style_regex_expression
     def generate_tokens(source_code, addition='', token_class=None):
         addition = addition +\
-            r"|(?:\w+\?)"
-        return CodeReader.generate_tokens(source_code, addition, token_class)
+            r"|(?:\$\w+)" + \
+            r"|(?:\w+\?)" + \
+            r"|`.*?`"
+        js_tokenizer = JSTokenizer()
+        for token in CodeReader.generate_tokens(
+                source_code, addition, token_class):
+            for tok in js_tokenizer(token):
+                yield tok
 
 
 class TypeScriptStates(JavaScriptStyleLanguageStates):
+    def __init__(self, context):
+        super().__init__(context)
+
+    def _state_global(self, token):
+        if not self.as_object:
+            if token == ':':
+                self._consume_type_annotation()
+                return
+        super()._state_global(token)
 
     def _expecting_func_opening_bracket(self, token):
         if token == ':':
-            self.next(self._expecting_default)
-            return
-        super(TypeScriptStates, self)._expecting_func_opening_bracket(token)
+            self._consume_type_annotation()
+        else:
+            super()._expecting_func_opening_bracket(token)
+
+    def _consume_type_annotation(self):
+        typeStates = TypeScriptTypeAnnotationStates(self.context)
+
+        def callback():
+            if typeStates.saved_token:
+                self(typeStates.saved_token)
+        self.sub_state(typeStates, callback)
 
-    def _expecting_default(self, token):
-        self.next(self._function_return_type)
+
+class TypeScriptTypeAnnotationStates(CodeStateMachine):
+    def __init__(self, context):
+        super().__init__(context)
+        self.saved_token = None
+
+    def _state_global(self, token):
         if token == '{':
-            self.read_object()
+            self.next(self._inline_type_annotation, token)
+        else:
+            self.next(self._state_simple_type, token)
+
+    def _state_simple_type(self, token):
+        if token in '{=;':
+            self.saved_token = token
+            self.statemachine_return()
 
-    def _function_return_type(self, token):
-        if token == ';':
-            self.next(self._state_global)
-        elif token == '{':
-            self.next(self._expecting_func_opening_bracket, token)
+    @CodeStateMachine.read_inside_brackets_then("{}")
+    def _inline_type_annotation(self, _):
+        self.statemachine_return()

lizard_languages/vue.py

@@ -0,0 +1,34 @@
+'''
+Language parser for Vue.js files
+'''
+
+from .code_reader import CodeReader, CodeStateMachine
+from .typescript import TypeScriptReader
+
+
+class VueReader(TypeScriptReader):
+    # pylint: disable=R0903
+
+    ext = ['vue']
+    language_names = ['vue', 'vuejs']
+
+    def __init__(self, context):
+        super(VueReader, self).__init__(context)
+
+    @staticmethod
+    def generate_tokens(source_code, addition='', token_class=None):
+        # Use the base token generator but ensure we capture Vue block tags
+        addition = addition + r"|(?:\<\/?\w+.*?\>)"
+        return TypeScriptReader.generate_tokens(source_code, addition, token_class)
+
+    def preprocess(self, tokens):
+        current_block = None
+
+        for token in tokens:
+            if token.startswith('<script'):
+                current_block = 'script'
+            elif token.startswith('</script'):
+                current_block = None
+            elif current_block == 'script':
+                if not token.isspace() or token == '\n':
+                    yield token