python-minifier 2.11.2__py2-none-any.whl

python_minifier/rename/resolve_names.py

@@ -0,0 +1,106 @@
+import python_minifier.ast_compat as ast
+
+from python_minifier.rename.binding import BuiltinBinding, NameBinding
+from python_minifier.rename.util import get_global_namespace, get_nonlocal_namespace, builtins
+from python_minifier.util import is_ast_node
+
+
+def get_binding(name, namespace):
+    if name in namespace.global_names and not isinstance(namespace, ast.Module):
+        return get_binding(name, get_global_namespace(namespace))
+    elif name in namespace.nonlocal_names and not isinstance(namespace, ast.Module):
+        return get_binding(name, get_nonlocal_namespace(namespace))
+
+    for binding in namespace.bindings:
+        if binding.name == name:
+            return binding
+
+    if not isinstance(namespace, ast.Module):
+        return get_binding(name, get_nonlocal_namespace(namespace))
+
+    else:
+        # This is unresolved at global scope - is it a builtin?
+        if name in dir(builtins):
+            if name in ['exec', 'eval', 'locals', 'globals', 'vars']:
+                namespace.tainted = True
+
+            binding = BuiltinBinding(name, namespace)
+            namespace.bindings.append(binding)
+            return binding
+
+        else:
+            binding = NameBinding(name)
+            binding.disallow_rename()
+            namespace.bindings.append(binding)
+            return binding
+
+def get_binding_disallow_class_namespace_rename(name, namespace):
+    binding = get_binding(name, namespace)
+
+    if isinstance(namespace, ast.ClassDef):
+        # This name will become an attribute of a class, so it can't be renamed
+        binding.disallow_rename()
+
+    return binding
+
+def resolve_names(node):
+    """
+    Resolve unbound names to a NameBinding
+
+    :param node: The module to resolve names in
+    :type node: :class:`ast.Module`
+
+    """
+
+    if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load):
+        get_binding(node.id, node.namespace).add_reference(node)
+    elif isinstance(node, ast.Name) and node.id in node.namespace.nonlocal_names:
+        binding = get_binding(node.id, node.namespace)
+        binding.add_reference(node)
+
+        if isinstance(node.ctx, ast.Store) and isinstance(node.namespace, ast.ClassDef):
+            binding.disallow_rename()
+
+    elif isinstance(node, ast.ClassDef) and node.name in node.namespace.nonlocal_names:
+        binding = get_binding_disallow_class_namespace_rename(node.name, node.namespace)
+        binding.add_reference(node)
+
+    elif is_ast_node(node, (ast.FunctionDef, 'AsyncFunctionDef')) and node.name in node.namespace.nonlocal_names:
+        binding = get_binding_disallow_class_namespace_rename(node.name, node.namespace)
+        binding.add_reference(node)
+
+    elif isinstance(node, ast.alias):
+
+        if node.asname is not None:
+            if node.asname in node.namespace.nonlocal_names:
+                binding = get_binding_disallow_class_namespace_rename(node.asname, node.namespace)
+                binding.add_reference(node)
+
+        else:
+            # This binds the root module only for a dotted import
+            root_module = node.name.split('.')[0]
+
+            if root_module in node.namespace.nonlocal_names:
+                binding = get_binding_disallow_class_namespace_rename(root_module, node.namespace)
+                binding.add_reference(node)
+
+                if '.' in node.name:
+                    binding.disallow_rename()
+
+    elif isinstance(node, ast.ExceptHandler) and node.name is not None:
+        if isinstance(node.name, str) and node.name in node.namespace.nonlocal_names:
+            get_binding_disallow_class_namespace_rename(node.name, node.namespace).add_reference(node)
+
+    elif is_ast_node(node, 'Nonlocal'):
+        for name in node.names:
+            get_binding_disallow_class_namespace_rename(name, node.namespace).add_reference(node)
+    elif is_ast_node(node, ('MatchAs', 'MatchStar')) and node.name in node.namespace.nonlocal_names:
+        get_binding_disallow_class_namespace_rename(node.name, node.namespace).add_reference(node)
+    elif is_ast_node(node, 'MatchMapping') and node.rest in node.namespace.nonlocal_names:
+        get_binding_disallow_class_namespace_rename(node.rest, node.namespace).add_reference(node)
+
+    elif is_ast_node(node, 'Exec'):
+        get_global_namespace(node).tainted = True
+
+    for child in ast.iter_child_nodes(node):
+        resolve_names(child)

python_minifier/rename/util.py

@@ -0,0 +1,203 @@
+import python_minifier.ast_compat as ast
+import sys
+
+from python_minifier.util import is_ast_node
+
+
+def create_is_namespace():
+
+    namespace_nodes = (ast.FunctionDef, ast.Lambda, ast.ClassDef, ast.Module, ast.GeneratorExp)
+
+    if sys.version_info >= (2, 7):
+        namespace_nodes += (ast.SetComp, ast.DictComp)
+
+    if sys.version_info >= (3, 0):
+        namespace_nodes += (ast.ListComp,)
+
+    if sys.version_info >= (3, 5):
+        namespace_nodes += (ast.AsyncFunctionDef,)
+
+    return lambda node: isinstance(node, namespace_nodes)
+
+
+is_namespace = create_is_namespace()
+
+
+def iter_child_namespaces(node):
+
+    for child in ast.iter_child_nodes(node):
+        if is_namespace(child):
+            yield child
+        else:
+            for c in iter_child_namespaces(child):
+                yield c
+
+def get_global_namespace(node):
+    """
+    Return the global namespace for a node
+
+    :rtype: :class:`ast.Module`
+
+    """
+
+    if node.namespace is node:
+        return node
+
+    return get_global_namespace(node.namespace)
+
+
+def get_nonlocal_namespace(node):
+    """
+    Return the nonlocal namespace for a node
+
+    The nonlocal namespace is the closest parent function scope's namespace.
+    """
+
+    if isinstance(node.namespace, ast.ClassDef):
+        return get_nonlocal_namespace(node.namespace)
+
+    return node.namespace
+
+
+def arg_rename_in_place(node):
+    """
+    Can this argument node by safely renamed
+
+    'self', 'cls', 'args', and 'kwargs' are not commonly referenced by the caller, so
+    can be safely renamed. Comprehension arguments are not accessible from outside, so
+    can be renamed.
+
+    If the argument is positional-only, it can be safely renamed
+
+    Other arguments may be referenced by the caller as keyword arguments, so should not be
+    renamed in place. The name assigner may still decide to bind the argument to a new name
+    inside the function namespace.
+
+    :param node: The argument node
+    :rtype node: :class:`ast.arg`
+    :rtype: bool
+
+    """
+
+    func = node.namespace
+
+    if isinstance(func, ast.comprehension):
+        return True
+
+    if isinstance(func.namespace, ast.ClassDef) and not isinstance(func, ast.Lambda):
+        all_args = (func.args.posonlyargs if hasattr(func.args, 'posonlyargs') else []) + func.args.args
+        if len(all_args) > 0 and node is all_args[0]:
+            if len(func.decorator_list) == 0:
+                # rename 'self'
+                return True
+            elif (
+                len(func.decorator_list) == 1
+                and isinstance(func.decorator_list[0], ast.Name)
+                and func.decorator_list[0].id == 'classmethod'
+            ):
+                # rename 'cls'
+                return True
+
+    if func.args.vararg is node or func.args.kwarg is node:
+        # starargs
+        return True
+
+    if hasattr(func.args, 'posonlyargs') and node in func.args.posonlyargs:
+        return True
+
+    return False
+
+
+def insert(suite, new_node):
+    """
+    Insert a node into a suite
+
+    Inserts new_node as early as possible in the suite, but after docstrings and `import __future__` statements.
+
+    :param suite: The existing suite to insert the node into
+    :param new_node: The node to insert
+    :return: :class:`collections.Iterable[Node]`
+
+    """
+
+    inserted = False
+    for node in suite:
+
+        if not inserted:
+            if (isinstance(node, ast.ImportFrom) and node.module == '__future__') or (
+                isinstance(node, ast.Expr) and is_ast_node(node.value, ast.Str)
+            ):
+                pass
+            else:
+                yield new_node
+                inserted = True
+
+        yield node
+
+    if not inserted:
+        yield new_node
+
+
+def allow_rename_locals(node, rename_locals, preserve_locals=None):
+
+    if preserve_locals is None:
+        preserve_locals = []
+
+    if not isinstance(node, ast.Module) and is_namespace(node):
+        for binding in node.bindings:
+            if rename_locals is False:
+                binding.disallow_rename()
+            elif binding.name in preserve_locals:
+                binding.disallow_rename()
+
+    for child in ast.iter_child_nodes(node):
+        allow_rename_locals(child, rename_locals, preserve_locals)
+
+
+def find__all__(module):
+
+    names = []
+
+    def is_assign_all_node(node):
+        if isinstance(node, ast.Assign):
+            for name in node.targets:
+                if isinstance(name, ast.Name) and name.id == '__all__':
+                    return True
+
+        elif is_ast_node(node, (ast.AugAssign, 'AnnAssign')):
+            if isinstance(node.target, ast.Name) and node.target.id == '__all__':
+                return True
+
+        return False
+
+    for node in ast.iter_child_nodes(module):
+        if not is_assign_all_node(node):
+            continue
+
+        if not isinstance(node.value, ast.List):
+            continue
+
+        for el in node.value.elts:
+            if is_ast_node(el, ast.Str):
+                names.append(el.s)
+
+    return names
+
+
+def allow_rename_globals(module, rename_globals=False, preserve_globals=None):
+
+    if preserve_globals is None:
+        preserve_globals = []
+
+    preserve_globals.extend(find__all__(module))
+
+    for binding in module.bindings:
+        if rename_globals is False or binding.name in preserve_globals:
+            binding.disallow_rename()
+
+
+try:
+    import builtins
+except ImportError:
+    # noinspection PyCompatibility
+    import __builtin__ as builtins  # type: ignore

python_minifier/token_printer.py

@@ -0,0 +1,299 @@
+"""Tools for assembling python code from tokens."""
+
+import re
+import sys
+
+class TokenTypes(object):
+    NoToken = 0
+    Identifier = 1
+    Keyword = 2
+    SoftKeyword = 3
+    NumberLiteral = 4
+    NonNumberLiteral = 5
+    Delimiter = 6
+    Operator = 7
+    NewLine = 8
+    EndStatement = 9
+
+class Delimiter(object):
+    def __init__(self, terminal_printer, delimiter=',', add_parens=False):
+        """
+        Delimited group printer
+
+        A group of items that should be delimited by a delimiter character.
+        Each call to new_item() will insert the delimiter character if necessary.
+
+        When used as a context manager, the group will be enclosed by the start and end characters if the group has any items.
+
+        >>> d = Delimiter(terminal_printer)
+        ... d.new_item()
+        ... terminal_printer.identifier('a')
+        ... print(terminal_printer.code)
+        a
+
+        >>> d.new_item()
+        ... terminal_printer.identifier('b')
+        ... print(terminal_printer.code)
+        a,b
+
+        >>> with Delimiter(terminal_printer, add_parens=True) as d:
+        ...     d.new_item()
+        ...     terminal_printer.identifier('a')
+        ... print(terminal_printer.code)
+        (a)
+
+        :param terminal_printer: The terminal printer to use.
+        :param delimiter: The delimiter to use.
+        :param add_parens: If the group should be enclosed by parentheses. Only used when used as a context manager.
+        """
+
+        self._terminal_printer = terminal_printer
+        self._delimiter = delimiter
+        self._add_parens = add_parens
+
+        self._first = True
+
+        self._context_manager = False
+
+    def __enter__(self):
+        """Open a delimited group."""
+        self._context_manager = True
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Close the delimited group."""
+        if not self._first and self._add_parens:
+            self._terminal_printer.delimiter(')')
+
+    def new_item(self):
+        """Add a new item to the delimited group."""
+        if self._first:
+            self._first = False
+            if self._context_manager and self._add_parens:
+                self._terminal_printer.delimiter('(')
+        else:
+            self._terminal_printer.delimiter(self._delimiter)
+
+class TokenPrinter(object):
+    """
+    Concatenates terminal symbols of the python grammar
+    """
+
+    def __init__(self, prefer_single_line=False, allow_invalid_num_warnings=False):
+        """
+        :param prefer_single_line: If True, chooses to put as much code as possible on a single line.
+        :param allow_invalid_num_warnings: If True, allows invalid number literals to be printe that may cause warnings.
+        """
+
+        self._prefer_single_line = prefer_single_line
+        self._allow_invalid_num_warnings = allow_invalid_num_warnings
+
+        self._code = ''
+        self.indent = 0
+        self.unicode_literals = False
+        self.previous_token = TokenTypes.NoToken
+
+    def __str__(self):
+        """Return the output code."""
+        return self._code
+
+    def identifier(self, name):
+        """Add an identifier to the output code."""
+        assert isinstance(name, str)
+
+        if self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword, TokenTypes.NumberLiteral]:
+            self.delimiter(' ')
+
+        self._code += name
+        self.previous_token = TokenTypes.Identifier
+
+    def keyword(self, kw):
+        """Add a keyword to the output code."""
+        assert kw in [
+            'False', 'None', 'True', 'and', 'as',
+            'assert', 'async', 'await', 'break',
+            'class', 'continue', 'def', 'del',
+            'elif', 'else', 'except', 'finally',
+            'for', 'from', 'global', 'if', 'import',
+            'in', 'is', 'lambda', 'nonlocal', 'not',
+            'or', 'pass', 'raise', 'return',
+            'try', 'while', 'with', 'yield', '_',
+            'case', 'match', 'print', 'exec',
+            'type'
+        ]
+
+        if self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword, TokenTypes.NumberLiteral]:
+            self.delimiter(' ')
+
+        self._code += kw
+
+        if kw in ['_', 'case', 'match', 'type']:
+            self.previous_token = TokenTypes.SoftKeyword
+        else:
+            self.previous_token = TokenTypes.Keyword
+
+    def stringliteral(self, value):
+        """Add a string literal to the output code."""
+        s = repr(value)
+
+        if sys.version_info < (3, 0) and self.unicode_literals:
+            if s[0] == 'u':
+                # Remove the u prefix since literals are unicode by default
+                s = s[1:]
+            else:
+                # Add a b prefix to indicate it is NOT unicode
+                s = 'b' + s
+
+        if len(s) > 0 and s[0].isalpha() and self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword]:
+            self.delimiter(' ')
+
+        self._code += s
+        self.previous_token = TokenTypes.NonNumberLiteral
+
+    def bytesliteral(self, value):
+        """Add a bytes literal to the output code."""
+        s = repr(value)
+
+        if len(s) > 0 and s[0].isalpha() and self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword]:
+            self.delimiter(' ')
+
+        self._code += s
+        self.previous_token = TokenTypes.NonNumberLiteral
+
+    def fstring(self, s):
+        """Add an f-string to the output code."""
+        assert isinstance(s, str)
+
+        if self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword, TokenTypes.SoftKeyword]:
+            self.delimiter(' ')
+
+        self._code += s
+        self.previous_token = TokenTypes.NonNumberLiteral
+
+    def delimiter(self, d):
+        """Add a delimiter to the output code."""
+        assert d in [
+            '(', ')', '[', ']', '{', '}', ' ',
+            ',', ':', '.', ';', '@', '=', '->',
+            '+=', '-=', '*=', '/=', '//=', '%=', '@=',
+            '&=', '|=', '^=', '>>=', '<<=', '**=', '|',
+            '`'
+        ]
+
+        self._code += d
+        self.previous_token = TokenTypes.Delimiter
+
+    def operator(self, o):
+        """Add an operator to the output code."""
+        assert o in [
+            '+', '-', '*', '**', '/', '//', '%', '@',
+            '<<', '>>', '&', '|', '^', '~', ':=',
+            '<', '>', '<=', '>=', '==', '!='
+        ]
+
+        self._code += o
+        self.previous_token = TokenTypes.Operator
+
+    def integer(self, v):
+        """Add an integer to the output code."""
+
+        s = repr(v)
+        h = hex(v)
+
+        if self.previous_token == TokenTypes.SoftKeyword:
+            self.delimiter(' ')
+        elif self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword]:
+            self.delimiter(' ')
+
+        self._code += h if len(h) < len(s) else s
+
+        self.previous_token = TokenTypes.NumberLiteral
+
+    def imagnumber(self, value):
+        """Add a complex number to the output code."""
+        assert isinstance(value, complex)
+
+        s = repr(value)
+
+        if s in ['infj', 'inf*j']:
+            s = '1e999j'
+        elif s in ['-infj', '-inf*j']:
+            s = '-1e999j'
+
+        if self.previous_token == TokenTypes.SoftKeyword:
+            self.delimiter(' ')
+        elif self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword]:
+            self.delimiter(' ')
+
+        self._code += s
+
+        self.previous_token = TokenTypes.NumberLiteral
+
+    def floatnumber(self, v):
+        """Add a float to the output code."""
+        assert isinstance(v, float)
+
+        s = repr(v)
+
+        s = s.replace('e+', 'e')
+
+        add_e = re.match(r'^(\d+?)(0+).0$', s)
+        if add_e:
+            s = add_e.group(1) + 'e' + str(len(add_e.group(2)))
+
+        if s == 'inf':
+            s = '1e999'
+        elif s == '-inf':
+            s = '-1e999'
+        elif s.startswith('0.'):
+            s = s[1:]
+        elif s.startswith('-0.'):
+            s = '-' + s[2:]
+        elif s.endswith('.0'):
+            s = s[:-1]
+
+        if self.previous_token == TokenTypes.SoftKeyword:
+            self.delimiter(' ')
+        elif self.previous_token in [TokenTypes.Identifier, TokenTypes.Keyword]:
+            self.delimiter(' ')
+
+        self._code += s
+
+        self.previous_token = TokenTypes.NumberLiteral
+
+    def newline(self):
+        """ Add a newline to the code. """
+        if self._code == '':
+            return
+
+        self._code = self._code.rstrip('\n\t;')
+        self._code += '\n'
+        self._code += '\t' * self.indent
+
+        self.previous_token = TokenTypes.NewLine
+
+    def enter_block(self):
+        """Enter a new block, indenting the code."""
+        self.indent += 1
+        self.newline()
+
+    def leave_block(self):
+        """Leave a block, un-indenting the code."""
+        self.indent -= 1
+        self.newline()
+
+    def end_statement(self):
+        """ End a statement with a newline, or a semi-colon if it saves characters. """
+
+        if self.indent == 0:
+            self.newline()
+        else:
+            if self._code[-1] != ';':
+                self._code += ';'
+
+        self.previous_token = TokenTypes.EndStatement
+
+    def append(self, code, token_type):
+        """ Append arbitrary string to the output."""
+        self._code += code
+        self.previous_token = token_type

python_minifier/transforms/combine_imports.py

@@ -0,0 +1,76 @@
+import python_minifier.ast_compat as ast
+
+from python_minifier.transforms.suite_transformer import SuiteTransformer
+
+
+class CombineImports(SuiteTransformer):
+    """
+    Combine multiple import statements where possible
+
+    This doesn't change the order of imports
+
+    """
+
+    def _combine_import(self, node_list, parent):
+
+        alias = []
+        namespace = None
+
+        for statement in node_list:
+            namespace = statement.namespace
+            if isinstance(statement, ast.Import):
+                alias += statement.names
+            else:
+                if alias:
+                    yield self.add_child(ast.Import(names=alias), parent=parent, namespace=namespace)
+                    alias = []
+
+                yield statement
+
+        if alias:
+            yield self.add_child(ast.Import(names=alias), parent=parent, namespace=namespace)
+
+
+    def _combine_import_from(self, node_list, parent):
+
+        prev_import = None
+        alias = []
+
+        def combine(statement):
+            if not isinstance(statement, ast.ImportFrom):
+                return False
+
+            if len(statement.names) == 1 and statement.names[0].name == '*':
+                return False
+
+            if prev_import is None:
+                return True
+
+            if statement.module == prev_import.module and statement.level == prev_import.level:
+                return True
+
+            return False
+
+        for statement in node_list:
+            if combine(statement):
+                prev_import = statement
+                alias += statement.names
+            else:
+                if alias:
+                    yield self.add_child(
+                        ast.ImportFrom(module=prev_import.module, names=alias, level=prev_import.level), parent=parent, namespace=prev_import.namespace
+                    )
+                    alias = []
+
+                yield statement
+
+        if alias:
+            yield self.add_child(
+                ast.ImportFrom(module=prev_import.module, names=alias, level=prev_import.level), parent=parent, namespace=prev_import.namespace
+            )
+
+    def suite(self, node_list, parent):
+        a = list(self._combine_import(node_list, parent))
+        b = list(self._combine_import_from(a, parent))
+
+        return [self.visit(n) for n in b]