python-minifier 2.11.2__py2-none-any.whl

python_minifier/rename/mapper.py

@@ -0,0 +1,175 @@
+"""
+For each node in an AST set the namespace to use for name binding and resolution
+"""
+
+import python_minifier.ast_compat as ast
+
+from python_minifier.rename.util import is_namespace
+from python_minifier.util import is_ast_node
+
+
+def add_parent_to_arguments(arguments, func):
+    arguments.parent = func
+    arguments.namespace = func
+
+    for arg in getattr(arguments, 'posonlyargs', []) + arguments.args:
+        add_parent(arg, arguments, func)
+        if hasattr(arg, 'annotation') and arg.annotation is not None:
+            add_parent(arg.annotation, arguments, func.namespace)
+
+    if hasattr(arguments, 'kwonlyargs'):
+        for arg in arguments.kwonlyargs:
+            add_parent(arg, arguments, func)
+            if arg.annotation is not None:
+                add_parent(arg.annotation, arguments, func.namespace)
+
+        for node in arguments.kw_defaults:
+            if node is not None:
+                add_parent(node, arguments, func.namespace)
+
+    for node in arguments.defaults:
+        add_parent(node, arguments, func.namespace)
+
+    if arguments.vararg:
+        if hasattr(arguments, 'varargannotation') and arguments.varargannotation is not None:
+            add_parent(arguments.varargannotation, arguments, func.namespace)
+        elif isinstance(arguments.vararg, str):
+            pass
+        else:
+            add_parent(arguments.vararg, arguments, func)
+
+    if arguments.kwarg:
+        if hasattr(arguments, 'kwargannotation') and arguments.kwargannotation is not None:
+            add_parent(arguments.kwargannotation, arguments, func.namespace)
+        elif isinstance(arguments.kwarg, str):
+            pass
+        else:
+            add_parent(arguments.kwarg, arguments, func)
+
+
+def add_parent_to_functiondef(functiondef):
+    """
+    Add correct parent and namespace attributes to functiondef nodes
+    """
+
+    if functiondef.args is not None:
+        add_parent_to_arguments(functiondef.args, func=functiondef)
+
+    for node in functiondef.body:
+        add_parent(node, parent=functiondef, namespace=functiondef)
+
+    for node in functiondef.decorator_list:
+        add_parent(node, parent=functiondef, namespace=functiondef.namespace)
+
+    if hasattr(functiondef, 'type_params') and functiondef.type_params is not None:
+        for node in functiondef.type_params:
+            add_parent(node, parent=functiondef, namespace=functiondef.namespace)
+
+    if hasattr(functiondef, 'returns') and functiondef.returns is not None:
+        add_parent(functiondef.returns, parent=functiondef, namespace=functiondef.namespace)
+
+
+def add_parent_to_classdef(classdef):
+    """
+    Add correct parent and namespace attributes to classdef nodes
+    """
+
+    for node in classdef.bases:
+        add_parent(node, parent=classdef, namespace=classdef.namespace)
+
+    if hasattr(classdef, 'keywords'):
+        for node in classdef.keywords:
+            add_parent(node, parent=classdef, namespace=classdef.namespace)
+
+    if hasattr(classdef, 'starargs') and classdef.starargs is not None:
+        add_parent(classdef.starargs, parent=classdef, namespace=classdef.namespace)
+
+    if hasattr(classdef, 'kwargs') and classdef.kwargs is not None:
+        add_parent(classdef.kwargs, parent=classdef, namespace=classdef.namespace)
+
+    for node in classdef.body:
+        add_parent(node, parent=classdef, namespace=classdef)
+
+    for node in classdef.decorator_list:
+        add_parent(node, parent=classdef, namespace=classdef.namespace)
+
+    if hasattr(classdef, 'type_params') and classdef.type_params is not None:
+        for node in classdef.type_params:
+            add_parent(node, parent=classdef, namespace=classdef.namespace)
+
+def add_parent_to_comprehension(node, namespace):
+    assert is_ast_node(node, (ast.GeneratorExp, 'SetComp', 'DictComp', 'ListComp'))
+
+    if hasattr(node, 'elt'):
+        add_parent(node.elt, parent=node, namespace=node)
+    elif hasattr(node, 'key'):
+        add_parent(node.key, parent=node, namespace=node)
+        add_parent(node.value, parent=node, namespace=node)
+
+    iter_namespace = namespace
+    for generator in node.generators:
+        generator.parent = node
+        generator.namespace = node
+
+        add_parent(generator.target, parent=generator, namespace=node)
+        add_parent(generator.iter, parent=generator, namespace=iter_namespace)
+        iter_namespace = node
+        for if_ in generator.ifs:
+            add_parent(if_, parent=generator, namespace=node)
+
+
+def add_parent(node, parent=None, namespace=None):
+    """
+    Add a parent attribute to child nodes
+    Add a namespace attribute to child nodes
+
+    :param node: The tree to add parent and namespace properties to
+    :type node: :class:`ast.AST`
+    :param parent: The parent node of this node
+    :type parent: :class:`ast.AST`
+    :param namespace: The namespace Node that this node is in
+    :type namespace: ast.Lambda or ast.Module or ast.FunctionDef or ast.AsyncFunctionDef or ast.ClassDef or ast.DictComp or ast.SetComp or ast.ListComp or ast.Generator
+
+    """
+
+    node.parent = parent if parent is not None else node
+    node.namespace = namespace if namespace is not None else node
+
+    if is_namespace(node):
+        node.bindings = []
+        node.global_names = set()
+        node.nonlocal_names = set()
+
+        if is_ast_node(node, (ast.FunctionDef, 'AsyncFunctionDef')):
+            add_parent_to_functiondef(node)
+        elif is_ast_node(node, (ast.GeneratorExp, 'SetComp', 'DictComp', 'ListComp')):
+            add_parent_to_comprehension(node, namespace=namespace)
+        elif isinstance(node, ast.Lambda):
+            add_parent_to_arguments(node.args, func=node)
+            add_parent(node.body, parent=node, namespace=node)
+        elif isinstance(node, ast.ClassDef):
+            add_parent_to_classdef(node)
+        else:
+            for child in ast.iter_child_nodes(node):
+                add_parent(child, parent=node, namespace=node)
+
+        return
+
+    if isinstance(node, ast.Global):
+        namespace.global_names.update(node.names)
+    if is_ast_node(node, 'Nonlocal'):
+        namespace.nonlocal_names.update(node.names)
+
+    if isinstance(node, ast.Name):
+        if isinstance(namespace, ast.ClassDef):
+            if isinstance(node.ctx, ast.Load):
+                namespace.nonlocal_names.add(node.id)
+            elif isinstance(node.ctx, ast.Store) and isinstance(node.parent, ast.AugAssign):
+                namespace.nonlocal_names.add(node.id)
+
+    for child in ast.iter_child_nodes(node):
+        add_parent(child, parent=node, namespace=namespace)
+
+
+def add_namespace(module):
+    add_parent(module)

python_minifier/rename/name_generator.py

@@ -0,0 +1,51 @@
+import itertools
+import keyword
+import random
+import string
+
+from python_minifier.rename.util import builtins
+
+
+def random_generator(length=40):
+    valid_first = string.ascii_uppercase + string.ascii_lowercase
+    valid_rest = string.digits + valid_first + '_'
+
+    while True:
+        first = [random.choice(valid_first)]
+        rest = [random.choice(valid_rest) for i in range(length - 1)]
+        yield ''.join(first + rest)
+
+
+def name_generator():
+    valid_first = string.ascii_uppercase + string.ascii_lowercase
+    valid_rest = string.digits + valid_first + '_'
+
+    for c in valid_first:
+        yield c
+
+    for length in itertools.count(1):
+        for first in valid_first:
+            for rest in itertools.product(valid_rest, repeat=length):
+                name = first
+                name += ''.join(rest)
+                yield name
+
+
+def name_filter():
+    """
+    Yield all valid python identifiers
+
+    Name are returned sorted by length, then string sort order.
+
+    Names that already have meaning in python (keywords and builtins)
+    will not be included in the output.
+
+    :rtype: Iterable[str]
+
+    """
+
+    reserved = keyword.kwlist + dir(builtins)
+
+    for name in name_generator():
+        if name not in reserved:
+            yield name

python_minifier/rename/rename_literals.py

@@ -0,0 +1,249 @@
+import python_minifier.ast_compat as ast
+
+from python_minifier.rename.binding import Binding
+from python_minifier.rename.util import insert
+from python_minifier.transforms.suite_transformer import NodeVisitor
+from python_minifier.util import is_ast_node
+
+
+def replace(old_node, new_node):
+    parent = old_node.parent
+    new_node.parent = parent
+    new_node.namespace = old_node.namespace
+
+    for field, old_value in ast.iter_fields(parent):
+        if old_value is old_node:
+            setattr(parent, field, new_node)
+            return
+
+        if isinstance(old_value, list):
+            for i, value in enumerate(old_value):
+                if value is old_node:
+                    old_value[i] = new_node
+                    return
+
+
+class HoistedBinding(Binding):
+    def __init__(self, value_node, *args, **kwargs):
+        super(HoistedBinding, self).__init__(*args, **kwargs)
+        self._value_node = value_node
+        self._local_namespace = None
+
+    def __eq__(self, other):
+        return type(self.value) is type(other.value) and self.value == other.value
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __hash__(self):
+        return hash(repr(self.value))
+
+    def set_local_namespace(self, node):
+        self._local_namespace = node
+
+    @property
+    def value(self):
+        if is_ast_node(self._value_node, (ast.Str, 'Bytes')):
+            return self._value_node.s
+        else:
+            return self._value_node.value
+
+    def __repr__(self):
+        return self.__class__.__name__ + '(value=%r)' % self.value
+
+    def new_mention_count(self):
+        # All mentions must be literals, which would be replaced
+        # Plus an Assign with the new name
+        return len(self.references) + 1
+
+    def old_mention_count(self):
+        # For hoisted bindings, the old 'name' is the literal
+        # It would be mentioned once, in the Assign
+        return 1
+
+    def additional_byte_cost(self):
+        return 2  # '=' + '\n'
+
+    def rename(self, new_name):
+
+        for node in self.references:
+            replace(node, ast.Name(id=new_name, ctx=ast.Load()))
+
+        self._local_namespace.body = list(
+            insert(
+                self._local_namespace.body,
+                ast.Assign(targets=[ast.Name(id=new_name, ctx=ast.Store())], value=self._value_node),
+            )
+        )
+
+        self._name = new_name
+
+    def should_rename(self, new_name):
+        current_cost = len(self.references) * len(repr(self.value))
+        rename_cost = (self.old_mention_count() * len(repr(self.value))) + ((self.new_mention_count()) * len(new_name)) + self.additional_byte_cost()
+
+        return rename_cost <= current_cost
+
+class HoistedValue(object):
+    """
+    HoistedValue comparator object
+
+    This is for wrapping a value in a set or dict key, and
+    ensures different types hash differently, even if they compare equal.
+
+    The problematic values are str/bytes/unicode and int/float.
+
+    """
+
+    def __init__(self, value):
+        self._value = value
+
+    def __hash__(self):
+        return hash(str(type(self._value)) + str(hash(self._value)))
+
+    def __eq__(self, other):
+        return type(self._value) == type(other._value) and self._value == other._value
+
+    def __ne__(self, other):
+        return not self == other
+
+
+class HoistLiterals(NodeVisitor):
+    """
+    Hoist literal strings to module level variables
+    """
+
+    def __call__(self, module, ignore_slots=True):
+        self.module = module
+        self._ignore_slots = ignore_slots
+        self._hoisted = {}
+        self.visit(module)
+        self.place_bindings()
+
+    def nearest_function_namespace(self, node):
+        """
+        Return the namespace node for the nearest function scope.
+
+        This could be itself.
+
+        :param node: The node to get the function namespace of
+        :type node: ast.Node
+        :rtype: ast.Node
+
+        """
+
+        if is_ast_node(node.namespace, (ast.FunctionDef, ast.Module, 'AsyncFunctionDef')):
+            return node.namespace
+        return self.nearest_function_namespace(node.namespace)
+
+    def namespace_path(self, node):
+        """
+        Return the path of function namespace nodes from the module node down to the input node
+
+        With the source module:
+        >>> def a():
+        ...   def b():
+        ...     c
+
+        >>> namespace_path(c)
+        [a, b, c]
+
+        :param node:
+        :type node: ast.Node
+        :rtype: list[ast.AST]
+
+        """
+
+        l = []
+
+        while True:
+            namespace = self.nearest_function_namespace(node)
+            l.insert(0, namespace)
+
+            if isinstance(namespace, ast.Module):
+                break
+
+            node = namespace
+
+        return l
+
+    def common_path(self, n1_path, n2_path):
+
+        path = []
+        for n1_step, n2_step in zip(n1_path, n2_path):
+            if n1_step is not n2_step:
+                return path
+            path.append(n1_step)
+        return path
+
+    def place_bindings(self):
+        for binding in self._hoisted.values():
+
+            namespace_path = []
+
+            for node in binding.references:
+                if not namespace_path:
+                    namespace_path = self.namespace_path(node)
+                else:
+                    namespace_path = self.common_path(namespace_path, self.namespace_path(node))
+
+            namespace_path[-1].bindings.append(binding)
+            binding.set_local_namespace(namespace_path[-1])
+
+    def get_binding(self, value, node):
+        hoisted_value = HoistedValue(value)
+        if hoisted_value in self._hoisted:
+            return self._hoisted[hoisted_value]
+
+        binding = HoistedBinding(node)
+        self._hoisted[hoisted_value] = binding
+        return binding
+
+    def visit_Str(self, node):
+
+        if isinstance(node.parent, ast.Expr):
+            # This is literal statement
+            # The RemoveLiteralStatements transformer must have left it here, so ignore it.
+            return
+
+        self.get_binding(node.s, node).add_reference(node)
+
+    def visit_Bytes(self, node):
+        self.visit_Str(node)
+
+    def visit_JoinedStr(self, node):
+        for v in node.values:
+            if is_ast_node(v, ast.Str):
+                # Can't hoist this!
+                continue
+            else:
+                self.visit(v)
+
+    def visit_NameConstant(self, node):
+        self.get_binding(node.value, node).add_reference(node)
+
+    def visit_match_case(self, node):
+        # Can't hoist literals in a pattern
+
+        if node.guard is not None:
+            self.visit(node.guard)
+
+        for n in node.body:
+            self.visit(n)
+
+    def visit_Assign(self, node):
+        if not self._ignore_slots:
+            return self.generic_visit(node)
+
+        if not is_ast_node(node.namespace, ast.ClassDef):
+            return self.generic_visit(node)
+
+        for target in node.targets:
+            if is_ast_node(target, ast.Name) and target.id == '__slots__':
+                # This is a __slots__ assignment, don't hoist the literals
+                return
+
+        return self.generic_visit(node)
+
+def rename_literals(module):
+    HoistLiterals()(module)

python_minifier/rename/renamer.py

@@ -0,0 +1,230 @@
+import python_minifier.ast_compat as ast
+
+from python_minifier.rename.binding import NameBinding
+from python_minifier.rename.name_generator import name_filter
+from python_minifier.rename.util import is_namespace
+
+
+def all_bindings(node):
+    """
+    All bindings in a module
+
+    :param node: The module to get bindings in
+    :type node: :class:`ast.AST`
+    :rtype: Iterable[ast.AST, Binding]
+
+    """
+
+    if is_namespace(node):
+        for binding in node.bindings:
+            yield node, binding
+
+    for child in ast.iter_child_nodes(node):
+        for namespace, binding in all_bindings(child):
+            yield namespace, binding
+
+
+def sorted_bindings(module):
+    """
+    All bindings in a modules sorted by descending number of references
+
+    :param module: The module to get bindings in
+    :type module: :class:`ast.AST`
+    :rtype: Iterable[ast.AST, Binding]
+
+    """
+
+    def comp(tup):
+        namespace, binding = tup
+        return binding.new_mention_count()
+
+    return sorted(all_bindings(module), key=comp, reverse=True)
+
+
+def reservation_scope(namespace, binding):
+    """
+    Get the namespaces that are in the bindings reservation scope
+
+    Returns the namespace nodes the binding name must be resolvable in
+
+    :param namespace: The local namespace of a binding
+    :type namespace: :class:`ast.AST`
+    :param binding: The binding to get the reservation scope for
+    :type binding: Binding
+    :rtype: set[ast.AST]
+
+    """
+
+    namespaces = set([namespace])
+
+    for node in binding.references:
+        while node is not namespace:
+            namespaces.add(node.namespace)
+            node = node.namespace
+
+    return namespaces
+
+
+def add_assigned(node):
+    """
+    Add the assigned_names attribute to namespace nodes in a module
+
+    :param node: The module to add the assigned_names attribute to
+    :type node: :class:`ast.Module`
+
+    """
+
+    if is_namespace(node):
+        node.assigned_names = set()
+
+    for child in ast.iter_child_nodes(node):
+        add_assigned(child)
+
+
+def reserve_name(name, reservation_scope):
+    """
+    Reserve a name in a reservation scope
+
+    :param str name: The name to reserve
+    :param reservation_scope:
+    :type reservation_scope: Iterable[:class:`ast.AST`]
+
+    """
+
+    for namespace in reservation_scope:
+        namespace.assigned_names.add(name)
+
+
+class UniqueNameAssigner(object):
+    """
+    Assign new names to renamed bindings
+
+    Assigns a unique name to every binding
+    """
+
+    def __init__(self):
+        self.name_generator = name_filter()
+        self.names = []
+
+    def available_name(self):
+        return next(self.name_generator)
+
+    def __call__(self, module):
+        assert isinstance(module, ast.Module)
+
+        for namespace, binding in sorted_bindings(module):
+            if binding.allow_rename:
+                binding.new_name = self.available_name()
+
+        return module
+
+
+class NameAssigner(object):
+    """
+    Assign new names to renamed bindings
+
+    This assigner creates a name 'reservation scope' containing each namespace a binding is referenced in, including
+    transitive namespaces. Bindings are then assigned the first available name that has no references in their
+    reservation scope. This means names will be reused in sibling namespaces, and shadowed where possible in child
+    namespaces.
+
+    Bindings are assigned names in order of most references, with names assigned shortest first.
+
+    """
+
+    def __init__(self, name_generator=None):
+        self.name_generator = name_generator if name_generator is not None else name_filter()
+        self.names = []
+
+    def iter_names(self):
+        for name in self.names:
+            yield name
+
+        while True:
+            name = next(self.name_generator)
+            self.names.append(name)
+            yield name
+
+    def available_name(self, reservation_scope, prefix=''):
+        """
+        Search for the first name that is not in reservation scope
+        """
+
+        for name in self.iter_names():
+            if self.is_available(prefix + name, reservation_scope):
+                return prefix + name
+
+    def is_available(self, name, reservation_scope):
+        """
+        Is a name unreserved in a reservation scope
+
+        :param str name: the name to check availability of
+        :param reservation_scope: The scope to check
+        :type reservation_scope: Iterable[:class:`ast.AST`]
+        :rtype: bool
+
+        """
+
+        for namespace in reservation_scope:
+            if name in namespace.assigned_names:
+                return False
+
+        return True
+
+    def __call__(self, module, prefix_globals, reserved_globals=None):
+        assert isinstance(module, ast.Module)
+        add_assigned(module)
+
+        for namespace, binding in all_bindings(module):
+            if binding.reserved is not None:
+                scope = reservation_scope(namespace, binding)
+                reserve_name(binding.reserved, scope)
+
+        if reserved_globals is not None:
+            for name in reserved_globals:
+                module.assigned_names.add(name)
+
+        for namespace, binding in sorted_bindings(module):
+            scope = reservation_scope(namespace, binding)
+
+            if binding.allow_rename:
+
+                if isinstance(namespace, ast.Module) and prefix_globals:
+                    name = self.available_name(scope, prefix='_')
+                else:
+                    name = self.available_name(scope)
+
+                def should_rename():
+                    if binding.should_rename(name):
+                        return True
+
+                    # It's no longer efficient to do this rename
+
+                    if isinstance(binding, NameBinding):
+                        # Check that the original name is still available
+
+                        if binding.reserved == binding.name:
+                            # We already reserved it (this is probably an arg)
+                            return False
+
+                        if not self.is_available(binding.name, scope):
+                            # The original name has already been assigned to another binding,
+                            # so we need to rename this anyway.
+                            return True
+
+                    return False
+
+                if should_rename():
+                    binding.rename(name)
+                else:
+                    # Any existing name will become reserved
+                    binding.disallow_rename()
+
+            if binding.name is not None:
+                reserve_name(binding.name, scope)
+
+        return module
+
+
+def rename(module, prefix_globals=False, preserved_globals=None):
+    NameAssigner()(module, prefix_globals, preserved_globals)