pyflyby 1.9.4__py3-none-any.whl

pyflyby/_docxref.py

@@ -0,0 +1,379 @@
+# pyflyby/_docxref.py.
+
+# Module for checking Epydoc cross-references.
+
+# Portions of the code below are derived from Epydoc, which is distributed
+# under the MIT license:
+#
+#   Permission is hereby granted, free of charge, to any person obtaining a
+#   copy of this software and any associated documentation files (the
+#   "Software"), to deal in the Software without restriction, including
+#   without limitation the rights to use, copy, modify, merge, publish,
+#   distribute, sublicense, and/or sell copies of the Software, and to permit
+#   persons to whom the Software is furnished to do so, subject to the
+#   following conditions:
+#
+#   The above copyright notice and this permission notice shall be included in
+#   all copies or substantial portions of the Software.
+#
+#   The software is provided "as is", without warranty of any kind, express or
+#   implied, including but not limited to the warranties of merchantability,
+#   fitness for a particular purpose and noninfringement. In no event shall
+#   the authors or copyright holders be liable for any claim, damages or other
+#   liability, whether in an action of contract, tort or otherwise, arising
+#   from, out of or in connection with the software or the use or other
+#   dealings in the software.
+
+
+import re
+import builtins
+from   textwrap                 import dedent
+
+from   epydoc.apidoc            import (ClassDoc, ModuleDoc, PropertyDoc,
+                                        RoutineDoc, UNKNOWN, VariableDoc)
+from   epydoc.docbuilder        import build_doc_index
+from   epydoc.markup.plaintext  import ParsedPlaintextDocstring
+
+from   pyflyby._file            import Filename
+from   pyflyby._idents          import DottedIdentifier
+from   pyflyby._log             import logger
+from   pyflyby._modules         import ModuleHandle
+from   pyflyby._util            import cached_attribute, memoize, prefixes
+
+# If someone references numpy.*, just assume it's OK - it's not worth
+# following into numpy because it's too slow.
+ASSUME_MODULES_OK = set(['numpy'])
+
+@memoize
+def map_strings_to_line_numbers(module):
+    """
+    Walk ``module.ast``, looking at all string literals.  Return a map from
+    string literals to line numbers (1-index).
+
+    :rtype:
+      ``dict`` from ``str`` to (``int``, ``str``)
+    """
+    d = {}
+    for field in module.block.string_literals():
+        # Dedent because epydoc dedents strings and we need to look up by
+        # those.  But keep track of original version because we need to count
+        # exact line numbers.
+        s = dedent(field.s).strip()
+        start_lineno = field.startpos.lineno
+        d[s] = (start_lineno, field.s)
+    return d
+
+
+def get_string_linenos(module, searchstring, within_string):
+    """
+    Return the line numbers (1-indexed) within ``filename`` that contain
+    ``searchstring``.  Only consider string literals (i.e. not comments).
+    First look for exact matches of ``within_string`` (modulo indenting) and
+    then search within that.  Only if the ``within_string`` is not found,
+    search the entire file.
+
+    [If there's a comment on the same line as a string that also contains the
+    searchstring, we'll get confused.]
+    """
+    module = ModuleHandle(module)
+    regexp = re.compile(searchstring)
+    map = map_strings_to_line_numbers(module)
+    results = []
+    def scan_within_string(results, start_lineno, orig_full_string):
+        for i, line in enumerate(orig_full_string.splitlines()):
+            if regexp.search(line):
+                results.append( start_lineno + i )
+    try:
+        lineno, orig_full_string = map[within_string.strip()]
+    except KeyError:
+        pass
+    else:
+        # We found the larger string exactly within the ast.
+        scan_within_string(results, lineno, orig_full_string)
+        if results:
+            return tuple(results)
+        # We could continue down if this ever happened.
+        raise Exception(
+            "Found superstring in %r but not substring %r within superstring"
+            % (module.filename, searchstring))
+    # Try a full text search.
+    for lineno, orig_full_string in map.values():
+        scan_within_string(results, lineno, orig_full_string)
+    if results:
+        return tuple(sorted(results))
+    raise Exception(
+        "Could not find %r anywhere in %r" % (searchstring, module.filename))
+
+
+def describe_xref(identifier, container):
+    module = ModuleHandle(str(container.defining_module.canonical_name))
+    assert module.filename == Filename(container.defining_module.filename)
+    linenos = get_string_linenos(
+        module,
+        "(L{|<)%s" % (identifier,),
+        container.docstring)
+    return (module, linenos, str(container.canonical_name), identifier)
+
+
+
+def safe_build_doc_index(modules):
+    # build_doc_index isn't re-entrant due to crappy caching! >:(
+    from epydoc.docintrospecter import clear_cache
+    clear_cache()
+    from epydoc.docparser import _moduledoc_cache
+    _moduledoc_cache.clear()
+    # Build a new DocIndex.  It swallows exceptions and returns None on error!
+    # >:(
+    result = build_doc_index(modules)
+    if result is None:
+        raise Exception("Failed to build doc index on %r" % (modules,))
+    return result
+
+
+class ExpandedDocIndex(object):
+    """
+    A wrapper around DocIndex that automatically expands with more modules as
+    needed.
+    """
+    # TODO: this is kludgy and inefficient since it re-reads modules.
+    def __init__(self, modules):
+        self.modules = set([ModuleHandle(m) for m in modules])
+
+    def add_module(self, module):
+        """
+        Adds ``module`` and recreates the DocIndex with the updated set of
+        modules.
+
+        :return:
+          Whether anything was added.
+        """
+        module = ModuleHandle(module)
+        for prefix in module.ancestors:
+            if prefix in self.modules:
+                # The module, or a prefix of it, was already added.
+                return False
+
+        for existing_module in sorted(self.modules):
+            if existing_module.startswith(module):
+                # This supersedes an existing module.
+                assert existing_module != module
+                self.modules.remove(existing_module)
+
+        logger.debug("Expanding docindex to include %r", module)
+        self.modules.add(module)
+        del self.docindex
+        return True
+
+    def find(self, a, b):
+        return self.docindex.find(a, b)
+
+    def get_vardoc(self, a):
+        return self.docindex.get_vardoc(a)
+
+    @cached_attribute
+    def docindex(self):
+        return safe_build_doc_index(
+            [str(m.name) for m in sorted(self.modules)])
+
+
+def remove_epydoc_sym_suffix(s):
+    """
+    Remove trailing "'" that Epydoc annoyingly adds to 'shadowed' names.
+
+      >>> remove_epydoc_sym_suffix("a.b'.c'.d")
+      'a.b.c.d'
+
+    """
+    return re.sub(r"'([.]|$)", r'\1', s)
+
+class XrefScanner(object):
+
+    def __init__(self, modules):
+        self.modules = modules
+        self.docindex = safe_build_doc_index(modules)
+
+    @cached_attribute
+    def expanded_docindex(self):
+        return ExpandedDocIndex(self.modules)
+
+    def scan(self):
+        self._failed_xrefs = []
+        valdocs = sorted(self.docindex.reachable_valdocs(
+            imports=False, packages=False, bases=False, submodules=False,
+            subclasses=False, private=True
+            ))
+        for doc in valdocs:
+            if isinstance(doc, ClassDoc):
+                self.scan_class(doc)
+            elif isinstance(doc, ModuleDoc):
+                self.scan_module(doc)
+        return tuple(sorted(self._failed_xrefs))
+
+    def scan_module(self, doc):
+        self.descr(doc)
+        if doc.is_package is True:
+            for submodule in doc.submodules:
+                self.scan_module(submodule)
+            # self.scan_module_list(doc)
+        self.scan_details_list(doc, "function")
+        self.scan_details_list(doc, "other")
+
+    def scan_class(self, doc):
+        self.descr(doc)
+        self.scan_details_list(doc, "method")
+        self.scan_details_list(doc, "classvariable")
+        self.scan_details_list(doc, "instancevariable")
+        self.scan_details_list(doc, "property")
+
+    def scan_details_list(self, doc, value_type):
+        detailed = True
+        if isinstance(doc, ClassDoc):
+            var_docs = doc.select_variables(value_type=value_type,
+                                            imported=False, inherited=False,
+                                            public=None,
+                                            detailed=detailed)
+        else:
+            var_docs = doc.select_variables(value_type=value_type,
+                                            imported=False,
+                                            public=None,
+                                            detailed=detailed)
+        for var_doc in var_docs:
+            self.scan_details(var_doc)
+
+    def scan_details(self, var_doc):
+        self.descr(var_doc)
+        if isinstance(var_doc.value, RoutineDoc):
+            self.return_type(var_doc)
+            self.return_descr(var_doc)
+            for (arg_names, arg_descr) in var_doc.value.arg_descrs:
+                self.scan_docstring(arg_descr, var_doc.value)
+            for arg in var_doc.value.arg_types:
+                self.scan_docstring(
+                    var_doc.value.arg_types[arg], var_doc.value)
+        elif isinstance(var_doc.value, PropertyDoc):
+            prop_doc = var_doc.value
+            self.return_type(prop_doc.fget)
+            self.return_type(prop_doc.fset)
+            self.return_type(prop_doc.fdel)
+        else:
+            self.type_descr(var_doc)
+
+    def _scan_attr(self, attr, api_doc):
+        if api_doc in (None, UNKNOWN):
+            return ''
+        pds = getattr(api_doc, attr, None) # pds = ParsedDocstring.
+        if pds not in (None, UNKNOWN):
+            self.scan_docstring(pds, api_doc)
+        elif isinstance(api_doc, VariableDoc):
+            self._scan_attr(attr, api_doc.value)
+
+    def summary(self, api_doc):
+        self._scan_attr('summary', api_doc)
+
+    def descr(self, api_doc):
+        self._scan_attr('descr', api_doc)
+
+    def type_descr(self, api_doc):
+        self._scan_attr('type_descr', api_doc)
+
+    def return_type(self, api_doc):
+        self._scan_attr('return_type', api_doc)
+
+    def return_descr(self, api_doc):
+        self._scan_attr('return_descr', api_doc)
+
+    def check_xref(self, identifier, container):
+        """
+        Check that ``identifier`` cross-references a proper symbol.
+
+        Look in modules that we weren't explicitly asked to look in, if
+        needed.
+        """
+        if identifier in builtins.__dict__:
+            return True
+        def check_container():
+            if self.expanded_docindex.find(identifier, container) is not None:
+                return True
+            if isinstance(container, RoutineDoc):
+                tcontainer = self.expanded_docindex.get_vardoc(
+                    container.canonical_name)
+                doc = self.expanded_docindex.find(identifier, tcontainer)
+                while (doc is not None and tcontainer not in (None, UNKNOWN)
+                       and tcontainer.overrides not in (None, UNKNOWN)):
+                    tcontainer = tcontainer.overrides
+                    doc = self.expanded_docindex.find(identifier, tcontainer)
+                return doc is not None
+            return False
+        def check_defining_module(x):
+            if x is None:
+                return False
+            defining_module_name = remove_epydoc_sym_suffix(str(
+                x.defining_module.canonical_name))
+            if defining_module_name in ASSUME_MODULES_OK:
+                return True
+            if self.expanded_docindex.add_module(defining_module_name):
+                if check_container():
+                    return True
+            return False
+        if check_container():
+            return True
+        if (isinstance(container, RoutineDoc) and
+            identifier in container.all_args()):
+            return True
+        if check_defining_module(container):
+            return True
+        # If the user has imported foo.bar.baz as baz and now uses
+        # ``baz.quux``, we need to add the module foo.bar.baz.
+        for prefix in reversed(list(prefixes(
+                    DottedIdentifier(remove_epydoc_sym_suffix(identifier))))):
+            if check_defining_module(
+                self.docindex.find(str(prefix), container)):
+                return True
+        try:
+            module = ModuleHandle.containing(identifier)
+        except ImportError:
+            pass
+        else:
+            if str(module.name) in ASSUME_MODULES_OK:
+                return True
+            if self.expanded_docindex.add_module(module):
+                if check_container():
+                    return True
+        return False
+
+    def scan_docstring(self, parsed_docstring, container):
+        if parsed_docstring in (None, UNKNOWN): return ''
+        if isinstance(parsed_docstring, ParsedPlaintextDocstring):
+            return ''
+
+        def scan_tree(tree):
+            if isinstance(tree, str):
+                return tree
+            variables = [scan_tree(child) for child in tree.children]
+            if tree.tag == 'link':
+                identifier = variables[1]
+                if not self.check_xref(identifier, container):
+                    self._failed_xrefs.append(
+                        describe_xref(identifier, container) )
+                return '?'
+            elif tree.tag == 'indexed':
+                return '?'
+            elif tree.tag in ('epytext', 'section', 'tag', 'arg',
+                              'name', 'target', 'html', 'para'):
+                return ''.join(variables)
+            return '?'
+
+        scan_tree(parsed_docstring._tree)
+
+
+def find_bad_doc_cross_references(names):
+    """
+    Find docstring cross references that fail to resolve.
+
+    :type names:
+      Sequence of module names or filenames.
+    :return:
+      Sequence of ``(module, linenos, container_name, identifier)`` tuples.
+    """
+    xrs = XrefScanner(names)
+    return xrs.scan()