pyflyby 1.10.1__cp311-cp311-manylinux_2_24_x86_64.whl

pyflyby/_docxref.py

@@ -0,0 +1,379 @@
+# pyflyby/_docxref.py.
+
+# Module for checking Epydoc cross-references.
+
+# Portions of the code below are derived from Epydoc, which is distributed
+# under the MIT license:
+#
+#   Permission is hereby granted, free of charge, to any person obtaining a
+#   copy of this software and any associated documentation files (the
+#   "Software"), to deal in the Software without restriction, including
+#   without limitation the rights to use, copy, modify, merge, publish,
+#   distribute, sublicense, and/or sell copies of the Software, and to permit
+#   persons to whom the Software is furnished to do so, subject to the
+#   following conditions:
+#
+#   The above copyright notice and this permission notice shall be included in
+#   all copies or substantial portions of the Software.
+#
+#   The software is provided "as is", without warranty of any kind, express or
+#   implied, including but not limited to the warranties of merchantability,
+#   fitness for a particular purpose and noninfringement. In no event shall
+#   the authors or copyright holders be liable for any claim, damages or other
+#   liability, whether in an action of contract, tort or otherwise, arising
+#   from, out of or in connection with the software or the use or other
+#   dealings in the software.
+
+
+import re
+import builtins
+from   textwrap                 import dedent
+
+from   epydoc.apidoc            import (ClassDoc, ModuleDoc, PropertyDoc,
+                                        RoutineDoc, UNKNOWN, VariableDoc)
+from   epydoc.docbuilder        import build_doc_index
+from   epydoc.markup.plaintext  import ParsedPlaintextDocstring
+
+from   pyflyby._file            import Filename
+from   pyflyby._idents          import DottedIdentifier
+from   pyflyby._log             import logger
+from   pyflyby._modules         import ModuleHandle
+from   pyflyby._util            import cached_attribute, memoize, prefixes
+
+# If someone references numpy.*, just assume it's OK - it's not worth
+# following into numpy because it's too slow.
+ASSUME_MODULES_OK = set(['numpy'])
+
+@memoize
+def map_strings_to_line_numbers(module):
+    """
+    Walk ``module.ast``, looking at all string literals.  Return a map from
+    string literals to line numbers (1-index).
+
+    :rtype:
+      ``dict`` from ``str`` to (``int``, ``str``)
+    """
+    d = {}
+    for field in module.block.string_literals():
+        # Dedent because epydoc dedents strings and we need to look up by
+        # those.  But keep track of original version because we need to count
+        # exact line numbers.
+        s = dedent(field.s).strip()
+        start_lineno = field.startpos.lineno
+        d[s] = (start_lineno, field.s)
+    return d
+
+
+def get_string_linenos(module, searchstring, within_string):
+    """
+    Return the line numbers (1-indexed) within ``filename`` that contain
+    ``searchstring``.  Only consider string literals (i.e. not comments).
+    First look for exact matches of ``within_string`` (modulo indenting) and
+    then search within that.  Only if the ``within_string`` is not found,
+    search the entire file.
+
+    [If there's a comment on the same line as a string that also contains the
+    searchstring, we'll get confused.]
+    """
+    module = ModuleHandle(module)
+    regexp = re.compile(searchstring)
+    map = map_strings_to_line_numbers(module)
+    results = []
+    def scan_within_string(results, start_lineno, orig_full_string):
+        for i, line in enumerate(orig_full_string.splitlines()):
+            if regexp.search(line):
+                results.append( start_lineno + i )
+    try:
+        lineno, orig_full_string = map[within_string.strip()]
+    except KeyError:
+        pass
+    else:
+        # We found the larger string exactly within the ast.
+        scan_within_string(results, lineno, orig_full_string)
+        if results:
+            return tuple(results)
+        # We could continue down if this ever happened.
+        raise Exception(
+            "Found superstring in %r but not substring %r within superstring"
+            % (module.filename, searchstring))
+    # Try a full text search.
+    for lineno, orig_full_string in map.values():
+        scan_within_string(results, lineno, orig_full_string)
+    if results:
+        return tuple(sorted(results))
+    raise Exception(
+        "Could not find %r anywhere in %r" % (searchstring, module.filename))
+
+
+def describe_xref(identifier, container):
+    module = ModuleHandle(str(container.defining_module.canonical_name))
+    assert module.filename == Filename(container.defining_module.filename)
+    linenos = get_string_linenos(
+        module,
+        "(L{|<)%s" % (identifier,),
+        container.docstring)
+    return (module, linenos, str(container.canonical_name), identifier)
+
+
+
+def safe_build_doc_index(modules):
+    # build_doc_index isn't re-entrant due to crappy caching! >:(
+    from epydoc.docintrospecter import clear_cache
+    clear_cache()
+    from epydoc.docparser import _moduledoc_cache
+    _moduledoc_cache.clear()
+    # Build a new DocIndex.  It swallows exceptions and returns None on error!
+    # >:(
+    result = build_doc_index(modules)
+    if result is None:
+        raise Exception("Failed to build doc index on %r" % (modules,))
+    return result
+
+
+class ExpandedDocIndex(object):
+    """
+    A wrapper around DocIndex that automatically expands with more modules as
+    needed.
+    """
+    # TODO: this is kludgy and inefficient since it re-reads modules.
+    def __init__(self, modules):
+        self.modules = set([ModuleHandle(m) for m in modules])
+
+    def add_module(self, module):
+        """
+        Adds ``module`` and recreates the DocIndex with the updated set of
+        modules.
+
+        :return:
+          Whether anything was added.
+        """
+        module = ModuleHandle(module)
+        for prefix in module.ancestors:
+            if prefix in self.modules:
+                # The module, or a prefix of it, was already added.
+                return False
+
+        for existing_module in sorted(self.modules):
+            if existing_module.startswith(module):
+                # This supersedes an existing module.
+                assert existing_module != module
+                self.modules.remove(existing_module)
+
+        logger.debug("Expanding docindex to include %r", module)
+        self.modules.add(module)
+        del self.docindex
+        return True
+
+    def find(self, a, b):
+        return self.docindex.find(a, b)
+
+    def get_vardoc(self, a):
+        return self.docindex.get_vardoc(a)
+
+    @cached_attribute
+    def docindex(self):
+        return safe_build_doc_index(
+            [str(m.name) for m in sorted(self.modules)])
+
+
+def remove_epydoc_sym_suffix(s):
+    """
+    Remove trailing "'" that Epydoc annoyingly adds to 'shadowed' names.
+
+      >>> remove_epydoc_sym_suffix("a.b'.c'.d")
+      'a.b.c.d'
+
+    """
+    return re.sub(r"'([.]|$)", r'\1', s)
+
+class XrefScanner(object):
+
+    def __init__(self, modules):
+        self.modules = modules
+        self.docindex = safe_build_doc_index(modules)
+
+    @cached_attribute
+    def expanded_docindex(self):
+        return ExpandedDocIndex(self.modules)
+
+    def scan(self):
+        self._failed_xrefs = []
+        valdocs = sorted(self.docindex.reachable_valdocs(
+            imports=False, packages=False, bases=False, submodules=False,
+            subclasses=False, private=True
+            ))
+        for doc in valdocs:
+            if isinstance(doc, ClassDoc):
+                self.scan_class(doc)
+            elif isinstance(doc, ModuleDoc):
+                self.scan_module(doc)
+        return tuple(sorted(self._failed_xrefs))
+
+    def scan_module(self, doc):
+        self.descr(doc)
+        if doc.is_package is True:
+            for submodule in doc.submodules:
+                self.scan_module(submodule)
+            # self.scan_module_list(doc)
+        self.scan_details_list(doc, "function")
+        self.scan_details_list(doc, "other")
+
+    def scan_class(self, doc):
+        self.descr(doc)
+        self.scan_details_list(doc, "method")
+        self.scan_details_list(doc, "classvariable")
+        self.scan_details_list(doc, "instancevariable")
+        self.scan_details_list(doc, "property")
+
+    def scan_details_list(self, doc, value_type):
+        detailed = True
+        if isinstance(doc, ClassDoc):
+            var_docs = doc.select_variables(value_type=value_type,
+                                            imported=False, inherited=False,
+                                            public=None,
+                                            detailed=detailed)
+        else:
+            var_docs = doc.select_variables(value_type=value_type,
+                                            imported=False,
+                                            public=None,
+                                            detailed=detailed)
+        for var_doc in var_docs:
+            self.scan_details(var_doc)
+
+    def scan_details(self, var_doc):
+        self.descr(var_doc)
+        if isinstance(var_doc.value, RoutineDoc):
+            self.return_type(var_doc)
+            self.return_descr(var_doc)
+            for (arg_names, arg_descr) in var_doc.value.arg_descrs:
+                self.scan_docstring(arg_descr, var_doc.value)
+            for arg in var_doc.value.arg_types:
+                self.scan_docstring(
+                    var_doc.value.arg_types[arg], var_doc.value)
+        elif isinstance(var_doc.value, PropertyDoc):
+            prop_doc = var_doc.value
+            self.return_type(prop_doc.fget)
+            self.return_type(prop_doc.fset)
+            self.return_type(prop_doc.fdel)
+        else:
+            self.type_descr(var_doc)
+
+    def _scan_attr(self, attr, api_doc):
+        if api_doc in (None, UNKNOWN):
+            return ''
+        pds = getattr(api_doc, attr, None) # pds = ParsedDocstring.
+        if pds not in (None, UNKNOWN):
+            self.scan_docstring(pds, api_doc)
+        elif isinstance(api_doc, VariableDoc):
+            self._scan_attr(attr, api_doc.value)
+
+    def summary(self, api_doc):
+        self._scan_attr('summary', api_doc)
+
+    def descr(self, api_doc):
+        self._scan_attr('descr', api_doc)
+
+    def type_descr(self, api_doc):
+        self._scan_attr('type_descr', api_doc)
+
+    def return_type(self, api_doc):
+        self._scan_attr('return_type', api_doc)
+
+    def return_descr(self, api_doc):
+        self._scan_attr('return_descr', api_doc)
+
+    def check_xref(self, identifier, container):
+        """
+        Check that ``identifier`` cross-references a proper symbol.
+
+        Look in modules that we weren't explicitly asked to look in, if
+        needed.
+        """
+        if identifier in builtins.__dict__:
+            return True
+        def check_container():
+            if self.expanded_docindex.find(identifier, container) is not None:
+                return True
+            if isinstance(container, RoutineDoc):
+                tcontainer = self.expanded_docindex.get_vardoc(
+                    container.canonical_name)
+                doc = self.expanded_docindex.find(identifier, tcontainer)
+                while (doc is not None and tcontainer not in (None, UNKNOWN)
+                       and tcontainer.overrides not in (None, UNKNOWN)):
+                    tcontainer = tcontainer.overrides
+                    doc = self.expanded_docindex.find(identifier, tcontainer)
+                return doc is not None
+            return False
+        def check_defining_module(x):
+            if x is None:
+                return False
+            defining_module_name = remove_epydoc_sym_suffix(str(
+                x.defining_module.canonical_name))
+            if defining_module_name in ASSUME_MODULES_OK:
+                return True
+            if self.expanded_docindex.add_module(defining_module_name):
+                if check_container():
+                    return True
+            return False
+        if check_container():
+            return True
+        if (isinstance(container, RoutineDoc) and
+            identifier in container.all_args()):
+            return True
+        if check_defining_module(container):
+            return True
+        # If the user has imported foo.bar.baz as baz and now uses
+        # ``baz.quux``, we need to add the module foo.bar.baz.
+        for prefix in reversed(list(prefixes(
+                    DottedIdentifier(remove_epydoc_sym_suffix(identifier))))):
+            if check_defining_module(
+                self.docindex.find(str(prefix), container)):
+                return True
+        try:
+            module = ModuleHandle.containing(identifier)
+        except ImportError:
+            pass
+        else:
+            if str(module.name) in ASSUME_MODULES_OK:
+                return True
+            if self.expanded_docindex.add_module(module):
+                if check_container():
+                    return True
+        return False
+
+    def scan_docstring(self, parsed_docstring, container):
+        if parsed_docstring in (None, UNKNOWN): return ''
+        if isinstance(parsed_docstring, ParsedPlaintextDocstring):
+            return ''
+
+        def scan_tree(tree):
+            if isinstance(tree, str):
+                return tree
+            variables = [scan_tree(child) for child in tree.children]
+            if tree.tag == 'link':
+                identifier = variables[1]
+                if not self.check_xref(identifier, container):
+                    self._failed_xrefs.append(
+                        describe_xref(identifier, container) )
+                return '?'
+            elif tree.tag == 'indexed':
+                return '?'
+            elif tree.tag in ('epytext', 'section', 'tag', 'arg',
+                              'name', 'target', 'html', 'para'):
+                return ''.join(variables)
+            return '?'
+
+        scan_tree(parsed_docstring._tree)
+
+
+def find_bad_doc_cross_references(names):
+    """
+    Find docstring cross references that fail to resolve.
+
+    :type names:
+      Sequence of module names or filenames.
+    :return:
+      Sequence of ``(module, linenos, container_name, identifier)`` tuples.
+    """
+    xrs = XrefScanner(names)
+    return xrs.scan()

pyflyby/_dynimp.py

@@ -0,0 +1,154 @@
+"""
+Virtual module to create dynamic import at runtime.
+
+It is sometime desirable to have auto import which are define only during
+a session and never exist on a on-disk file.
+
+This is injects a Dict module loader as well as a dictionary registry of in
+memory module.
+
+This is mostly use in IPython for lazy variable initialisation without having
+to use proxy objects.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py`), or in your IPython
+configuration file:
+
+
+.. code:: python
+
+    from pyflyby._dynimp import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''',
+    )
+
+Now at the IPython prompt, if the pyflyby extension is loaded (either because
+you started  using the ``py`` cli, or some configuration options like  ``ipython
+--TerminalIPythonApp.extra_extensions=pyflyby``. When trying to use an undefined
+variable like ``foo``, ``df`` or ``data``, the corresponding module will be
+executed and the relevant variable imported.
+
+
+"""
+import importlib.abc
+import importlib.util
+import sys
+
+from textwrap import dedent
+from typing import FrozenSet
+
+from pyflyby._importclns import ImportSet, Import
+
+module_dict = {}
+
+PYFLYBY_LAZY_LOAD_PREFIX = "from pyflyby_autoimport_"
+
+def add_import(names: str, code: str, *, strict: bool = True):
+    """
+    Add a runtime generated import module
+
+    Parameters
+    ----------
+    names: str
+        name, or comma separated list variable names that should be created by
+        executing and importing `code`.
+    code: str
+        potentially multiline string that will be turned into a module,
+        executed and from which variables listed in names can be imported.
+    strict: bool
+        Raise in case of problem loading IPython of if pyflyby extension not installed.
+        otherwise just ignore error
+
+
+
+    Examples
+    --------
+
+    >>> add_import('pd, df', '''
+    ...     import pandas a pd
+    ...
+    ...     df = pd.DataFrame([[1,2], [3,4]])
+    ... ''', strict=False)  # don't fail doctest
+
+    """
+    try:
+        ip = _raise_if_problem()
+    except Exception:
+        if strict:
+            raise
+        else:
+            return
+    return _add_import(ip, names, code)
+
+
+def _raise_if_problem():
+    try:
+        import IPython
+    except ModuleNotFoundError as e:
+        raise ImportError("Dynamic autoimport requires IPython to be installed") from e
+
+    ip = IPython.get_ipython()
+    if ip is None:
+        raise ImportError("Dynamic autoimport only work from within IPython")
+
+    if not hasattr(ip, "_auto_importer"):
+        raise ValueError(
+            "IPython needs to be loaded with pyflyby extension for lazy variable to work"
+        )
+    return ip
+
+
+def _add_import(ip, names: str, code: str) -> None:
+    """
+    private version of add_import
+    """
+    assert ip is not None
+    module = PYFLYBY_LAZY_LOAD_PREFIX.split()[1]
+    mang = module + names.replace(",", "_").replace(" ", "_")
+    a: FrozenSet[Import] = ImportSet(f"from {mang} import {names}")._importset
+    b: FrozenSet[Import] = ip._auto_importer.db.known_imports._importset
+    s_import: FrozenSet[Import] = a | b
+
+    ip._auto_importer.db.known_imports = ImportSet._from_imports(list(s_import))
+    module_dict[mang] = dedent(code)
+
+class DictLoader(importlib.abc.Loader):
+    """
+    A dict based loader for in-memory module definition.
+    """
+    def __init__(self, module_name, module_code):
+        self.module_name = module_name
+        self.module_code = module_code
+
+    def create_module(self, spec):
+        return None  # Use default module creation semantics
+
+    def exec_module(self, module):
+        """
+        we exec module code directly in memory
+        """
+        exec(self.module_code, module.__dict__)
+
+
+class DictFinder(importlib.abc.MetaPathFinder):
+    """
+    A meta path finder for abode DictLoader
+    """
+    def find_spec(self, fullname, path, target=None):
+        if fullname in module_dict:
+            module_code = module_dict[fullname]
+            loader = DictLoader(fullname, module_code)
+            return importlib.util.spec_from_loader(fullname, loader)
+        return None
+
+
+def inject():
+    sys.meta_path.insert(0, DictFinder())