pyflyby 1.9.3__tar.gz → 1.9.5__tar.gz

{pyflyby-1.9.3/lib/python/pyflyby.egg-info → pyflyby-1.9.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pyflyby
-Version: 1.9.3
+Version: 1.9.5
 Summary: pyflyby - Python development productivity tools, in particular automatic import management
 Home-page: https://pypi.org/project/pyflyby/
 Author: Karl Chen
@@ -16,8 +16,12 @@ Classifier: Topic :: Software Development :: Interpreters
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python
-Requires-Python: >3.6, <4
+Requires-Python: >3.8, <4
 License-File: LICENSE.txt
+Requires-Dist: six
+Requires-Dist: toml
+Requires-Dist: black
+Requires-Dist: typing_extensions>=4.6; python_version < "3.12"
 
 #########
  Pyflyby
@@ -29,7 +33,7 @@ License-File: LICENSE.txt
 .. image:: https://travis-ci.org/deshaw/pyflyby.png?branch=master
    :target: https://travis-ci.org/deshaw/pyflyby
 
-Pyflyby is a set of Python programming productivity tools for Python 3.7+.
+Pyflyby is a set of Python programming productivity tools for Python 3.8+.
 
 For command-line interaction:
   * ``py``: command-line multitool
@@ -102,6 +106,36 @@ or
    [PYFLYBY] from base64 import b64decode
    Out[1]: 'hello'
 
+Auto importer lazy variables
+----------------------------
+
+It is possible to use the autoimporter to lazily define variables.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py``), or in your IPython
+configuration file:
+
+.. code:: python
+
+   
+    from pyflyby import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data as dd",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''')
+
+
+You can add the keyword ``strict=False`` to not fail if not in IPython or of the
+pyflyby extensions is not loaded.
+
+
+
 
 Quick start: ``py`` command-line multi-tool
 ===========================================

{pyflyby-1.9.3 → pyflyby-1.9.5}/README.rst

@@ -8,7 +8,7 @@
 .. image:: https://travis-ci.org/deshaw/pyflyby.png?branch=master
    :target: https://travis-ci.org/deshaw/pyflyby
 
-Pyflyby is a set of Python programming productivity tools for Python 3.7+.
+Pyflyby is a set of Python programming productivity tools for Python 3.8+.
 
 For command-line interaction:
   * ``py``: command-line multitool
@@ -81,6 +81,36 @@ or
    [PYFLYBY] from base64 import b64decode
    Out[1]: 'hello'
 
+Auto importer lazy variables
+----------------------------
+
+It is possible to use the autoimporter to lazily define variables.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py``), or in your IPython
+configuration file:
+
+.. code:: python
+
+   
+    from pyflyby import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data as dd",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''')
+
+
+You can add the keyword ``strict=False`` to not fail if not in IPython or of the
+pyflyby extensions is not loaded.
+
+
+
 
 Quick start: ``py`` command-line multi-tool
 ===========================================

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/autopython

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/usr/bin/env -S bash -e
 
 # pyflyby/autopython
 

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/create-imports

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/usr/bin/env -S bash -e
 
 # pyflyby/create-imports
 

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/pyflyby/__init__.py

@@ -6,6 +6,7 @@
 
 from   pyflyby._autoimp         import (auto_eval, auto_import,
                                         find_missing_imports)
+from   pyflyby._dynimp          import add_import
 from   pyflyby._dbg             import (add_debug_functions_to_builtins,
                                         attach_debugger, debug_on_exception,
                                         debug_statement, debugger,
@@ -40,6 +41,8 @@ from   pyflyby._dbg             import (breakpoint, debug_exception,
                                         enable_signal_handler_breakpoint,
                                         waitpoint)
 
+from   typing                   import Sequence
+
 
 # Promote the function & classes that we've chosen to expose publicly to be
 # known as pyflyby.Foo instead of pyflyby._module.Foo.
@@ -51,4 +54,4 @@ del x
 
 # Discourage "from pyflyby import *".
 # Use the tidy-imports/autoimporter instead!
-__all__ = []
+__all__:Sequence[str] = []

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/pyflyby/_autoimp.py

@@ -6,13 +6,9 @@
 
 import ast
 import builtins
+from   collections.abc          import Sequence
 import contextlib
 import copy
-from   six                      import reraise
-import sys
-import types
-
-from collections.abc import Sequence
 
 from   pyflyby._file            import FileText, Filename
 from   pyflyby._flags           import CompilerFlags
@@ -22,7 +18,13 @@ from   pyflyby._importdb        import ImportDB
 from   pyflyby._importstmt      import Import
 from   pyflyby._log             import logger
 from   pyflyby._modules         import ModuleHandle
-from   pyflyby._parse           import PythonBlock, infer_compile_mode, _is_ast_str
+from   pyflyby._parse           import (PythonBlock, _is_ast_str,
+                                        infer_compile_mode, MatchAs)
+
+from   six                      import reraise
+import sys
+import types
+from   typing                   import Any, Set
 
 if sys.version_info >= (3, 12):
     ATTRIBUTE_NAME = "value"
@@ -34,13 +36,13 @@ if sys.version_info > (3, 11):
 else:
     LOAD_SHIFT = 0
 
-if sys.version_info > (3, 11):
-    LOAD_SHIFT = 1
+
+if sys.version_info >= (3, 10):
+    from types import NoneType, EllipsisType
 else:
-    LOAD_SHIFT = 0
+    NoneType = type(None)
+    EllipsisType = type(Ellipsis)
 
-NoneType = type(None)
-EllipsisType = type(Ellipsis)
 
 class _ClassScope(dict):
     pass
@@ -419,7 +421,8 @@ class _MissingImportFinder(object):
 
     def scan_for_import_issues(self, codeblock):
         # See global `scan_for_import_issues`
-        codeblock = PythonBlock(codeblock)
+        if not isinstance(codeblock, PythonBlock):
+            codeblock = PythonBlock(codeblock)
         node = codeblock.ast_node
         self._scan_node(node)
         # Get missing imports now, before handling docstrings.  We don't want
@@ -488,8 +491,11 @@ class _MissingImportFinder(object):
                 logger.debug(
                     "_MissingImportFinder has no method %r, using generic_visit", method
                 )
-
-            visitor = getattr(self, method, self.generic_visit)
+            if hasattr(self, method):
+                visitor = getattr(self, method)
+            else:
+                logger.debug("No method `%s`, using `generic_visit`", method)
+                visitor = self.generic_visit
             return visitor(node)
         else:
             raise TypeError("unexpected %s" % (type(node).__name__,))
@@ -589,7 +595,10 @@ class _MissingImportFinder(object):
                 logger.warning("Don't know how to handle __all__ with list elements other than str")
                 return
             for e in node.value.elts:
-                self._visit_Load_defered_global(e.s)
+                if sys.version_info > (3,10):
+                    self._visit_Load_defered_global(e.value)
+                else:
+                    self._visit_Load_defered_global(e.s)
 
     def visit_ClassDef(self, node):
         logger.debug("visit_ClassDef(%r)", node)
@@ -634,6 +643,10 @@ class _MissingImportFinder(object):
         else:
             assert node._fields ==  ('name', 'args', 'body', 'decorator_list', 'returns', 'type_comment'), node._fields
         with self._NewScopeCtx(include_class_scopes=True):
+            # we want `__class__` to only be defined in
+            # methods and not class body
+            if self._in_class_def:
+                self.scopestack[-1]["__class__"] = None  # we just need to to be defined
             self.visit(node.decorator_list)
             self.visit(node.args)
             if node.returns:
@@ -817,6 +830,47 @@ class _MissingImportFinder(object):
         self._visit_StoreImport(node, modulename)
         self.generic_visit(node)
 
+    if sys.version_info >= (3,10):
+        def visit_match_case(self, node:ast.match_case):
+            logger.debug("visit_match_case(%r)", node)
+            return self.generic_visit(node)
+
+        def visit_Match(self, node:ast.Match):
+            logger.debug("visit_Match(%r)", node)
+            return self.generic_visit(node)
+
+        def visit_MatchMapping(self, node:ast.MatchMapping):
+            logger.debug("visit_MatchMapping(%r)", node)
+            return self.generic_visit(node)
+
+        def visit_MatchAs(self, node:MatchAs):
+            logger.debug("visit_MatchAs(%r)", node)
+            if node.name is None:
+                return
+            isinstance(node.name, str), node.name
+            return self._visit_Store(node.name)
+
+    def visit_Call(self, node:ast.Call):
+        logger.debug("visit_Call(%r)", node)
+        return self.generic_visit(node)
+
+    def visit_Pass(self, node:ast.Pass):
+        logger.debug("visit_Pass(%r)", node)
+        return self.generic_visit(node)
+
+    def visit_Constant(self, node:ast.Constant):
+        logger.debug("visit_Constant(%r)", node)
+        return self.generic_visit(node)
+
+    def visit_Module(self, node:ast.Module):
+        logger.debug("visit_Module(%r)", node)
+        return self.generic_visit(node)
+
+    def visit_Expr(self, node:ast.Expr):
+        logger.debug("visit_Expr(%r)", node)
+        return self.generic_visit(node)
+
+
     def visit_Name(self, node):
         logger.debug("visit_Name(%r)", node.id)
         self._visit_fullname(node.id, node.ctx)
@@ -874,7 +928,7 @@ class _MissingImportFinder(object):
             value = _UseChecker(name, imp, self._lineno)
         self._visit_Store(name, value)
 
-    def _visit_Store(self, fullname, value=None):
+    def _visit_Store(self, fullname:str, value=None):
         logger.debug("_visit_Store(%r)", fullname)
         if fullname is None:
             return
@@ -1064,7 +1118,8 @@ def scan_for_import_issues(codeblock, find_unused_imports=True, parse_docstrings
 
     """
     logger.debug("scan_for_import_issues()")
-    codeblock = PythonBlock(codeblock)
+    if not isinstance(codeblock, PythonBlock):
+        codeblock = PythonBlock(codeblock)
     namespaces = ScopeStack([{}])
     finder = _MissingImportFinder(namespaces,
                                   find_unused_imports=find_unused_imports,
@@ -1152,10 +1207,13 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
             "_find_loads_without_stores_in_code(): expected a CodeType; got a %s"
             % (type(co).__name__,))
     # Initialize local constants for fast access.
-    from opcode import HAVE_ARGUMENT, EXTENDED_ARG, opmap
+    from opcode import EXTENDED_ARG, opmap
 
     LOAD_ATTR    = opmap['LOAD_ATTR']
+    # LOAD_METHOD is _supposed_ to be removed in 3.12 but still present in opmap
+    # if sys.version_info < (3, 12):
     LOAD_METHOD  = opmap['LOAD_METHOD']
+    # endif
     LOAD_GLOBAL  = opmap['LOAD_GLOBAL']
     LOAD_NAME    = opmap['LOAD_NAME']
     STORE_ATTR   = opmap['STORE_ATTR']
@@ -1255,12 +1313,11 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
     earliest_backjump_label = _find_earliest_backjump_label(bytecode)
     # Loop through bytecode.
     while i < n:
-        c = bytecode[i]
-        op = _op(c)
+        op = bytecode[i]
         i += 1
         if op == CACHE:
             continue
-        if op >= HAVE_ARGUMENT:
+        if take_arg(op):
             oparg = bytecode[i] | extended_arg
             extended_arg = 0
             if op == EXTENDED_ARG:
@@ -1277,9 +1334,40 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
                 stores.add(fullname)
                 continue
             if op in [LOAD_ATTR, LOAD_METHOD]:
-                # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* so far;
-                # possibly more LOAD_ATTR/STORE_ATTR will follow
-                pending.append(co.co_names[oparg])
+                if sys.version_info >= (3,12):
+                    # from the docs:
+                    #
+                    # If the low bit of namei is not set, this replaces
+                    # STACK[-1] with getattr(STACK[-1], co_names[namei>>1]).
+                    #
+                    # If the low bit of namei is set, this will attempt to load
+                    # a method named co_names[namei>>1] from the STACK[-1]
+                    # object. STACK[-1] is popped. This bytecode distinguishes
+                    # two cases: if STACK[-1] has a method with the correct
+                    # name, the bytecode pushes the unbound method and
+                    # STACK[-1]. STACK[-1] will be used as the first argument
+                    # (self) by CALL when calling the unbound method. Otherwise,
+                    # NULL and the object returned by the attribute lookup are
+                    # pushed.
+                    #
+                    # Changed in version 3.12: If the low bit of namei is set,
+                    # then a NULL or self is pushed to the stack before the
+                    # attribute or unbound method respectively.
+                    #
+                    # Implication for Pyflyby
+                    #
+                    # In our case I think it means we are always looking at
+                    # oparg>>1 as the name of the names we need to load,
+                    # Though we don't keep track of the stack, and so we may get
+                    # wrong results ?
+                    #
+                    # In any case this seem to match what load_method was doing
+                    # before.
+                    pending.append(co.co_names[oparg>>1])
+                else:
+                    # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* so far;
+                    # possibly more LOAD_ATTR/STORE_ATTR will follow
+                    pending.append(co.co_names[oparg])
                 continue
             # {LOAD_GLOBAL|LOAD_NAME} {LOAD_ATTR}* (and no more
             # LOAD_ATTR/STORE_ATTR)
@@ -1366,10 +1454,14 @@ def _find_loads_without_stores_in_code(co, loads_without_stores):
         if isinstance(arg, types.CodeType):
             _find_loads_without_stores_in_code(arg, loads_without_stores)
 
-
-def _op(c):
-    return c
-
+if sys.version_info >= (3,12):
+    from dis import hasarg
+    def take_arg(op):
+        return op in hasarg
+else:
+    def take_arg(op):
+        from opcode import HAVE_ARGUMENT
+        return op >= HAVE_ARGUMENT
 
 def _find_earliest_backjump_label(bytecode):
     """
@@ -1411,19 +1503,20 @@ def _find_earliest_backjump_label(bytecode):
       The earliest target of a backward jump, as an offset into the bytecode.
     """
     # Code based on dis.findlabels().
-    from opcode import HAVE_ARGUMENT, hasjrel, hasjabs
+    from opcode import hasjrel, hasjabs
     if not isinstance(bytecode, bytes):
         raise TypeError
     n = len(bytecode)
     earliest_backjump_label = n
     i = 0
     while i < n:
-        c = bytecode[i]
-        op = _op(c)
+        op = bytecode[i]
         i += 1
-        if op < HAVE_ARGUMENT:
+        if not take_arg(op):
             continue
-        oparg = _op(bytecode[i]) + _op(bytecode[i+1])*256
+        if i+1 >= len(bytecode):
+            break
+        oparg = bytecode[i] + bytecode[i+1]*256
         i += 2
         label = None
         if op in hasjrel:
@@ -1605,7 +1698,7 @@ def get_known_import(fullname, db=None):
     return None
 
 
-_IMPORT_FAILED = set()
+_IMPORT_FAILED:Set[Any] = set()
 """
 Set of imports we've already attempted and failed.
 """
@@ -1811,7 +1904,7 @@ def auto_import_symbol(fullname, namespaces, db=None, autoimported=None, post_im
     return True
 
 
-def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None):
+def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=None, *, extra_db=None):
     """
     Parse ``arg`` for symbols that need to be imported and automatically import
     them.
@@ -1865,6 +1958,8 @@ def auto_import(arg, namespaces, db=None, autoimported=None, post_import_hook=No
     if autoimported is None:
         autoimported = {}
     db = ImportDB.interpret_arg(db, target_filename=filename)
+    if extra_db:
+        db = db|extra_db
     ok = True
     for fullname in fullnames:
         ok &= auto_import_symbol(fullname, namespaces, db, autoimported, post_import_hook=post_import_hook)

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/pyflyby/_cmdline.py

@@ -4,10 +4,10 @@
 
 
 
+from   builtins                 import input
 import optparse
 import os
 import signal
-from   builtins                 import input
 import sys
 from   textwrap                 import dedent
 import traceback
@@ -285,7 +285,8 @@ def parse_args(
 def _default_on_error(filename):
     raise SystemExit("bad filename %s" % (filename,))
 
-def filename_args(args, on_error=_default_on_error):
+
+def filename_args(args: List[str], on_error=_default_on_error):
     """
     Return list of filenames given command-line arguments.
 
@@ -293,7 +294,9 @@ def filename_args(args, on_error=_default_on_error):
       ``list`` of `Filename`
     """
     if args:
-        return expand_py_files_from_args(args, on_error)
+        for a in args:
+            assert isinstance(a, str)
+        return expand_py_files_from_args([Filename(f) for f in args], on_error)
     elif not os.isatty(0):
         return [Filename.STDIN]
     else:
@@ -415,8 +418,8 @@ def process_actions(filenames:List[str], actions, modify_function,
             continue
     if errors:
         msg = "\n%s: encountered the following problems:\n" % (sys.argv[0],)
-        for e in errors:
-            lines = e.splitlines()
+        for er in errors:
+            lines = er.splitlines()
             msg += "    " + lines[0] + '\n'.join(
                 ("            %s"%line for line in lines[1:]))
         raise SystemExit(msg)

{pyflyby-1.9.3 → pyflyby-1.9.5}/bin/pyflyby/_comms.py

@@ -9,6 +9,8 @@ from   pyflyby._imports2s       import (SourceToSourceFileImportsTransformation,
 from   pyflyby._importstmt      import Import
 from   pyflyby._log             import logger
 
+from   typing                   import Dict, Any
+
 # These are comm targets that the frontend (lab/notebook) is expected to
 # open. At this point, we handle only missing imports and
 # formatting imports
@@ -23,7 +25,7 @@ PYFLYBY_END_MSG = "# END AUTO-GENERATED BLOCK\n"
 pyflyby_comm_targets= [MISSING_IMPORTS, FORMATTING_IMPORTS, TIDY_IMPORTS]
 
 # A map of the comms opened with a given target name.
-comms = {}
+comms:Dict[str, Any] = {}
 
 # TODO: Document the expected contract for the different
 # custom comm messages

pyflyby-1.9.5/bin/pyflyby/_dynimp.py

@@ -0,0 +1,152 @@
+"""
+Virtual module to create dynamic import at runtime.
+
+It is sometime desirable to have auto import which are define only during
+a session and never exist on a on-disk file.
+
+This is injects a Dict module loader as well as a dictionary registry of in
+memory module.
+
+This is mostly use in IPython for lazy variable initialisation without having
+to use proxy objects.
+
+To use, put the following in your IPython startup files
+(``~/.ipython/profile_default/startup/autoimp.py`), or in your IPython
+configuration file:
+
+
+.. code:: python
+
+    from pyflyby._dynimp import add_import
+
+    add_import("foo", "foo = 1")
+
+    add_import(
+        "df, data",
+        '''
+        import pandas as pd
+        data = [1,2,3]
+        df =  pd.DataFrame(data)
+    ''',
+    )
+
+Now at the IPython prompt, if the pyflyby extension is loaded (either because
+you started  using the ``py`` cli, or some configuration options like  ``ipython
+--TerminalIPythonApp.extra_extensions=pyflyby``. When trying to use an undefined
+variable like ``foo``, ``df`` or ``data``, the corresponding module will be
+executed and the relevant variable imported.
+
+
+"""
+import importlib.abc
+import importlib.util
+import sys
+
+from textwrap import dedent
+from typing import FrozenSet
+
+from pyflyby._importclns import ImportSet, Import
+
+module_dict = {}
+
+
+def add_import(names: str, code: str, *, strict: bool = True):
+    """
+    Add a runtime generated import module
+
+    Parameters
+    ----------
+    names: str
+        name, or comma separated list variable names that should be created by
+        executing and importing `code`.
+    code: str
+        potentially multiline string that will be turned into a module,
+        executed and from which variables listed in names can be imported.
+    strict: bool
+        Raise in case of problem loading IPython of if pyflyby extension not installed.
+        otherwise just ignore error
+
+
+
+    Examples
+    --------
+
+    >>> add_import('pd, df', '''
+    ...     import pandas a pd
+    ...
+    ...     df = pd.DataFrame([[1,2], [3,4]])
+    ... ''', strict=False)  # don't fail doctest
+
+    """
+    try:
+        ip = _raise_if_problem()
+    except Exception:
+        if strict:
+            raise
+        else:
+            return
+    return _add_import(ip, names, code)
+
+
+def _raise_if_problem():
+    try:
+        import IPython
+    except ModuleNotFoundError as e:
+        raise ImportError("Dynamic autoimport requires IPython to be installed") from e
+
+    ip = IPython.get_ipython()
+    if ip is None:
+        raise ImportError("Dynamic autoimport only work from within IPython")
+
+    if not hasattr(ip, "_auto_importer"):
+        raise ValueError(
+            "IPython needs to be loaded with pyflyby extension for lazy variable to work"
+        )
+    return ip
+
+
+def _add_import(ip, names: str, code: str) -> None:
+    """
+    private version of add_import
+    """
+    assert ip is not None
+    mang = "pyflyby_autoimport_" + names.replace(",", "_").replace(" ", "_")
+    a: FrozenSet[Import] = ImportSet(f"from {mang} import {names}")._importset
+    b: FrozenSet[Import] = ip._auto_importer.db.known_imports._importset
+    s_import: FrozenSet[Import] = a | b
+
+    ip._auto_importer.db.known_imports = ImportSet._from_imports(list(s_import))
+    module_dict[mang] = dedent(code)
+
+class DictLoader(importlib.abc.Loader):
+    """
+    A dict based loader for in-memory module definition.
+    """
+    def __init__(self, module_name, module_code):
+        self.module_name = module_name
+        self.module_code = module_code
+
+    def create_module(self, spec):
+        return None  # Use default module creation semantics
+
+    def exec_module(self, module):
+        """
+        we exec module code directly in memory
+        """
+        exec(self.module_code, module.__dict__)
+
+
+class DictFinder(importlib.abc.MetaPathFinder):
+    """
+    A meta path finder for abode DictLoader
+    """
+    def find_spec(self, fullname, path, target=None):
+        if fullname in module_dict:
+            module_code = module_dict[fullname]
+            loader = DictLoader(fullname, module_code)
+            return importlib.util.spec_from_loader(fullname, loader)
+        return None
+
+
+def inject():
+    sys.meta_path.insert(0, DictFinder())