ipython 9.2.0__tar.gz → 9.4.0__tar.gz

{ipython-9.2.0 → ipython-9.4.0}/IPython/core/completer.py

@@ -203,15 +203,11 @@ from types import SimpleNamespace
 from typing import (
     Iterable,
     Iterator,
-    List,
-    Tuple,
     Union,
     Any,
     Sequence,
-    Dict,
     Optional,
     TYPE_CHECKING,
-    Set,
     Sized,
     TypeVar,
     Literal,
@@ -236,10 +232,12 @@ from traitlets import (
     List as ListTrait,
     Unicode,
     Dict as DictTrait,
+    DottedObjectName,
     Union as UnionTrait,
     observe,
 )
 from traitlets.config.configurable import Configurable
+from traitlets.utils.importstring import import_item
 
 import __main__
 
@@ -987,7 +985,7 @@ class Completer(Configurable):
 
         - ``forbidden``: no evaluation of code is permitted,
         - ``minimal``: evaluation of literals and access to built-in namespace;
-          no item/attribute evaluationm no access to locals/globals,
+          no item/attribute evaluation, no access to locals/globals,
           no evaluation of any operations or comparisons.
         - ``limited``: access to all namespaces, evaluation of hard-coded methods
           (for example: :any:`dict.keys`, :any:`object.__getattr__`,
@@ -995,7 +993,9 @@ class Completer(Configurable):
           :any:`dict`, :any:`list`, :any:`tuple`, ``pandas.Series``),
         - ``unsafe``: evaluation of all methods and function calls but not of
           syntax with side-effects like `del x`,
-        - ``dangerous``: completely arbitrary evaluation.
+        - ``dangerous``: completely arbitrary evaluation; does not support auto-import.
+
+        To override specific elements of the policy, you can use ``policy_overrides`` trait.
         """,
     ).tag(config=True)
 
@@ -1031,6 +1031,35 @@ class Completer(Configurable):
         """,
     ).tag(config=True)
 
+    policy_overrides = DictTrait(
+        default_value={},
+        key_trait=Unicode(),
+        help="""Overrides for policy evaluation.
+
+        For example, to enable auto-import on completion specify:
+
+        .. code-block::
+
+            ipython --Completer.policy_overrides='{"allow_auto_import": True}' --Completer.use_jedi=False
+
+        """,
+    ).tag(config=True)
+
+    auto_import_method = DottedObjectName(
+        default_value="importlib.import_module",
+        allow_none=True,
+        help="""\
+        Provisional:
+              This is a provisional API in IPython 9.3, it may change without warnings.
+
+        A fully qualified path to an auto-import method for use by completer.
+        The function should take a single string and return `ModuleType` and
+        can raise `ImportError` exception if module is not found.
+
+        The default auto-import implementation does not populate the user namespace with the imported module.
+        """,
+    ).tag(config=True)
+
     def __init__(self, namespace=None, global_namespace=None, **kwargs):
         """Create a new completer for the command line.
 
@@ -1134,6 +1163,48 @@ class Completer(Configurable):
     # we simple attribute matching with normal identifiers.
     _ATTR_MATCH_RE = re.compile(r"(.+)\.(\w*)$")
 
+    def _strip_code_before_operator(self, code: str) -> str:
+        o_parens = {"(", "[", "{"}
+        c_parens = {")", "]", "}"}
+
+        # Dry-run tokenize to catch errors
+        try:
+            _ = list(tokenize.generate_tokens(iter(code.splitlines()).__next__))
+        except tokenize.TokenError:
+            # Try trimming the expression and retrying
+            trimmed_code = self._trim_expr(code)
+            try:
+                _ = list(
+                    tokenize.generate_tokens(iter(trimmed_code.splitlines()).__next__)
+                )
+                code = trimmed_code
+            except tokenize.TokenError:
+                return code
+
+        tokens = _parse_tokens(code)
+        encountered_operator = False
+        after_operator = []
+        nesting_level = 0
+
+        for t in tokens:
+            if t.type == tokenize.OP:
+                if t.string in o_parens:
+                    nesting_level += 1
+                elif t.string in c_parens:
+                    nesting_level -= 1
+                elif t.string != "." and nesting_level == 0:
+                    encountered_operator = True
+                    after_operator = []
+                    continue
+
+            if encountered_operator:
+                after_operator.append(t.string)
+
+        if encountered_operator:
+            return "".join(after_operator)
+        else:
+            return code
+
     def _attr_matches(
         self, text: str, include_prefix: bool = True
     ) -> tuple[Sequence[str], str]:
@@ -1141,9 +1212,12 @@ class Completer(Configurable):
         if not m2:
             return [], ""
         expr, attr = m2.group(1, 2)
+        try:
+            expr = self._strip_code_before_operator(expr)
+        except tokenize.TokenError:
+            pass
 
         obj = self._evaluate_expr(expr)
-
         if obj is not_found:
             return [], ""
 
@@ -1234,6 +1308,8 @@ class Completer(Configurable):
                         globals=self.global_namespace,
                         locals=self.namespace,
                         evaluation=self.evaluation,
+                        auto_import=self._auto_import,
+                        policy_overrides=self.policy_overrides,
                     ),
                 )
                 done = True
@@ -1247,6 +1323,14 @@ class Completer(Configurable):
                 expr = self._trim_expr(expr)
         return obj
 
+    @property
+    def _auto_import(self):
+        if self.auto_import_method is None:
+            return None
+        if not hasattr(self, "_auto_import_func"):
+            self._auto_import_func = import_item(self.auto_import_method)
+        return self._auto_import_func
+
 def get__all__entries(obj):
     """returns the strings in the __all__ attribute"""
     try:
@@ -2787,6 +2871,8 @@ class IPCompleter(Completer):
                 locals=self.namespace,
                 evaluation=self.evaluation,  # type: ignore
                 in_subscript=True,
+                auto_import=self._auto_import,
+                policy_overrides=self.policy_overrides,
             ),
         )
 

{ipython-9.2.0 → ipython-9.4.0}/IPython/core/debugger.py

@@ -133,6 +133,7 @@ from contextlib import contextmanager
 from functools import lru_cache
 
 from IPython import get_ipython
+from IPython.core.debugger_backport import PdbClosureBackport
 from IPython.utils import PyColorize
 from IPython.utils.PyColorize import TokenStream
 
@@ -140,9 +141,18 @@ from typing import TYPE_CHECKING
 from types import FrameType
 
 # We have to check this directly from sys.argv, config struct not yet available
-from pdb import Pdb as OldPdb
+from pdb import Pdb as _OldPdb
 from pygments.token import Token
 
+
+if sys.version_info < (3, 13):
+
+    class OldPdb(PdbClosureBackport, _OldPdb):
+        pass
+
+else:
+    OldPdb = _OldPdb
+
 if TYPE_CHECKING:
     # otherwise circular import
     from IPython.core.interactiveshell import InteractiveShell

ipython-9.4.0/IPython/core/debugger_backport.py

@@ -0,0 +1,206 @@
+"""
+The code in this module is a backport of cPython changes in Pdb
+that were introduced in Python 3.13 by gh-83151: Make closure work on pdb
+https://github.com/python/cpython/pull/111094.
+This file should be removed once IPython drops supports for Python 3.12.
+
+The only changes are:
+- reformatting by darker (black) formatter
+- addition of type-ignore comments to satisfy mypy
+
+Copyright (c) 2001 Python Software Foundation; All Rights Reserved
+
+PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
+--------------------------------------------
+
+1. This LICENSE AGREEMENT is between the Python Software Foundation
+("PSF"), and the Individual or Organization ("Licensee") accessing and
+otherwise using this software ("Python") in source or binary form and
+its associated documentation.
+
+2. Subject to the terms and conditions of this License Agreement, PSF hereby
+grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce,
+analyze, test, perform and/or display publicly, prepare derivative works,
+distribute, and otherwise use Python alone or in any derivative version,
+provided, however, that PSF's License Agreement and PSF's notice of copyright,
+i.e., "Copyright (c) 2001 Python Software Foundation; All Rights Reserved"
+are retained in Python alone or in any derivative version prepared by Licensee.
+
+3. In the event Licensee prepares a derivative work that is based on
+or incorporates Python or any part thereof, and wants to make
+the derivative work available to others as provided herein, then
+Licensee hereby agrees to include in any such work a brief summary of
+the changes made to Python.
+
+4. PSF is making Python available to Licensee on an "AS IS"
+basis.  PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR
+IMPLIED.  BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND
+DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS
+FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT
+INFRINGE ANY THIRD PARTY RIGHTS.
+
+5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON
+FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS
+A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON,
+OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF.
+
+6. This License Agreement will automatically terminate upon a material
+breach of its terms and conditions.
+
+7. Nothing in this License Agreement shall be deemed to create any
+relationship of agency, partnership, or joint venture between PSF and
+Licensee.  This License Agreement does not grant permission to use PSF
+trademarks or trade name in a trademark sense to endorse or promote
+products or services of Licensee, or any third party.
+
+8. By copying, installing or otherwise using Python, Licensee
+agrees to be bound by the terms and conditions of this License
+Agreement.
+"""
+
+import sys
+import types
+import codeop
+import textwrap
+from types import CodeType
+
+
+class PdbClosureBackport:
+    def _exec_in_closure(self, source, globals, locals):  # type: ignore[no-untyped-def]
+        """Run source code in closure so code object created within source
+        can find variables in locals correctly
+        returns True if the source is executed, False otherwise
+        """
+
+        # Determine if the source should be executed in closure. Only when the
+        # source compiled to multiple code objects, we should use this feature.
+        # Otherwise, we can just raise an exception and normal exec will be used.
+
+        code = compile(source, "<string>", "exec")
+        if not any(isinstance(const, CodeType) for const in code.co_consts):
+            return False
+
+        # locals could be a proxy which does not support pop
+        # copy it first to avoid modifying the original locals
+        locals_copy = dict(locals)
+
+        locals_copy["__pdb_eval__"] = {"result": None, "write_back": {}}
+
+        # If the source is an expression, we need to print its value
+        try:
+            compile(source, "<string>", "eval")
+        except SyntaxError:
+            pass
+        else:
+            source = "__pdb_eval__['result'] = " + source
+
+        # Add write-back to update the locals
+        source = (
+            "try:\n"
+            + textwrap.indent(source, "  ")
+            + "\n"
+            + "finally:\n"
+            + "  __pdb_eval__['write_back'] = locals()"
+        )
+
+        # Build a closure source code with freevars from locals like:
+        # def __pdb_outer():
+        #   var = None
+        #   def __pdb_scope():  # This is the code object we want to execute
+        #     nonlocal var
+        #     <source>
+        #   return __pdb_scope.__code__
+        source_with_closure = (
+            "def __pdb_outer():\n"
+            + "\n".join(f"  {var} = None" for var in locals_copy)
+            + "\n"
+            + "  def __pdb_scope():\n"
+            + "\n".join(f"    nonlocal {var}" for var in locals_copy)
+            + "\n"
+            + textwrap.indent(source, "    ")
+            + "\n"
+            + "  return __pdb_scope.__code__"
+        )
+
+        # Get the code object of __pdb_scope()
+        # The exec fills locals_copy with the __pdb_outer() function and we can call
+        # that to get the code object of __pdb_scope()
+        ns = {}
+        try:
+            exec(source_with_closure, {}, ns)
+        except Exception:
+            return False
+        code = ns["__pdb_outer"]()
+
+        cells = tuple(types.CellType(locals_copy.get(var)) for var in code.co_freevars)
+
+        try:
+            exec(code, globals, locals_copy, closure=cells)
+        except Exception:
+            return False
+
+        # get the data we need from the statement
+        pdb_eval = locals_copy["__pdb_eval__"]
+
+        # __pdb_eval__ should not be updated back to locals
+        pdb_eval["write_back"].pop("__pdb_eval__")
+
+        # Write all local variables back to locals
+        locals.update(pdb_eval["write_back"])
+        eval_result = pdb_eval["result"]
+        if eval_result is not None:
+            print(repr(eval_result))
+
+        return True
+
+    def default(self, line):  # type: ignore[no-untyped-def]
+        if line[:1] == "!":
+            line = line[1:].strip()
+        locals = self.curframe_locals
+        globals = self.curframe.f_globals
+        try:
+            buffer = line
+            if (
+                code := codeop.compile_command(line + "\n", "<stdin>", "single")
+            ) is None:
+                # Multi-line mode
+                with self._disable_command_completion():
+                    buffer = line
+                    continue_prompt = "...   "
+                    while (
+                        code := codeop.compile_command(buffer, "<stdin>", "single")
+                    ) is None:
+                        if self.use_rawinput:
+                            try:
+                                line = input(continue_prompt)
+                            except (EOFError, KeyboardInterrupt):
+                                self.lastcmd = ""
+                                print("\n")
+                                return
+                        else:
+                            self.stdout.write(continue_prompt)
+                            self.stdout.flush()
+                            line = self.stdin.readline()
+                            if not len(line):
+                                self.lastcmd = ""
+                                self.stdout.write("\n")
+                                self.stdout.flush()
+                                return
+                            else:
+                                line = line.rstrip("\r\n")
+                        buffer += "\n" + line
+            save_stdout = sys.stdout
+            save_stdin = sys.stdin
+            save_displayhook = sys.displayhook
+            try:
+                sys.stdin = self.stdin
+                sys.stdout = self.stdout
+                sys.displayhook = self.displayhook
+                if not self._exec_in_closure(buffer, globals, locals):
+                    exec(code, globals, locals)
+            finally:
+                sys.stdout = save_stdout
+                sys.stdin = save_stdin
+                sys.displayhook = save_displayhook
+        except:
+            self._error_exc()

{ipython-9.2.0 → ipython-9.4.0}/IPython/core/guarded_eval.py

@@ -1,19 +1,15 @@
+from copy import copy
 from inspect import isclass, signature, Signature
-from pygments.formatters.terminal256 import Terminal256Formatter
 from typing import (
     Annotated,
     AnyStr,
     Callable,
-    Dict,
     Literal,
     NamedTuple,
     NewType,
     Optional,
     Protocol,
-    Set,
     Sequence,
-    Tuple,
-    Type,
     TypeGuard,
     Union,
     get_args,
@@ -97,6 +93,7 @@ class EvaluationPolicy:
     allow_builtins_access: bool = False
     allow_all_operations: bool = False
     allow_any_calls: bool = False
+    allow_auto_import: bool = False
     allowed_calls: set[Callable] = field(default_factory=set)
 
     def can_get_item(self, value, item):
@@ -330,6 +327,10 @@ class EvaluationContext(NamedTuple):
     #: Whether the evaluation of code takes place inside of a subscript.
     #: Useful for evaluating ``:-1, 'col'`` in ``df[:-1, 'col']``.
     in_subscript: bool = False
+    #: Auto import method
+    auto_import: Callable[list[str], ModuleType] | None = None
+    #: Overrides for evaluation policy
+    policy_overrides: dict = {}
 
 
 class _IdentitySubscript:
@@ -463,6 +464,17 @@ def _find_dunder(node_op, dunders) -> Union[tuple[str, ...], None]:
     return dunder
 
 
+def get_policy(context: EvaluationContext) -> EvaluationPolicy:
+    policy = copy(EVALUATION_POLICIES[context.evaluation])
+
+    for key, value in context.policy_overrides.items():
+        if hasattr(policy, key):
+            setattr(policy, key, value)
+        else:
+            print(f"Incorrect policy override key: {key}")
+    return policy
+
+
 def eval_node(node: Union[ast.AST, None], context: EvaluationContext):
     """Evaluate AST node in provided context.
 
@@ -490,7 +502,8 @@ def eval_node(node: Union[ast.AST, None], context: EvaluationContext):
     The purpose of this function is to guard against unwanted side-effects;
     it does not give guarantees on protection from malicious code execution.
     """
-    policy = EVALUATION_POLICIES[context.evaluation]
+    policy = get_policy(context)
+
     if node is None:
         return None
     if isinstance(node, ast.Expression):
@@ -711,7 +724,7 @@ def _resolve_annotation(
 
 
 def _eval_node_name(node_id: str, context: EvaluationContext):
-    policy = EVALUATION_POLICIES[context.evaluation]
+    policy = get_policy(context)
     if policy.allow_locals_access and node_id in context.locals:
         return context.locals[node_id]
     if policy.allow_globals_access and node_id in context.globals:
@@ -719,6 +732,8 @@ def _eval_node_name(node_id: str, context: EvaluationContext):
     if policy.allow_builtins_access and hasattr(builtins, node_id):
         # note: do not use __builtins__, it is implementation detail of cPython
         return getattr(builtins, node_id)
+    if policy.allow_auto_import and context.auto_import:
+        return context.auto_import(node_id)
     if not policy.allow_globals_access and not policy.allow_locals_access:
         raise GuardRejection(
             f"Namespace access not allowed in {context.evaluation} mode"
@@ -728,7 +743,7 @@ def _eval_node_name(node_id: str, context: EvaluationContext):
 
 
 def _eval_or_create_duck(duck_type, node: ast.Call, context: EvaluationContext):
-    policy = EVALUATION_POLICIES[context.evaluation]
+    policy = get_policy(context)
     # if allow-listed builtin is on type annotation, instantiate it
     if policy.can_call(duck_type) and not node.keywords:
         args = [eval_node(arg, context) for arg in node.args]

{ipython-9.2.0 → ipython-9.4.0}/IPython/core/interactiveshell.py

@@ -252,13 +252,23 @@ class ExecutionInfo:
     Stores information about what is going to happen.
     """
     raw_cell = None
+    transformed_cell = None
     store_history = False
     silent = False
     shell_futures = True
     cell_id = None
 
-    def __init__(self, raw_cell, store_history, silent, shell_futures, cell_id):
+    def __init__(
+        self,
+        raw_cell,
+        store_history,
+        silent,
+        shell_futures,
+        cell_id,
+        transformed_cell=None,
+    ):
         self.raw_cell = raw_cell
+        self.transformed_cell = transformed_cell
         self.store_history = store_history
         self.silent = silent
         self.shell_futures = shell_futures
@@ -269,12 +279,18 @@ class ExecutionInfo:
         raw_cell = (
             (self.raw_cell[:50] + "..") if len(self.raw_cell) > 50 else self.raw_cell
         )
+        transformed_cell = (
+            (self.transformed_cell[:50] + "..")
+            if self.transformed_cell and len(self.transformed_cell) > 50
+            else self.transformed_cell
+        )
         return (
-            '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s cell_id=%s>'
+            '<%s object at %x, raw_cell="%s" transformed_cell="%s" store_history=%s silent=%s shell_futures=%s cell_id=%s>'
             % (
                 name,
                 id(self),
                 raw_cell,
+                transformed_cell,
                 self.store_history,
                 self.silent,
                 self.shell_futures,
@@ -290,7 +306,7 @@ class ExecutionResult:
     """
 
     execution_count: Optional[int] = None
-    error_before_exec: Optional[bool] = None
+    error_before_exec: Optional[BaseException] = None
     error_in_exec: Optional[BaseException] = None
     info = None
     result = None
@@ -314,6 +330,7 @@ class ExecutionResult:
         return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
                 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
 
+
 @functools.wraps(io_open)
 def _modified_open(file, *args, **kwargs):
     if file in {0, 1, 2}:
@@ -331,7 +348,7 @@ class InteractiveShell(SingletonConfigurable):
 
     _instance = None
     _user_ns: dict
-    _sys_modules_keys: set[str, AnyType]
+    _sys_modules_keys: set[str]
 
     inspector: oinspect.Inspector
 
@@ -1572,7 +1589,7 @@ class InteractiveShell(SingletonConfigurable):
             if isinstance(variables, str):
                 vlist = variables.split()
             else:
-                vlist = variables
+                vlist = list(variables)
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
@@ -1887,11 +1904,12 @@ class InteractiveShell(SingletonConfigurable):
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
-                docformat = (
-                    sphinxify(self.object_inspect(oname))
-                    if self.sphinxify_docstring
-                    else None
-                )
+                if self.sphinxify_docstring:
+                    if sphinxify is None:
+                        raise ImportError("Module ``docrepr`` required but missing")
+                    docformat = sphinxify(self.object_inspect(oname))
+                else:
+                    docformat = None
                 return self.inspector._get_info(
                     info.obj,
                     oname,
@@ -2196,7 +2214,7 @@ class InteractiveShell(SingletonConfigurable):
         except KeyboardInterrupt:
             print('\n' + self.get_exception_only(), file=sys.stderr)
 
-    def _showtraceback(self, etype, evalue, stb: str):
+    def _showtraceback(self, etype, evalue, stb: list[str]):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
@@ -3154,14 +3172,18 @@ class InteractiveShell(SingletonConfigurable):
         except BaseException as e:
             try:
                 info = ExecutionInfo(
-                    raw_cell, store_history, silent, shell_futures, cell_id
+                    raw_cell,
+                    store_history,
+                    silent,
+                    shell_futures,
+                    cell_id,
+                    transformed_cell=transformed_cell,
                 )
                 result = ExecutionResult(info)
                 result.error_in_exec = e
                 self.showtraceback(running_compiled_code=True)
             except:
                 pass
-
         return result
 
     def should_run_async(
@@ -3246,7 +3268,14 @@ class InteractiveShell(SingletonConfigurable):
 
         .. versionadded:: 7.0
         """
-        info = ExecutionInfo(raw_cell, store_history, silent, shell_futures, cell_id)
+        info = ExecutionInfo(
+            raw_cell,
+            store_history,
+            silent,
+            shell_futures,
+            cell_id,
+            transformed_cell=transformed_cell,
+        )
         result = ExecutionResult(info)
 
         if (not raw_cell) or raw_cell.isspace():
@@ -3599,7 +3628,7 @@ class InteractiveShell(SingletonConfigurable):
                 if mode == "exec":
                     mod = Module([node], [])
                 elif mode == "single":
-                    mod = ast.Interactive([node])  # type: ignore
+                    mod = ast.Interactive([node])
                 with compiler.extra_flags(
                     getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0)
                     if self.autoawait