Cython 3.2.0b3__py3-none-any.whl → 3.2.2__py3-none-any.whl

Cython/Compiler/Builtin.py

@@ -20,9 +20,16 @@ pyexec_globals_utility_code = UtilityCode.load("PyExecGlobals", "Builtins.c")
 globals_utility_code = UtilityCode.load("Globals", "Builtins.c")
 range_utility_code = UtilityCode.load("PyRange_Check", "Builtins.c")
 include_std_lib_h_utility_code = UtilityCode.load("IncludeStdlibH", "ModuleSetupCode.c")
-pysequence_multiply_utility_code = UtilityCode.load("PySequenceMultiply", "ObjectHandling.c")
 slice_accessor_utility_code = UtilityCode.load("PySliceAccessors", "Builtins.c")
 
+def make_sequence_multiply_method(typeobj_cname):
+    pysequence_multiply_utility_code = TempitaUtilityCode.load(
+        "BuiltinSequenceMultiply", "ObjectHandling.c",
+        context={'typeobj': typeobj_cname})
+    return BuiltinMethod("__mul__",  "Tz",   "T", f"__Pyx_{typeobj_cname}_Multiply",
+                         utility_code=pysequence_multiply_utility_code)
+
+
 # mapping from builtins to their C-level equivalents
 
 class _BuiltinOverride:
@@ -378,22 +385,18 @@ builtin_types_table = [
                                     ]),
 
     ("bytearray", "&PyByteArray_Type", [
-                                    BuiltinMethod("__mul__",  "Tz",   "T", "__Pyx_PySequence_Multiply",
-                                                  utility_code=pysequence_multiply_utility_code),
+                                    make_sequence_multiply_method("PyByteArray_Type"),
                                     ]),
     ("bytes",   "&PyBytes_Type",   [BuiltinMethod("join",  "TO",   "T", "__Pyx_PyBytes_Join",
                                                   utility_code=UtilityCode.load("StringJoin", "StringTools.c")),
-                                    BuiltinMethod("__mul__",  "Tz",   "T", "__Pyx_PySequence_Multiply",
-                                                  utility_code=pysequence_multiply_utility_code),
+                                    make_sequence_multiply_method("PyBytes_Type"),
                                     ]),
     ("str",     "&PyUnicode_Type", [BuiltinMethod("__contains__",  "TO",   "b", "PyUnicode_Contains"),
                                     BuiltinMethod("join",  "TO",   "T", "PyUnicode_Join"),
-                                    BuiltinMethod("__mul__",  "Tz",   "T", "__Pyx_PySequence_Multiply",
-                                                  utility_code=pysequence_multiply_utility_code),
+                                    make_sequence_multiply_method("PyUnicode_Type"),
                                     ]),
 
-    ("tuple",  "&PyTuple_Type",    [BuiltinMethod("__mul__", "Tz", "T", "__Pyx_PySequence_Multiply",
-                                                  utility_code=pysequence_multiply_utility_code),
+    ("tuple",  "&PyTuple_Type",    [make_sequence_multiply_method("PyTuple_Type"),
                                     ]),
 
     ("list",   "&PyList_Type",     [BuiltinMethod("insert",  "TzO",  "r", "PyList_Insert"),
@@ -402,8 +405,7 @@ builtin_types_table = [
                                                   utility_code=UtilityCode.load("ListAppend", "Optimize.c")),
                                     BuiltinMethod("extend",  "TO",   "r", "__Pyx_PyList_Extend",
                                                   utility_code=UtilityCode.load("ListExtend", "Optimize.c")),
-                                    BuiltinMethod("__mul__",  "Tz",   "T", "__Pyx_PySequence_Multiply",
-                                                  utility_code=pysequence_multiply_utility_code),
+                                    make_sequence_multiply_method("PyList_Type"),
                                     ]),
 
     ("dict",   "&PyDict_Type",     [BuiltinMethod("__contains__",  "TO",   "b", "PyDict_Contains"),

Cython/Compiler/Code.py

@@ -402,7 +402,7 @@ class AbstractUtilityCode:
 
     requires = None
 
-    def put_code(self, output):
+    def put_code(self, globalstate: "GlobalState", used_by=None) -> None:
         pass
 
     def get_tree(self, **kwargs):
@@ -738,6 +738,8 @@ class UtilityCode(UtilityCodeBase):
         self.file = file
         self.export = export
         self.shared_utility_functions = self.parse_export_functions(export) if export else []
+        if export:
+            self._validate_suitable_for_sharing()
 
         # cached for use in hash and eq
         self._parts_tuple = tuple(getattr(self, part, None) for part in self.code_parts)
@@ -824,8 +826,14 @@ class UtilityCode(UtilityCodeBase):
             self.specialize_list.append(s)
             return s
 
+    def _validate_suitable_for_sharing(self):
+        code_string = getattr(self, "impl")
+        if not code_string: return
+        assert "NAMED_CGLOBAL(moddict_cname)" not in code_string, \
+            f"moddict_cname should not be shared: {self}"
+
     @cython.final
-    def _put_code_section(self, writer: "CCodeWriter", output: "GlobalState", code_type: str):
+    def _put_code_section(self, writer: "CCodeWriter", output: "GlobalState", code_type: str, used_by=None):
         code_string = getattr(self, code_type)
         if not code_string:
             return
@@ -834,8 +842,10 @@ class UtilityCode(UtilityCodeBase):
 
         code_string, result_is_module_specific = process_utility_ccode(self, output, code_string)
 
-        code_type_name = code_type if code_type != 'impl' else ''
-        writer.putln(f"/* {self.name}{'.' if code_type_name else ''}{code_type_name} */")
+        used_by = f" (used by {used_by})" if used_by else ''
+        name = f"{self.name}.{code_type}" if code_type != 'impl' else self.name
+
+        writer.putln(f"/* {name}{used_by} */")
 
         if can_be_reused and not result_is_module_specific:
             # can be reused across modules
@@ -860,36 +870,36 @@ class UtilityCode(UtilityCodeBase):
             code.putln(f'static {shared.ret}(*{shared.name})({shared.params}); /*proto*/')
         code.putln()
 
-    def put_code(self, output: "GlobalState") -> None:
+    def put_code(self, globalstate: "GlobalState", used_by=None) -> None:
         has_shared_utility_code = bool(
-            self.shared_utility_functions and output.module_node.scope.context.shared_utility_qualified_name
+            self.shared_utility_functions and globalstate.module_node.scope.context.shared_utility_qualified_name
         )
 
         if self.requires and not has_shared_utility_code:
             for dependency in self.requires:
-                output.use_utility_code(dependency)
+                globalstate.use_utility_code(dependency, used_by=self.name)
 
         if has_shared_utility_code:
-            self._put_shared_function_declarations(output[self.proto_block])
-        output.shared_utility_functions.extend(self.shared_utility_functions)
+            self._put_shared_function_declarations(globalstate[self.proto_block])
+        globalstate.shared_utility_functions.extend(self.shared_utility_functions)
 
         if self.proto:
-            self._put_code_section(output[self.proto_block], output, 'proto')
+            self._put_code_section(globalstate[self.proto_block], globalstate, 'proto', used_by=used_by)
         if not has_shared_utility_code:
-            self._put_code_section(output[self.proto_block], output, 'export')
+            self._put_code_section(globalstate[self.proto_block], globalstate, 'export')
         if self.impl and not has_shared_utility_code:
-            self._put_code_section(output['utility_code_def'], output, 'impl')
+            self._put_code_section(globalstate['utility_code_def'], globalstate, 'impl', used_by=used_by)
         if self.cleanup and Options.generate_cleanup_code:
-            self._put_code_section(output['cleanup_globals'], output, 'cleanup')
+            self._put_code_section(globalstate['cleanup_globals'], globalstate, 'cleanup')
         if self.module_state_decls:
-            self._put_code_section(output['module_state_contents'], output, 'module_state_decls')
+            self._put_code_section(globalstate['module_state_contents'], globalstate, 'module_state_decls')
         if self.module_state_traverse:
-            self._put_code_section(output['module_state_traverse_contents'], output, 'module_state_traverse')
+            self._put_code_section(globalstate['module_state_traverse_contents'], globalstate, 'module_state_traverse')
         if self.module_state_clear:
-            self._put_code_section(output['module_state_clear_contents'], output, 'module_state_clear')
+            self._put_code_section(globalstate['module_state_clear_contents'], globalstate, 'module_state_clear')
 
         if self.init:
-            self._put_init_code_section(output)
+            self._put_init_code_section(globalstate)
 
 
 def add_macro_processor(*macro_names, regex=None, is_module_specific=False, _last_macro_processor = [None]):
@@ -1088,9 +1098,9 @@ class LazyUtilityCode(UtilityCodeBase):
     def __init__(self, callback):
         self.callback = callback
 
-    def put_code(self, globalstate):
+    def put_code(self, globalstate: "GlobalState", used_by=None) -> None:
         utility = self.callback(globalstate.rootwriter)
-        globalstate.use_utility_code(utility)
+        globalstate.use_utility_code(utility, used_by=used_by)
 
 
 class FunctionState:
@@ -2471,7 +2481,7 @@ class GlobalState:
     # Utility code state
     #
 
-    def use_utility_code(self, utility_code):
+    def use_utility_code(self, utility_code, used_by=None):
         """
         Adds code to the C file. utility_code should
         a) implement __eq__/__hash__ for the purpose of knowing whether the same
@@ -2482,7 +2492,7 @@ class GlobalState:
         """
         if utility_code and utility_code not in self.utility_codes:
             self.utility_codes.add(utility_code)
-            utility_code.put_code(self)
+            utility_code.put_code(self, used_by=used_by)
 
     def use_entry_utility_code(self, entry):
         if entry is None:

Cython/Compiler/ExprNodes.py

@@ -1813,7 +1813,7 @@ class BytesNode(ConstNode):
             node.type = dst_type
             return node
         elif dst_type in (PyrexTypes.c_uchar_ptr_type, PyrexTypes.c_const_uchar_ptr_type, PyrexTypes.c_void_ptr_type):
-            node.type = (PyrexTypes.c_const_char_ptr_type if dst_type == PyrexTypes.c_const_uchar_ptr_type
+            node.type = (PyrexTypes.c_const_char_ptr_type if dst_type.base_type.is_const
                          else PyrexTypes.c_char_ptr_type)
             return CastNode(node, dst_type)
         elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
@@ -1830,7 +1830,7 @@ class BytesNode(ConstNode):
     def generate_evaluation_code(self, code):
         if self.type.is_pyobject:
             result = code.get_py_string_const(self.value)
-        elif self.type.is_const:
+        elif (self.type.is_ptr or self.type.is_array) and self.type.base_type.is_const:
             result = code.get_string_const(self.value)
         else:
             # not const => use plain C string literal and cast to mutable type
@@ -1900,6 +1900,9 @@ class UnicodeNode(ConstNode):
                     dst_type.is_ptr and dst_type.base_type.is_void):
                 # Allow using '-3' enforced unicode literals in a C char/char*/void* context.
                 if self.bytes_value is not None:
+                    if dst_type.is_array:
+                        # Prevent an invalid assignment from a C string array and use a pointer instead.
+                        dst_type = dst_type.element_ptr_type()
                     return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env)
                 if env.directives['c_string_encoding']:
                     try:
@@ -9052,11 +9055,14 @@ class TupleNode(SequenceNode):
         return self.result_code
 
     def calculate_constant_result(self):
+        if self.mult_factor:
+            raise ValueError()  # may exceed the compile time memory
         self.constant_result = tuple([
                 arg.constant_result for arg in self.args])
 
     def compile_time_value(self, denv):
         values = self.compile_time_value_list(denv)
+        assert self.mult_factor is None, self.mult_factor  # set only after parsing
         try:
             return tuple(values)
         except Exception as e:
@@ -9170,9 +9176,12 @@ class ListNode(SequenceNode):
             else:
                 if len(self.args) < len(dst_type.scope.var_entries):
                     warning(self.pos, "Too few members for '%s'" % dst_type, 1)
-                for i, (arg, member) in enumerate(zip(self.original_args, dst_type.scope.var_entries)):
+                for i, (arg, coerced_arg, member) in enumerate(zip(self.original_args, self.args, dst_type.scope.var_entries)):
                     if isinstance(arg, CoerceToPyTypeNode):
                         arg = arg.arg
+                    elif member.type.is_struct_or_union and coerced_arg.is_dict_literal:
+                        # For struct assignments, use the coerced dict directly, not a struct type call etc.
+                        arg = coerced_arg
                     self.args[i] = arg.coerce_to(member.type, env)
             self.type = dst_type
         elif dst_type.is_ctuple:

Cython/Compiler/MemoryView.py

@@ -518,7 +518,6 @@ def get_copy_new_utility(pos, from_memview, to_memview):
             func_cname=copy_c_or_fortran_cname(to_memview),
             dtype_is_object=int(to_memview.dtype.is_pyobject),
         ),
-        requires=[copy_contents_new_utility],
     )
 
 
@@ -847,12 +846,7 @@ overlapping_utility = load_memview_c_utility("OverlappingSlices")
 refcount_utility = load_memview_c_utility("MemviewRefcount")
 slice_init_utility = load_memview_c_utility("MemviewSliceInit")
 memviewslice_declare_code = load_memview_c_utility("MemviewSliceStruct", context=template_context)
-
-copy_contents_new_utility = load_memview_c_utility(
-    "MemviewSliceCopyTemplate",
-    context=template_context,
-    # Requires general memoryview code - dependency is added below.
-)
+copy_contents_new_utility = load_memview_c_utility("MemviewSliceCopy")
 
 
 @Utils.cached_function
@@ -887,7 +881,6 @@ def _get_memoryview_shared_utility_code(shared_utility_qualified_name):
                 memviewslice_declare_code,
                 refcount_utility,
                 atomic_utility,
-                copy_contents_new_utility,
                 ],
     )
 

Cython/Compiler/Nodes.py

@@ -766,6 +766,7 @@ class CFuncDeclaratorNode(CDeclaratorNode):
                     level=2)
 
         if self.exception_check == '+':
+            self.cpp_check(env)
             env.add_include_file('ios')         # for std::ios_base::failure
             env.add_include_file('new')         # for std::bad_alloc
             env.add_include_file('stdexcept')
@@ -2019,6 +2020,10 @@ class FuncDefNode(StatNode, BlockNode):
                                 outer_scope=genv,
                                 parent_scope=env,
                                 scope_name=self.entry.cname)
+            # FIXME: why do GeneratorDefNode and GeneratorBodyDefNode use the same scope?
+            # This should hit the GeneratorBodyDefNode, not the GeneratorDefNode.
+            if self.is_generator_body or self.is_generator:
+                lenv.is_generator_scope = True
         else:
             lenv = LocalScope(name=self.entry.name,
                               outer_scope=genv,
@@ -3250,7 +3255,7 @@ class DefNode(FuncDefNode):
             from .ExprNodes import ConstNode
             exception_value = ConstNode.for_type(
                 self.pos, value=str(cfunc_type.exception_value), type=cfunc_type.return_type,
-                constant_result=cfunc_type.exception_value)
+                constant_result=cfunc_type.exception_value.python_value)
         declarator = CFuncDeclaratorNode(self.pos,
                                          base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                          args=self.args,
@@ -8992,7 +8997,6 @@ class CriticalSectionStatNode(TryFinallyStatNode):
     child_attrs = ["args"] + TryFinallyStatNode.child_attrs
 
     var_type = None
-    state_temp = None
     preserve_exception = False
     is_pymutex_critical_section = False
 
@@ -9002,8 +9006,6 @@ class CriticalSectionStatNode(TryFinallyStatNode):
         else:
             self.var_type = PyrexTypes.c_py_critical_section_type
 
-        self.create_state_temp_if_needed(pos, body)
-
         self.length_tag = str(len(args)) if len(args) > 1 else ""
 
         super().__init__(
@@ -9011,19 +9013,16 @@ class CriticalSectionStatNode(TryFinallyStatNode):
             args=args,
             body=body,
             finally_clause=CriticalSectionExitNode(
-                pos, length_tag=self.length_tag, critical_section=self),
+                pos, length_tag=self.length_tag),
             **kwds,
         )
 
-    def create_state_temp_if_needed(self, pos, body):
+    def check_for_yields(self):
         from .ParseTreeTransforms import YieldNodeCollector
         collector = YieldNodeCollector()
-        collector.visitchildren(body)
-        if not collector.yields:
-            return
-
-        from . import ExprNodes
-        self.state_temp = ExprNodes.TempNode(pos, self.var_type)
+        collector.visitchildren(self.body)
+        if collector.yields:
+            error(self.pos, f"Cannot yield while in a cython.critical_section.")
 
     def analyse_declarations(self, env):
         for arg in self.args:
@@ -9031,6 +9030,7 @@ class CriticalSectionStatNode(TryFinallyStatNode):
         return super().analyse_declarations(env)
 
     def analyse_expressions(self, env):
+        self.check_for_yields()
         cy_pymutex_type = PyrexTypes.get_cy_pymutex_type()
         mutex_count = 0
         for i, arg in enumerate(self.args):
@@ -9074,12 +9074,8 @@ class CriticalSectionStatNode(TryFinallyStatNode):
 
         code.mark_pos(self.pos)
         code.begin_block()
-        if self.state_temp:
-            self.state_temp.allocate(code)
-            variable = self.state_temp.result()
-        else:
-            variable = Naming.critical_section_variable
-            code.putln(f"{self.var_type.declaration_code(variable)};")
+        variable = Naming.critical_section_variable
+        code.putln(f"{self.var_type.declaration_code(variable)};")
 
         for arg in self.args:
             arg.generate_evaluation_code(code)
@@ -9097,16 +9093,13 @@ class CriticalSectionStatNode(TryFinallyStatNode):
             arg.generate_disposal_code(code)
             arg.free_temps(code)
 
-        if self.state_temp:
-            self.state_temp.release(code)
-
         code.end_block()
 
     def nogil_check(self, env):
         error(self.pos, "Critical sections require the GIL")
 
 
-class CriticalSectionExitNode(StatNode, CopyWithUpTreeRefsMixin):
+class CriticalSectionExitNode(StatNode):
     """
     critical_section - the CriticalSectionStatNode that owns this
     """
@@ -9117,13 +9110,8 @@ class CriticalSectionExitNode(StatNode, CopyWithUpTreeRefsMixin):
         return self
 
     def generate_execution_code(self, code):
-        if self.critical_section.state_temp:
-            variable_name = self.critical_section.state_temp.result()
-        else:
-            variable_name =  Naming.critical_section_variable
-
         code.putln(
-            f"__Pyx_PyCriticalSection{self.length_tag}_End(&{variable_name});"
+            f"__Pyx_PyCriticalSection{self.length_tag}_End(&{ Naming.critical_section_variable});"
         )
 
 class CythonLockStatNode(TryFinallyStatNode):
@@ -9162,29 +9150,12 @@ class CythonLockStatNode(TryFinallyStatNode):
         result.finally_except_clause = result.finally_clause
         return result
 
-    def check_for_yields(self):
-        from .ParseTreeTransforms import YieldNodeCollector
-        collector = YieldNodeCollector()
-        collector.visitchildren(self.body)
-        if collector.yields:
-            # DW - I've disallowed this because it seems like a deadlock disaster waiting to happen.
-            # I'm sure it's technically possible, and we can revise it if people have legitimate
-            # uses.
-            typename = self.arg.type.empty_declaration_code(pyrex=True).strip()
-            error(
-                self.pos,
-                f"Cannot use a 'with' statement with a '{typename}' in a generator. "
-                "If you really want to do this (and you are confident that there are no deadlocks) "
-                "then use try-finally."
-            )
-
     def analyse_declarations(self, env):
         self.arg.analyse_declarations(env)
         return super().analyse_declarations(env)
 
     def analyse_expressions(self, env):
         self.arg = self.arg.analyse_expressions(env)
-        self.check_for_yields()
         body = self.body
         if isinstance(body, StatListNode) and len(body.stats) >= 1:
             body = body.stats[0]

Cython/Compiler/Optimize.py

@@ -219,13 +219,14 @@ class IterationTransform(Visitor.EnvTransform):
         return self._optimise_for_loop(node, node.iterator.sequence)
 
     def _optimise_for_loop(self, node, iterable, reversed=False):
+        iter_type = iterable.type
         annotation_type = None
         if (iterable.is_name or iterable.is_attribute) and iterable.entry and iterable.entry.annotation:
             annotation = iterable.entry.annotation.expr
             if annotation.is_subscript:
                 annotation = annotation.base  # container base type
 
-        if Builtin.dict_type in (iterable.type, annotation_type):
+        if Builtin.dict_type in (iter_type, annotation_type):
             # like iterating over dict.keys()
             if reversed:
                 # CPython raises an error here: not a sequence
@@ -233,29 +234,31 @@ class IterationTransform(Visitor.EnvTransform):
             return self._transform_dict_iteration(
                 node, dict_obj=iterable, method=None, keys=True, values=False)
 
-        if (Builtin.set_type in (iterable.type, annotation_type) or
-                Builtin.frozenset_type in (iterable.type, annotation_type)):
+        if (Builtin.set_type in (iter_type, annotation_type) or
+                Builtin.frozenset_type in (iter_type, annotation_type)):
             if reversed:
                 # CPython raises an error here: not a sequence
                 return node
             return self._transform_set_iteration(node, iterable)
 
+        env = self.current_env()
+
         # C array (slice) iteration?
-        if iterable.type.is_ptr or iterable.type.is_array:
+        if iter_type.is_ptr or iter_type.is_array:
             return self._transform_carray_iteration(node, iterable, reversed=reversed)
-        if iterable.is_sequence_constructor:
+        if iterable.is_sequence_constructor and not env.is_generator_scope:
             # Convert iteration over homogeneous sequences of C types into array iteration.
-            env = self.current_env()
+            # FIXME: using ListNode in this way currently generates invalid C code inside of generator loops.
             item_type = ExprNodes.infer_sequence_item_type(
-                env, iterable, seq_type=iterable.type)
+                env, iterable, seq_type=iter_type)
             if item_type and not item_type.is_pyobject and not any(item.is_starred for item in iterable.args):
                 iterable = ExprNodes.ListNode(iterable.pos, args=iterable.args).analyse_types(env).coerce_to(
                     PyrexTypes.c_array_type(item_type, len(iterable.args)), env)
                 return self._transform_carray_iteration(node, iterable, reversed=reversed)
-        if iterable.is_string_literal:
+        if iterable.is_string_literal and not env.is_generator_scope:
             # Iterate over C array of single character values.
-            env = self.current_env()
-            if iterable.type is Builtin.unicode_type:
+            # FIXME: using ListNode in this way currently generates invalid C code inside of generator loops.
+            if iter_type is Builtin.unicode_type:
                 item_type = PyrexTypes.c_py_ucs4_type
                 items = map(ord, iterable.value)
             else:
@@ -266,13 +269,16 @@ class IterationTransform(Visitor.EnvTransform):
             iterable = ExprNodes.ListNode(iterable.pos, args=[as_int_node(ch)for ch in items])
             iterable = iterable.analyse_types(env).coerce_to(PyrexTypes.c_array_type(item_type, len(iterable.args)), env)
             return self._transform_carray_iteration(node, iterable, reversed=reversed)
-        if iterable.type is Builtin.bytes_type:
-            return self._transform_bytes_iteration(node, iterable, reversed=reversed)
-        if iterable.type is Builtin.unicode_type:
+        if iter_type is Builtin.bytes_type:
+            if env.is_generator_scope:
+                return self._transform_indexable_iteration(node, iterable, is_mutable=False, reversed=reversed)
+            else:
+                return self._transform_bytes_iteration(node, iterable, reversed=reversed)
+        if iter_type is Builtin.unicode_type:
             return self._transform_unicode_iteration(node, iterable, reversed=reversed)
         # in principle _transform_indexable_iteration would work on most of the above, and
         # also tuple and list. However, it probably isn't quite as optimized
-        if iterable.type is Builtin.bytearray_type:
+        if iter_type is Builtin.bytearray_type:
             return self._transform_indexable_iteration(node, iterable, is_mutable=True, reversed=reversed)
         if isinstance(iterable, ExprNodes.CoerceToPyTypeNode) and iterable.arg.type.is_memoryviewslice:
             return self._transform_indexable_iteration(node, iterable.arg, is_mutable=False, reversed=reversed)
@@ -601,7 +607,8 @@ class IterationTransform(Visitor.EnvTransform):
         exception_value=-1)
 
     def _transform_unicode_iteration(self, node, slice_node, reversed=False):
-        if slice_node.is_literal:
+        env = self.current_env()
+        if slice_node.is_literal and not env.is_generator_scope:
             # try to reduce to byte iteration for plain Latin-1 strings
             try:
                 bytes_value = bytes_literal(slice_node.value.encode('latin1'), 'iso8859-1')
@@ -614,7 +621,7 @@ class IterationTransform(Visitor.EnvTransform):
                         slice_node.pos, value=bytes_value,
                         constant_result=bytes_value,
                         type=PyrexTypes.c_const_char_ptr_type).coerce_to(
-                            PyrexTypes.c_const_uchar_ptr_type, self.current_env()),
+                            PyrexTypes.c_const_uchar_ptr_type, env),
                     start=None,
                     stop=ExprNodes.IntNode.for_size(slice_node.pos, len(bytes_value)),
                     type=Builtin.unicode_type,  # hint for Python conversion
@@ -646,8 +653,7 @@ class IterationTransform(Visitor.EnvTransform):
             is_temp = False,
             )
         if target_value.type != node.target.type:
-            target_value = target_value.coerce_to(node.target.type,
-                                                  self.current_env())
+            target_value = target_value.coerce_to(node.target.type, env)
         target_assign = Nodes.SingleAssignmentNode(
             pos = node.target.pos,
             lhs = node.target,

Cython/Compiler/PyrexTypes.py

@@ -832,8 +832,7 @@ class MemoryViewSliceType(PyrexType):
                     copy_func_type, pos=pos, defining=1,
                     cname=copy_cname)
 
-                utility = MemoryView.get_copy_new_utility(pos, self, to_memview)
-                env.use_utility_code(utility)
+                entry.utility_code_definition = MemoryView.get_copy_new_utility(pos, self, to_memview)
 
             MemoryView.use_cython_array_utility_code(env)
 
@@ -3822,9 +3821,9 @@ class ToPyStructUtilityCode(AbstractUtilityCode):
     def __hash__(self):
         return hash(self.header)
 
-    def put_code(self, output):
-        code = output['utility_code_def']
-        proto = output['utility_code_proto']
+    def put_code(self, globalstate, used_by=None):
+        code = globalstate['utility_code_def']
+        proto = globalstate['utility_code_proto']
 
         code.enter_cfunc_scope(self.env.global_scope())
         code.putln("%s {" % self.header)

Cython/Compiler/Symtab.py

@@ -364,6 +364,7 @@ class Scope:
     # is_c_class_scope  boolean            Is an extension type scope
     # is_local_scope    boolean            Is a local (i.e. function/method/generator) scope
     # is_closure_scope  boolean            Is a closure scope
+    # is_generator_scope  boolean          Is a closure scope of a generator
     # is_generator_expression_scope boolean   A subset of closure scope used for generator expressions
     # is_passthrough    boolean            Outer scope is passed directly
     # is_cpp_class_scope  boolean          Is a C++ class scope
@@ -386,6 +387,7 @@ class Scope:
     is_c_class_scope = 0
     is_closure_scope = 0
     is_local_scope = False
+    is_generator_scope = False
     is_generator_expression_scope = 0
     is_comprehension_scope = 0
     is_passthrough = 0
@@ -1255,6 +1257,20 @@ class Scope:
         return False
 
 
+class PreImportScope(Scope):
+
+    namespace_cname = Naming.preimport_cname
+
+    def __init__(self):
+        Scope.__init__(self, Options.pre_import, None, None)
+
+    def declare_builtin(self, name, pos):
+        entry = self.declare(name, name, py_object_type, pos, 'private')
+        entry.is_variable = True
+        entry.is_pyglobal = True
+        return entry
+
+
 class BuiltinScope(Scope):
     #  The builtin namespace.
 
@@ -2240,6 +2256,7 @@ class ClosureScope(LocalScope):
 
 
 class GeneratorExpressionScope(ClosureScope):
+    is_generator_scope = True
     is_generator_expression_scope = True
 
     def declare_assignment_expression_target(self, name, type, pos):

Cython/Compiler/UtilityCode.py

@@ -201,7 +201,7 @@ class CythonUtilityCode(Code.UtilityCodeBase):
         self.tree = tree
         return tree
 
-    def put_code(self, output):
+    def put_code(self, globalstate, used_by=None):
         pass
 
     @classmethod

Cython/Compiler/Visitor.py

@@ -549,7 +549,7 @@ class MethodDispatcherTransform(EnvTransform):
                 if Future.division in self.current_env().context.future_directives:
                     special_method_name = '__truediv__'
             obj_type = operand1.type
-            if obj_type.is_builtin_type:
+            if obj_type.is_builtin_type and not obj_type.is_exception_type:
                 type_name = obj_type.name
             else:
                 type_name = "object"  # safety measure
@@ -564,7 +564,7 @@ class MethodDispatcherTransform(EnvTransform):
         if special_method_name:
             operand = node.operand
             obj_type = operand.type
-            if obj_type.is_builtin_type:
+            if obj_type.is_builtin_type and not obj_type.is_exception_type:
                 type_name = obj_type.name
             else:
                 type_name = "object"  # safety measure
@@ -619,7 +619,8 @@ class MethodDispatcherTransform(EnvTransform):
                     # => see if it's usable instead
                     return self._delegate_to_assigned_value(
                         node, function, arg_list, kwargs)
-                if arg_list and entry.is_cmethod and entry.scope and entry.scope.parent_type.is_builtin_type:
+                if (arg_list and entry.is_cmethod and entry.scope and
+                        entry.scope.parent_type.is_builtin_type and not entry.scope.parent_type.is_exception_type):
                     if entry.scope.parent_type is arg_list[0].type:
                         # Optimised (unbound) method of a builtin type => try to "de-optimise".
                         return self._dispatch_to_method_handler(
@@ -650,7 +651,8 @@ class MethodDispatcherTransform(EnvTransform):
                 return node
             obj_type = self_arg.type
             is_unbound_method = False
-            if obj_type.is_builtin_type:
+            # Exceptions aren't necessarily exact types so could have unknown methods
+            if obj_type.is_builtin_type and not obj_type.is_exception_type:
                 if obj_type is Builtin.type_type and self_arg.is_name and arg_list and arg_list[0].type.is_pyobject:
                     # calling an unbound method like 'list.append(L,x)'
                     # (ignoring 'type.mro()' here ...)