Cython 3.1.0a1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (301) hide show
  1. Cython/Build/BuildExecutable.py +169 -0
  2. Cython/Build/Cache.py +199 -0
  3. Cython/Build/Cythonize.py +250 -0
  4. Cython/Build/Dependencies.py +1275 -0
  5. Cython/Build/Distutils.py +1 -0
  6. Cython/Build/Inline.py +342 -0
  7. Cython/Build/IpythonMagic.py +560 -0
  8. Cython/Build/Tests/TestCyCache.py +119 -0
  9. Cython/Build/Tests/TestCythonizeArgsParser.py +481 -0
  10. Cython/Build/Tests/TestDependencies.py +133 -0
  11. Cython/Build/Tests/TestInline.py +112 -0
  12. Cython/Build/Tests/TestIpythonMagic.py +287 -0
  13. Cython/Build/Tests/TestRecythonize.py +212 -0
  14. Cython/Build/Tests/TestStripLiterals.py +155 -0
  15. Cython/Build/Tests/__init__.py +1 -0
  16. Cython/Build/__init__.py +8 -0
  17. Cython/CodeWriter.py +811 -0
  18. Cython/Compiler/AnalysedTreeTransforms.py +97 -0
  19. Cython/Compiler/Annotate.py +326 -0
  20. Cython/Compiler/AutoDocTransforms.py +314 -0
  21. Cython/Compiler/Buffer.py +680 -0
  22. Cython/Compiler/Builtin.py +862 -0
  23. Cython/Compiler/CmdLine.py +243 -0
  24. Cython/Compiler/Code.pxd +145 -0
  25. Cython/Compiler/Code.py +3328 -0
  26. Cython/Compiler/CodeGeneration.py +33 -0
  27. Cython/Compiler/CythonScope.py +179 -0
  28. Cython/Compiler/Dataclass.py +868 -0
  29. Cython/Compiler/DebugFlags.py +21 -0
  30. Cython/Compiler/Errors.py +295 -0
  31. Cython/Compiler/ExprNodes.py +15051 -0
  32. Cython/Compiler/FlowControl.pxd +97 -0
  33. Cython/Compiler/FlowControl.py +1438 -0
  34. Cython/Compiler/FusedNode.py +998 -0
  35. Cython/Compiler/Future.py +16 -0
  36. Cython/Compiler/Interpreter.py +57 -0
  37. Cython/Compiler/Lexicon.py +340 -0
  38. Cython/Compiler/LineTable.py +114 -0
  39. Cython/Compiler/Main.py +779 -0
  40. Cython/Compiler/MatchCaseNodes.py +259 -0
  41. Cython/Compiler/MemoryView.py +860 -0
  42. Cython/Compiler/ModuleNode.py +4065 -0
  43. Cython/Compiler/Naming.py +369 -0
  44. Cython/Compiler/Nodes.py +10557 -0
  45. Cython/Compiler/Optimize.py +5269 -0
  46. Cython/Compiler/Options.py +828 -0
  47. Cython/Compiler/ParseTreeTransforms.pxd +78 -0
  48. Cython/Compiler/ParseTreeTransforms.py +4441 -0
  49. Cython/Compiler/Parsing.pxd +9 -0
  50. Cython/Compiler/Parsing.py +4797 -0
  51. Cython/Compiler/Pipeline.py +425 -0
  52. Cython/Compiler/PyrexTypes.py +5572 -0
  53. Cython/Compiler/Pythran.py +223 -0
  54. Cython/Compiler/Scanning.pxd +40 -0
  55. Cython/Compiler/Scanning.py +574 -0
  56. Cython/Compiler/StringEncoding.py +347 -0
  57. Cython/Compiler/Symtab.py +2998 -0
  58. Cython/Compiler/Tests/TestBuffer.py +105 -0
  59. Cython/Compiler/Tests/TestBuiltin.py +72 -0
  60. Cython/Compiler/Tests/TestCmdLine.py +573 -0
  61. Cython/Compiler/Tests/TestCode.py +86 -0
  62. Cython/Compiler/Tests/TestFlowControl.py +65 -0
  63. Cython/Compiler/Tests/TestGrammar.py +202 -0
  64. Cython/Compiler/Tests/TestMemView.py +71 -0
  65. Cython/Compiler/Tests/TestParseTreeTransforms.py +285 -0
  66. Cython/Compiler/Tests/TestScanning.py +134 -0
  67. Cython/Compiler/Tests/TestSignatureMatching.py +73 -0
  68. Cython/Compiler/Tests/TestStringEncoding.py +33 -0
  69. Cython/Compiler/Tests/TestTreeFragment.py +63 -0
  70. Cython/Compiler/Tests/TestTreePath.py +93 -0
  71. Cython/Compiler/Tests/TestTypes.py +75 -0
  72. Cython/Compiler/Tests/TestUtilityLoad.py +112 -0
  73. Cython/Compiler/Tests/TestVisitor.py +61 -0
  74. Cython/Compiler/Tests/Utils.py +36 -0
  75. Cython/Compiler/Tests/__init__.py +1 -0
  76. Cython/Compiler/TreeFragment.py +278 -0
  77. Cython/Compiler/TreePath.py +290 -0
  78. Cython/Compiler/TypeInference.py +584 -0
  79. Cython/Compiler/TypeSlots.py +1181 -0
  80. Cython/Compiler/UFuncs.py +311 -0
  81. Cython/Compiler/UtilNodes.py +387 -0
  82. Cython/Compiler/UtilityCode.py +274 -0
  83. Cython/Compiler/Version.py +8 -0
  84. Cython/Compiler/Visitor.pxd +53 -0
  85. Cython/Compiler/Visitor.py +861 -0
  86. Cython/Compiler/__init__.py +1 -0
  87. Cython/Coverage.py +443 -0
  88. Cython/Debugger/Cygdb.py +179 -0
  89. Cython/Debugger/DebugWriter.py +82 -0
  90. Cython/Debugger/Tests/TestLibCython.py +275 -0
  91. Cython/Debugger/Tests/__init__.py +1 -0
  92. Cython/Debugger/Tests/cfuncs.c +8 -0
  93. Cython/Debugger/Tests/codefile +49 -0
  94. Cython/Debugger/Tests/test_libcython_in_gdb.py +578 -0
  95. Cython/Debugger/Tests/test_libpython_in_gdb.py +90 -0
  96. Cython/Debugger/__init__.py +1 -0
  97. Cython/Debugger/libcython.py +1549 -0
  98. Cython/Debugger/libpython.py +2821 -0
  99. Cython/Debugging.py +20 -0
  100. Cython/Distutils/__init__.py +2 -0
  101. Cython/Distutils/build_ext.py +137 -0
  102. Cython/Distutils/extension.py +96 -0
  103. Cython/Distutils/old_build_ext.py +351 -0
  104. Cython/Includes/cpython/__init__.pxd +173 -0
  105. Cython/Includes/cpython/array.pxd +174 -0
  106. Cython/Includes/cpython/bool.pxd +37 -0
  107. Cython/Includes/cpython/buffer.pxd +112 -0
  108. Cython/Includes/cpython/bytearray.pxd +33 -0
  109. Cython/Includes/cpython/bytes.pxd +200 -0
  110. Cython/Includes/cpython/cellobject.pxd +35 -0
  111. Cython/Includes/cpython/ceval.pxd +8 -0
  112. Cython/Includes/cpython/codecs.pxd +121 -0
  113. Cython/Includes/cpython/complex.pxd +55 -0
  114. Cython/Includes/cpython/contextvars.pxd +141 -0
  115. Cython/Includes/cpython/conversion.pxd +36 -0
  116. Cython/Includes/cpython/datetime.pxd +384 -0
  117. Cython/Includes/cpython/descr.pxd +26 -0
  118. Cython/Includes/cpython/dict.pxd +187 -0
  119. Cython/Includes/cpython/exc.pxd +263 -0
  120. Cython/Includes/cpython/fileobject.pxd +57 -0
  121. Cython/Includes/cpython/float.pxd +47 -0
  122. Cython/Includes/cpython/function.pxd +65 -0
  123. Cython/Includes/cpython/genobject.pxd +25 -0
  124. Cython/Includes/cpython/getargs.pxd +12 -0
  125. Cython/Includes/cpython/instance.pxd +25 -0
  126. Cython/Includes/cpython/iterator.pxd +36 -0
  127. Cython/Includes/cpython/iterobject.pxd +24 -0
  128. Cython/Includes/cpython/list.pxd +92 -0
  129. Cython/Includes/cpython/long.pxd +149 -0
  130. Cython/Includes/cpython/longintrepr.pxd +19 -0
  131. Cython/Includes/cpython/mapping.pxd +63 -0
  132. Cython/Includes/cpython/marshal.pxd +66 -0
  133. Cython/Includes/cpython/mem.pxd +120 -0
  134. Cython/Includes/cpython/memoryview.pxd +50 -0
  135. Cython/Includes/cpython/method.pxd +49 -0
  136. Cython/Includes/cpython/module.pxd +208 -0
  137. Cython/Includes/cpython/number.pxd +258 -0
  138. Cython/Includes/cpython/object.pxd +433 -0
  139. Cython/Includes/cpython/pycapsule.pxd +143 -0
  140. Cython/Includes/cpython/pylifecycle.pxd +68 -0
  141. Cython/Includes/cpython/pyport.pxd +8 -0
  142. Cython/Includes/cpython/pystate.pxd +95 -0
  143. Cython/Includes/cpython/pythread.pxd +53 -0
  144. Cython/Includes/cpython/ref.pxd +67 -0
  145. Cython/Includes/cpython/sequence.pxd +134 -0
  146. Cython/Includes/cpython/set.pxd +119 -0
  147. Cython/Includes/cpython/slice.pxd +70 -0
  148. Cython/Includes/cpython/time.pxd +129 -0
  149. Cython/Includes/cpython/tuple.pxd +72 -0
  150. Cython/Includes/cpython/type.pxd +53 -0
  151. Cython/Includes/cpython/unicode.pxd +639 -0
  152. Cython/Includes/cpython/version.pxd +32 -0
  153. Cython/Includes/cpython/weakref.pxd +42 -0
  154. Cython/Includes/libc/__init__.pxd +1 -0
  155. Cython/Includes/libc/complex.pxd +35 -0
  156. Cython/Includes/libc/errno.pxd +127 -0
  157. Cython/Includes/libc/float.pxd +43 -0
  158. Cython/Includes/libc/limits.pxd +28 -0
  159. Cython/Includes/libc/locale.pxd +46 -0
  160. Cython/Includes/libc/math.pxd +209 -0
  161. Cython/Includes/libc/setjmp.pxd +10 -0
  162. Cython/Includes/libc/signal.pxd +64 -0
  163. Cython/Includes/libc/stddef.pxd +9 -0
  164. Cython/Includes/libc/stdint.pxd +105 -0
  165. Cython/Includes/libc/stdio.pxd +80 -0
  166. Cython/Includes/libc/stdlib.pxd +72 -0
  167. Cython/Includes/libc/string.pxd +50 -0
  168. Cython/Includes/libc/time.pxd +47 -0
  169. Cython/Includes/libcpp/__init__.pxd +4 -0
  170. Cython/Includes/libcpp/algorithm.pxd +320 -0
  171. Cython/Includes/libcpp/any.pxd +16 -0
  172. Cython/Includes/libcpp/atomic.pxd +59 -0
  173. Cython/Includes/libcpp/bit.pxd +29 -0
  174. Cython/Includes/libcpp/cast.pxd +12 -0
  175. Cython/Includes/libcpp/cmath.pxd +518 -0
  176. Cython/Includes/libcpp/complex.pxd +106 -0
  177. Cython/Includes/libcpp/deque.pxd +165 -0
  178. Cython/Includes/libcpp/execution.pxd +15 -0
  179. Cython/Includes/libcpp/forward_list.pxd +63 -0
  180. Cython/Includes/libcpp/functional.pxd +26 -0
  181. Cython/Includes/libcpp/iterator.pxd +34 -0
  182. Cython/Includes/libcpp/limits.pxd +61 -0
  183. Cython/Includes/libcpp/list.pxd +117 -0
  184. Cython/Includes/libcpp/map.pxd +252 -0
  185. Cython/Includes/libcpp/memory.pxd +115 -0
  186. Cython/Includes/libcpp/numbers.pxd +15 -0
  187. Cython/Includes/libcpp/numeric.pxd +131 -0
  188. Cython/Includes/libcpp/optional.pxd +34 -0
  189. Cython/Includes/libcpp/pair.pxd +1 -0
  190. Cython/Includes/libcpp/queue.pxd +25 -0
  191. Cython/Includes/libcpp/random.pxd +166 -0
  192. Cython/Includes/libcpp/set.pxd +228 -0
  193. Cython/Includes/libcpp/stack.pxd +11 -0
  194. Cython/Includes/libcpp/string.pxd +333 -0
  195. Cython/Includes/libcpp/typeindex.pxd +15 -0
  196. Cython/Includes/libcpp/typeinfo.pxd +10 -0
  197. Cython/Includes/libcpp/unordered_map.pxd +193 -0
  198. Cython/Includes/libcpp/unordered_set.pxd +152 -0
  199. Cython/Includes/libcpp/utility.pxd +30 -0
  200. Cython/Includes/libcpp/vector.pxd +167 -0
  201. Cython/Includes/openmp.pxd +50 -0
  202. Cython/Includes/posix/__init__.pxd +1 -0
  203. Cython/Includes/posix/dlfcn.pxd +14 -0
  204. Cython/Includes/posix/fcntl.pxd +86 -0
  205. Cython/Includes/posix/ioctl.pxd +4 -0
  206. Cython/Includes/posix/mman.pxd +101 -0
  207. Cython/Includes/posix/resource.pxd +57 -0
  208. Cython/Includes/posix/select.pxd +21 -0
  209. Cython/Includes/posix/signal.pxd +73 -0
  210. Cython/Includes/posix/stat.pxd +98 -0
  211. Cython/Includes/posix/stdio.pxd +37 -0
  212. Cython/Includes/posix/stdlib.pxd +29 -0
  213. Cython/Includes/posix/strings.pxd +9 -0
  214. Cython/Includes/posix/time.pxd +71 -0
  215. Cython/Includes/posix/types.pxd +30 -0
  216. Cython/Includes/posix/uio.pxd +26 -0
  217. Cython/Includes/posix/unistd.pxd +271 -0
  218. Cython/Includes/posix/wait.pxd +38 -0
  219. Cython/Plex/Actions.pxd +24 -0
  220. Cython/Plex/Actions.py +119 -0
  221. Cython/Plex/DFA.pxd +14 -0
  222. Cython/Plex/DFA.py +164 -0
  223. Cython/Plex/Errors.py +48 -0
  224. Cython/Plex/Lexicons.py +178 -0
  225. Cython/Plex/Machines.pxd +36 -0
  226. Cython/Plex/Machines.py +238 -0
  227. Cython/Plex/Regexps.py +539 -0
  228. Cython/Plex/Scanners.pxd +47 -0
  229. Cython/Plex/Scanners.py +360 -0
  230. Cython/Plex/Transitions.pxd +14 -0
  231. Cython/Plex/Transitions.py +239 -0
  232. Cython/Plex/__init__.py +34 -0
  233. Cython/Runtime/__init__.py +1 -0
  234. Cython/Runtime/refnanny.pyx +261 -0
  235. Cython/Shadow.py +656 -0
  236. Cython/Shadow.pyi +521 -0
  237. Cython/StringIOTree.py +170 -0
  238. Cython/Tempita/__init__.py +4 -0
  239. Cython/Tempita/_looper.py +154 -0
  240. Cython/Tempita/_tempita.py +1091 -0
  241. Cython/TestUtils.py +417 -0
  242. Cython/Tests/TestCodeWriter.py +128 -0
  243. Cython/Tests/TestCythonUtils.py +202 -0
  244. Cython/Tests/TestJediTyper.py +223 -0
  245. Cython/Tests/TestShadow.py +114 -0
  246. Cython/Tests/TestStringIOTree.py +67 -0
  247. Cython/Tests/TestTestUtils.py +90 -0
  248. Cython/Tests/__init__.py +1 -0
  249. Cython/Tests/xmlrunner.py +390 -0
  250. Cython/Utility/AsyncGen.c +1263 -0
  251. Cython/Utility/Buffer.c +875 -0
  252. Cython/Utility/Builtins.c +660 -0
  253. Cython/Utility/CConvert.pyx +134 -0
  254. Cython/Utility/CMath.c +95 -0
  255. Cython/Utility/CommonStructures.c +139 -0
  256. Cython/Utility/Complex.c +378 -0
  257. Cython/Utility/Coroutine.c +2413 -0
  258. Cython/Utility/CpdefEnums.pyx +108 -0
  259. Cython/Utility/CppConvert.pyx +279 -0
  260. Cython/Utility/CppSupport.cpp +133 -0
  261. Cython/Utility/CythonFunction.c +1851 -0
  262. Cython/Utility/Dataclasses.c +185 -0
  263. Cython/Utility/Dataclasses.py +112 -0
  264. Cython/Utility/Embed.c +125 -0
  265. Cython/Utility/Exceptions.c +1017 -0
  266. Cython/Utility/ExtensionTypes.c +797 -0
  267. Cython/Utility/FunctionArguments.c +573 -0
  268. Cython/Utility/ImportExport.c +912 -0
  269. Cython/Utility/MemoryView.pyx +1478 -0
  270. Cython/Utility/MemoryView_C.c +992 -0
  271. Cython/Utility/ModuleSetupCode.c +2501 -0
  272. Cython/Utility/NumpyImportArray.c +46 -0
  273. Cython/Utility/ObjectHandling.c +3054 -0
  274. Cython/Utility/Optimize.c +1533 -0
  275. Cython/Utility/Overflow.c +404 -0
  276. Cython/Utility/Printing.c +86 -0
  277. Cython/Utility/Profile.c +660 -0
  278. Cython/Utility/StringTools.c +1206 -0
  279. Cython/Utility/TestCyUtilityLoader.pyx +8 -0
  280. Cython/Utility/TestCythonScope.pyx +75 -0
  281. Cython/Utility/TestUtilityLoader.c +12 -0
  282. Cython/Utility/TypeConversion.c +1329 -0
  283. Cython/Utility/UFuncs.pyx +50 -0
  284. Cython/Utility/UFuncs_C.c +89 -0
  285. Cython/Utility/__init__.py +28 -0
  286. Cython/Utility/arrayarray.h +143 -0
  287. Cython/Utils.py +687 -0
  288. Cython/__init__.py +10 -0
  289. Cython/__init__.pyi +7 -0
  290. Cython/py.typed +0 -0
  291. Cython-3.1.0a1.dist-info/COPYING.txt +19 -0
  292. Cython-3.1.0a1.dist-info/LICENSE.txt +176 -0
  293. Cython-3.1.0a1.dist-info/METADATA +67 -0
  294. Cython-3.1.0a1.dist-info/RECORD +301 -0
  295. Cython-3.1.0a1.dist-info/WHEEL +5 -0
  296. Cython-3.1.0a1.dist-info/entry_points.txt +4 -0
  297. Cython-3.1.0a1.dist-info/top_level.txt +3 -0
  298. cython.py +29 -0
  299. pyximport/__init__.py +4 -0
  300. pyximport/pyxbuild.py +160 -0
  301. pyximport/pyximport.py +482 -0
@@ -0,0 +1,4441 @@
1
+ import cython
2
+ cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
3
+ Options=object, UtilNodes=object, LetNode=object,
4
+ LetRefNode=object, TreeFragment=object, EncodedString=object,
5
+ error=object, warning=object, copy=object, hashlib=object, sys=object,
6
+ itemgetter=object)
7
+
8
+ import copy
9
+ import hashlib
10
+ import sys
11
+ from operator import itemgetter
12
+
13
+ from . import PyrexTypes
14
+ from . import Naming
15
+ from . import ExprNodes
16
+ from . import Nodes
17
+ from . import Options
18
+ from . import Builtin
19
+ from . import Errors
20
+
21
+ from .Visitor import VisitorTransform, TreeVisitor
22
+ from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
23
+ from .UtilNodes import LetNode, LetRefNode
24
+ from .TreeFragment import TreeFragment
25
+ from .StringEncoding import EncodedString
26
+ from .Errors import error, warning, CompileError, InternalError
27
+
28
+
29
+ class SkipDeclarations:
30
+ """
31
+ Variable and function declarations can often have a deep tree structure,
32
+ and yet most transformations don't need to descend to this depth.
33
+
34
+ Declaration nodes are removed after AnalyseDeclarationsTransform, so there
35
+ is no need to use this for transformations after that point.
36
+ """
37
+ def visit_CTypeDefNode(self, node):
38
+ return node
39
+
40
+ def visit_CVarDefNode(self, node):
41
+ return node
42
+
43
+ def visit_CDeclaratorNode(self, node):
44
+ return node
45
+
46
+ def visit_CBaseTypeNode(self, node):
47
+ return node
48
+
49
+ def visit_CEnumDefNode(self, node):
50
+ return node
51
+
52
+ def visit_CStructOrUnionDefNode(self, node):
53
+ return node
54
+
55
+ def visit_CppClassNode(self, node):
56
+ if node.visibility != "extern":
57
+ # Need to traverse methods.
58
+ self.visitchildren(node)
59
+ return node
60
+
61
+
62
+ class NormalizeTree(CythonTransform):
63
+ """
64
+ This transform fixes up a few things after parsing
65
+ in order to make the parse tree more suitable for
66
+ transforms.
67
+
68
+ a) After parsing, blocks with only one statement will
69
+ be represented by that statement, not by a StatListNode.
70
+ When doing transforms this is annoying and inconsistent,
71
+ as one cannot in general remove a statement in a consistent
72
+ way and so on. This transform wraps any single statements
73
+ in a StatListNode containing a single statement.
74
+
75
+ b) The PassStatNode is a noop and serves no purpose beyond
76
+ plugging such one-statement blocks; i.e., once parsed a
77
+ ` "pass" can just as well be represented using an empty
78
+ StatListNode. This means less special cases to worry about
79
+ in subsequent transforms (one always checks to see if a
80
+ StatListNode has no children to see if the block is empty).
81
+ """
82
+
83
+ def __init__(self, context):
84
+ super().__init__(context)
85
+ self.is_in_statlist = False
86
+ self.is_in_expr = False
87
+
88
+ def visit_ModuleNode(self, node):
89
+ self.visitchildren(node)
90
+ if not isinstance(node.body, Nodes.StatListNode):
91
+ # This can happen when the body only consists of a single (unused) declaration and no statements.
92
+ node.body = Nodes.StatListNode(pos=node.pos, stats=[node.body])
93
+ return node
94
+
95
+ def visit_ExprNode(self, node):
96
+ stacktmp = self.is_in_expr
97
+ self.is_in_expr = True
98
+ self.visitchildren(node)
99
+ self.is_in_expr = stacktmp
100
+ return node
101
+
102
+ def visit_StatNode(self, node, is_listcontainer=False):
103
+ stacktmp = self.is_in_statlist
104
+ self.is_in_statlist = is_listcontainer
105
+ self.visitchildren(node)
106
+ self.is_in_statlist = stacktmp
107
+ if not self.is_in_statlist and not self.is_in_expr:
108
+ return Nodes.StatListNode(pos=node.pos, stats=[node])
109
+ else:
110
+ return node
111
+
112
+ def visit_StatListNode(self, node):
113
+ self.is_in_statlist = True
114
+ self.visitchildren(node)
115
+ self.is_in_statlist = False
116
+ return node
117
+
118
+ def visit_ParallelAssignmentNode(self, node):
119
+ return self.visit_StatNode(node, True)
120
+
121
+ def visit_CEnumDefNode(self, node):
122
+ return self.visit_StatNode(node, True)
123
+
124
+ def visit_CStructOrUnionDefNode(self, node):
125
+ return self.visit_StatNode(node, True)
126
+
127
+ def visit_ExprStatNode(self, node):
128
+ """Eliminate useless string literals"""
129
+ if node.expr.is_string_literal:
130
+ return Nodes.PassStatNode(node.expr.pos)
131
+ else:
132
+ return self.visit_StatNode(node)
133
+
134
+ def visit_CDeclaratorNode(self, node):
135
+ return node
136
+
137
+
138
+ class PostParseError(CompileError): pass
139
+
140
+ # error strings checked by unit tests, so define them
141
+ ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
142
+ ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
143
+ ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
144
+ class PostParse(ScopeTrackingTransform):
145
+ """
146
+ Basic interpretation of the parse tree, as well as validity
147
+ checking that can be done on a very basic level on the parse
148
+ tree (while still not being a problem with the basic syntax,
149
+ as such).
150
+
151
+ Specifically:
152
+ - Default values to cdef assignments are turned into single
153
+ assignments following the declaration (everywhere but in class
154
+ bodies, where they raise a compile error)
155
+
156
+ - Interpret some node structures into Python runtime values.
157
+ Some nodes take compile-time arguments (currently:
158
+ TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
159
+ which should be interpreted. This happens in a general way
160
+ and other steps should be taken to ensure validity.
161
+
162
+ Type arguments cannot be interpreted in this way.
163
+
164
+ - For __cythonbufferdefaults__ the arguments are checked for
165
+ validity.
166
+
167
+ TemplatedTypeNode has its directives interpreted:
168
+ Any first positional argument goes into the "dtype" attribute,
169
+ any "ndim" keyword argument goes into the "ndim" attribute and
170
+ so on. Also it is checked that the directive combination is valid.
171
+ - __cythonbufferdefaults__ attributes are parsed and put into the
172
+ type information.
173
+
174
+ Note: Currently Parsing.py does a lot of interpretation and
175
+ reorganization that can be refactored into this transform
176
+ if a more pure Abstract Syntax Tree is wanted.
177
+
178
+ - Some invalid uses of := assignment expressions are detected
179
+ """
180
+ def __init__(self, context):
181
+ super().__init__(context)
182
+ self.specialattribute_handlers = {
183
+ '__cythonbufferdefaults__' : self.handle_bufferdefaults
184
+ }
185
+ self.in_pattern_node = False
186
+
187
+ def visit_LambdaNode(self, node):
188
+ # unpack a lambda expression into the corresponding DefNode
189
+ collector = YieldNodeCollector()
190
+ collector.visitchildren(node.result_expr)
191
+ if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode):
192
+ body = Nodes.ExprStatNode(
193
+ node.result_expr.pos, expr=node.result_expr)
194
+ else:
195
+ body = Nodes.ReturnStatNode(
196
+ node.result_expr.pos, value=node.result_expr)
197
+ node.def_node = Nodes.DefNode(
198
+ node.pos, name=node.name,
199
+ args=node.args, star_arg=node.star_arg,
200
+ starstar_arg=node.starstar_arg,
201
+ body=body, doc=None)
202
+ self.visitchildren(node)
203
+ return node
204
+
205
+ def visit_GeneratorExpressionNode(self, node):
206
+ # unpack a generator expression into the corresponding DefNode
207
+ collector = YieldNodeCollector()
208
+ collector.visitchildren(node.loop, attrs=None, exclude=["iterator"])
209
+ node.def_node = Nodes.DefNode(
210
+ node.pos, name=node.name, doc=None,
211
+ args=[], star_arg=None, starstar_arg=None,
212
+ body=node.loop, is_async_def=collector.has_await,
213
+ is_generator_expression=True)
214
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
215
+ self.visitchildren(node)
216
+ return node
217
+
218
+ def visit_ComprehensionNode(self, node):
219
+ # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...)
220
+ if not node.has_local_scope:
221
+ collector = YieldNodeCollector()
222
+ collector.visitchildren(node.loop)
223
+ if collector.has_await:
224
+ node.has_local_scope = True
225
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
226
+ self.visitchildren(node)
227
+ return node
228
+
229
+ # cdef variables
230
+ def handle_bufferdefaults(self, decl):
231
+ if not isinstance(decl.default, ExprNodes.DictNode):
232
+ raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
233
+ self.scope_node.buffer_defaults_node = decl.default
234
+ self.scope_node.buffer_defaults_pos = decl.pos
235
+
236
+ def visit_CVarDefNode(self, node):
237
+ # This assumes only plain names and pointers are assignable on
238
+ # declaration. Also, it makes use of the fact that a cdef decl
239
+ # must appear before the first use, so we don't have to deal with
240
+ # "i = 3; cdef int i = i" and can simply move the nodes around.
241
+ try:
242
+ self.visitchildren(node)
243
+ stats = [node]
244
+ newdecls = []
245
+ for decl in node.declarators:
246
+ declbase = decl
247
+ while isinstance(declbase, Nodes.CPtrDeclaratorNode):
248
+ declbase = declbase.base
249
+ if isinstance(declbase, Nodes.CNameDeclaratorNode):
250
+ if declbase.default is not None:
251
+ if self.scope_type in ('cclass', 'pyclass', 'struct'):
252
+ if isinstance(self.scope_node, Nodes.CClassDefNode):
253
+ handler = self.specialattribute_handlers.get(decl.name)
254
+ if handler:
255
+ if decl is not declbase:
256
+ raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
257
+ handler(decl)
258
+ continue # Remove declaration
259
+ raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
260
+ first_assignment = self.scope_type != 'module'
261
+ stats.append(Nodes.SingleAssignmentNode(node.pos,
262
+ lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
263
+ rhs=declbase.default, first=first_assignment))
264
+ declbase.default = None
265
+ newdecls.append(decl)
266
+ node.declarators = newdecls
267
+ return stats
268
+ except PostParseError as e:
269
+ # An error in a cdef clause is ok, simply remove the declaration
270
+ # and try to move on to report more errors
271
+ self.context.nonfatal_error(e)
272
+ return None
273
+
274
+ # Split parallel assignments (a,b = b,a) into separate partial
275
+ # assignments that are executed rhs-first using temps. This
276
+ # restructuring must be applied before type analysis so that known
277
+ # types on rhs and lhs can be matched directly. It is required in
278
+ # the case that the types cannot be coerced to a Python type in
279
+ # order to assign from a tuple.
280
+
281
+ def visit_SingleAssignmentNode(self, node):
282
+ self.visitchildren(node)
283
+ return self._visit_assignment_node(node, [node.lhs, node.rhs])
284
+
285
+ def visit_CascadedAssignmentNode(self, node):
286
+ self.visitchildren(node)
287
+ return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
288
+
289
+ def _visit_assignment_node(self, node, expr_list):
290
+ """Flatten parallel assignments into separate single
291
+ assignments or cascaded assignments.
292
+ """
293
+ if sum([ 1 for expr in expr_list
294
+ if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
295
+ # no parallel assignments => nothing to do
296
+ return node
297
+
298
+ expr_list_list = []
299
+ flatten_parallel_assignments(expr_list, expr_list_list)
300
+ temp_refs = []
301
+ eliminate_rhs_duplicates(expr_list_list, temp_refs)
302
+
303
+ nodes = []
304
+ for expr_list in expr_list_list:
305
+ lhs_list = expr_list[:-1]
306
+ rhs = expr_list[-1]
307
+ if len(lhs_list) == 1:
308
+ node = Nodes.SingleAssignmentNode(rhs.pos,
309
+ lhs = lhs_list[0], rhs = rhs)
310
+ else:
311
+ node = Nodes.CascadedAssignmentNode(rhs.pos,
312
+ lhs_list = lhs_list, rhs = rhs)
313
+ nodes.append(node)
314
+
315
+ if len(nodes) == 1:
316
+ assign_node = nodes[0]
317
+ else:
318
+ assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
319
+
320
+ if temp_refs:
321
+ duplicates_and_temps = [ (temp.expression, temp)
322
+ for temp in temp_refs ]
323
+ sort_common_subsequences(duplicates_and_temps)
324
+ for _, temp_ref in duplicates_and_temps[::-1]:
325
+ assign_node = LetNode(temp_ref, assign_node)
326
+
327
+ return assign_node
328
+
329
+ def _flatten_sequence(self, seq, result):
330
+ for arg in seq.args:
331
+ if arg.is_sequence_constructor:
332
+ self._flatten_sequence(arg, result)
333
+ else:
334
+ result.append(arg)
335
+ return result
336
+
337
+ def visit_DelStatNode(self, node):
338
+ self.visitchildren(node)
339
+ node.args = self._flatten_sequence(node, [])
340
+ return node
341
+
342
+ def visit_ExceptClauseNode(self, node):
343
+ if node.is_except_as:
344
+ # except-as must delete NameNode target at the end
345
+ del_target = Nodes.DelStatNode(
346
+ node.pos,
347
+ args=[ExprNodes.NameNode(
348
+ node.target.pos, name=node.target.name)],
349
+ ignore_nonexisting=True)
350
+ node.body = Nodes.StatListNode(
351
+ node.pos,
352
+ stats=[Nodes.TryFinallyStatNode(
353
+ node.pos,
354
+ body=node.body,
355
+ finally_clause=Nodes.StatListNode(
356
+ node.pos,
357
+ stats=[del_target]))])
358
+ self.visitchildren(node)
359
+ return node
360
+
361
+ def visit_AssertStatNode(self, node):
362
+ """Extract the exception raising into a RaiseStatNode to simplify GIL handling.
363
+ """
364
+ if node.exception is None:
365
+ node.exception = Nodes.RaiseStatNode(
366
+ node.pos,
367
+ exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")),
368
+ exc_value=node.value,
369
+ exc_tb=None,
370
+ cause=None,
371
+ builtin_exc_name="AssertionError",
372
+ wrap_tuple_value=True,
373
+ )
374
+ node.value = None
375
+ self.visitchildren(node)
376
+ return node
377
+
378
+ def visit_ErrorNode(self, node):
379
+ error(node.pos, node.what)
380
+ return None
381
+
382
+ def visit_MatchCaseNode(self, node):
383
+ node.validate_targets()
384
+ self.visitchildren(node)
385
+ return node
386
+
387
+ def visit_MatchNode(self, node):
388
+ node.validate_irrefutable()
389
+ self.visitchildren(node)
390
+ return node
391
+
392
+ def visit_PatternNode(self, node):
393
+ in_pattern_node, self.in_pattern_node = self.in_pattern_node, True
394
+ self.visitchildren(node)
395
+ self.in_pattern_node = in_pattern_node
396
+ return node
397
+
398
+ def visit_JoinedStrNode(self, node):
399
+ if self.in_pattern_node:
400
+ error(node.pos, "f-strings are not accepted for pattern matching")
401
+ self.visitchildren(node)
402
+ return node
403
+
404
+ def visit_DefNode(self, node):
405
+ if (self.scope_type == "cclass" and
406
+ node.name in ["__getreadbuffer__", "__getwritebuffer__", "__getsegcount__", "__getcharbuffer__"]):
407
+ warning(node.pos, f"'{node.name}' relates to the old Python 2 buffer protocol "
408
+ "and is no longer used.", 2)
409
+ return None # drop the node - the arguments are invalid for a def node
410
+ return self.visit_FuncDefNode(node)
411
+
412
+
413
+ class _AssignmentExpressionTargetNameFinder(TreeVisitor):
414
+ def __init__(self):
415
+ super().__init__()
416
+ self.target_names = {}
417
+
418
+ def find_target_names(self, target):
419
+ if target.is_name:
420
+ return [target.name]
421
+ elif target.is_sequence_constructor:
422
+ names = []
423
+ for arg in target.args:
424
+ names.extend(self.find_target_names(arg))
425
+ return names
426
+ # other targets are possible, but it isn't necessary to investigate them here
427
+ return []
428
+
429
+ def visit_ForInStatNode(self, node):
430
+ self.target_names[node] = tuple(self.find_target_names(node.target))
431
+ self.visitchildren(node)
432
+
433
+ def visit_ComprehensionNode(self, node):
434
+ pass # don't recurse into nested comprehensions
435
+
436
+ def visit_LambdaNode(self, node):
437
+ pass # don't recurse into nested lambdas/generator expressions
438
+
439
+ def visit_Node(self, node):
440
+ self.visitchildren(node)
441
+
442
+
443
+ class _AssignmentExpressionChecker(TreeVisitor):
444
+ """
445
+ Enforces rules on AssignmentExpressions within generator expressions and comprehensions
446
+ """
447
+ def __init__(self, loop_node, scope_is_class):
448
+ super().__init__()
449
+
450
+ target_name_finder = _AssignmentExpressionTargetNameFinder()
451
+ target_name_finder.visit(loop_node)
452
+ self.target_names_dict = target_name_finder.target_names
453
+ self.in_iterator = False
454
+ self.in_nested_generator = False
455
+ self.scope_is_class = scope_is_class
456
+ self.current_target_names = ()
457
+ self.all_target_names = set()
458
+ for names in self.target_names_dict.values():
459
+ self.all_target_names.update(names)
460
+
461
+ def _reset_state(self):
462
+ old_state = (self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names)
463
+ # note: not resetting self.in_iterator here, see visit_LambdaNode() below
464
+ self.in_nested_generator = False
465
+ self.scope_is_class = False
466
+ self.current_target_names = ()
467
+ self.all_target_names = set()
468
+ return old_state
469
+
470
+ def _set_state(self, old_state):
471
+ self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names = old_state
472
+
473
+ @classmethod
474
+ def do_checks(cls, loop_node, scope_is_class):
475
+ checker = cls(loop_node, scope_is_class)
476
+ checker.visit(loop_node)
477
+
478
+ def visit_ForInStatNode(self, node):
479
+ if self.in_nested_generator:
480
+ self.visitchildren(node) # once nested, don't do anything special
481
+ return
482
+
483
+ current_target_names = self.current_target_names
484
+ target_name = self.target_names_dict.get(node, None)
485
+ if target_name:
486
+ self.current_target_names += target_name
487
+
488
+ self.in_iterator = True
489
+ self.visit(node.iterator)
490
+ self.in_iterator = False
491
+ self.visitchildren(node, exclude=("iterator",))
492
+
493
+ self.current_target_names = current_target_names
494
+
495
+ def visit_AssignmentExpressionNode(self, node):
496
+ if self.in_iterator:
497
+ error(node.pos, "assignment expression cannot be used in a comprehension iterable expression")
498
+ if self.scope_is_class:
499
+ error(node.pos, "assignment expression within a comprehension cannot be used in a class body")
500
+ if node.target_name in self.current_target_names:
501
+ error(node.pos, "assignment expression cannot rebind comprehension iteration variable '%s'" %
502
+ node.target_name)
503
+ elif node.target_name in self.all_target_names:
504
+ error(node.pos, "comprehension inner loop cannot rebind assignment expression target '%s'" %
505
+ node.target_name)
506
+
507
+ def visit_LambdaNode(self, node):
508
+ # Don't reset "in_iterator" - an assignment expression in a lambda in an
509
+ # iterator is explicitly tested by the Python testcases and banned.
510
+ old_state = self._reset_state()
511
+ # the lambda node's "def_node" is not set up at this point, so we need to recurse into it explicitly.
512
+ self.visit(node.result_expr)
513
+ self._set_state(old_state)
514
+
515
+ def visit_ComprehensionNode(self, node):
516
+ in_nested_generator = self.in_nested_generator
517
+ self.in_nested_generator = True
518
+ self.visitchildren(node)
519
+ self.in_nested_generator = in_nested_generator
520
+
521
+ def visit_GeneratorExpressionNode(self, node):
522
+ in_nested_generator = self.in_nested_generator
523
+ self.in_nested_generator = True
524
+ # def_node isn't set up yet, so we need to visit the loop directly.
525
+ self.visit(node.loop)
526
+ self.in_nested_generator = in_nested_generator
527
+
528
+ def visit_Node(self, node):
529
+ self.visitchildren(node)
530
+
531
+
532
+ def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
533
+ """Replace rhs items by LetRefNodes if they appear more than once.
534
+ Creates a sequence of LetRefNodes that set up the required temps
535
+ and appends them to ref_node_sequence. The input list is modified
536
+ in-place.
537
+ """
538
+ seen_nodes = set()
539
+ ref_nodes = {}
540
+ def find_duplicates(node):
541
+ if node.is_literal or node.is_name:
542
+ # no need to replace those; can't include attributes here
543
+ # as their access is not necessarily side-effect free
544
+ return
545
+ if node in seen_nodes:
546
+ if node not in ref_nodes:
547
+ ref_node = LetRefNode(node)
548
+ ref_nodes[node] = ref_node
549
+ ref_node_sequence.append(ref_node)
550
+ else:
551
+ seen_nodes.add(node)
552
+ if node.is_sequence_constructor:
553
+ for item in node.args:
554
+ find_duplicates(item)
555
+
556
+ for expr_list in expr_list_list:
557
+ rhs = expr_list[-1]
558
+ find_duplicates(rhs)
559
+ if not ref_nodes:
560
+ return
561
+
562
+ def substitute_nodes(node):
563
+ if node in ref_nodes:
564
+ return ref_nodes[node]
565
+ elif node.is_sequence_constructor:
566
+ node.args = list(map(substitute_nodes, node.args))
567
+ return node
568
+
569
+ # replace nodes inside of the common subexpressions
570
+ for node in ref_nodes:
571
+ if node.is_sequence_constructor:
572
+ node.args = list(map(substitute_nodes, node.args))
573
+
574
+ # replace common subexpressions on all rhs items
575
+ for expr_list in expr_list_list:
576
+ expr_list[-1] = substitute_nodes(expr_list[-1])
577
+
578
+ def sort_common_subsequences(items):
579
+ """Sort items/subsequences so that all items and subsequences that
580
+ an item contains appear before the item itself. This is needed
581
+ because each rhs item must only be evaluated once, so its value
582
+ must be evaluated first and then reused when packing sequences
583
+ that contain it.
584
+
585
+ This implies a partial order, and the sort must be stable to
586
+ preserve the original order as much as possible, so we use a
587
+ simple insertion sort (which is very fast for short sequences, the
588
+ normal case in practice).
589
+ """
590
+ def contains(seq, x):
591
+ for item in seq:
592
+ if item is x:
593
+ return True
594
+ elif item.is_sequence_constructor and contains(item.args, x):
595
+ return True
596
+ return False
597
+ def lower_than(a,b):
598
+ return b.is_sequence_constructor and contains(b.args, a)
599
+
600
+ for pos, item in enumerate(items):
601
+ key = item[1] # the ResultRefNode which has already been injected into the sequences
602
+ new_pos = pos
603
+ for i in range(pos-1, -1, -1):
604
+ if lower_than(key, items[i][0]):
605
+ new_pos = i
606
+ if new_pos != pos:
607
+ for i in range(pos, new_pos, -1):
608
+ items[i] = items[i-1]
609
+ items[new_pos] = item
610
+
611
+
612
+ def unpack_string_to_character_literals(literal):
613
+ chars = []
614
+ pos = literal.pos
615
+ stype = literal.__class__
616
+ sval = literal.value
617
+ sval_type = sval.__class__
618
+ for char in sval:
619
+ cval = sval_type(char)
620
+ chars.append(stype(pos, value=cval))
621
+ return chars
622
+
623
+
624
+ @cython.cfunc
625
+ def flatten_parallel_assignments(input: list, output: list):
626
+ # The input is a list of expression nodes, representing the LHSs
627
+ # and RHS of one (possibly cascaded) assignment statement. For
628
+ # sequence constructors, rearranges the matching parts of both
629
+ # sides into a list of equivalent assignments between the
630
+ # individual elements. This transformation is applied
631
+ # recursively, so that nested structures get matched as well.
632
+ rhs = input[-1]
633
+ if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
634
+ or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
635
+ output.append(input)
636
+ return
637
+
638
+ complete_assignments = []
639
+
640
+ if rhs.is_sequence_constructor:
641
+ rhs_args = rhs.args
642
+ elif rhs.is_string_literal:
643
+ rhs_args = unpack_string_to_character_literals(rhs)
644
+
645
+ starred_targets: cython.Py_ssize_t
646
+ lhs_size: cython.Py_ssize_t
647
+ rhs_size: cython.Py_ssize_t = len(rhs_args)
648
+ lhs_targets = [[] for _ in range(rhs_size)]
649
+ starred_assignments = []
650
+
651
+ for lhs in input[:-1]:
652
+ if not lhs.is_sequence_constructor:
653
+ if lhs.is_starred:
654
+ error(lhs.pos, "starred assignment target must be in a list or tuple")
655
+ complete_assignments.append(lhs)
656
+ continue
657
+ lhs_size = len(lhs.args)
658
+ starred_targets = 0
659
+ for expr in lhs.args:
660
+ starred_targets += bool(expr.is_starred)
661
+ if starred_targets > 1:
662
+ error(lhs.pos, "more than 1 starred expression in assignment")
663
+ output.append([lhs,rhs])
664
+ continue
665
+ elif lhs_size - starred_targets > rhs_size:
666
+ error(lhs.pos, "need more than %d value%s to unpack"
667
+ % (rhs_size, (rhs_size != 1) and 's' or ''))
668
+ output.append([lhs,rhs])
669
+ continue
670
+ elif starred_targets:
671
+ map_starred_assignment(lhs_targets, starred_assignments,
672
+ lhs.args, rhs_args)
673
+ elif lhs_size < rhs_size:
674
+ error(lhs.pos, "too many values to unpack (expected %d, got %d)"
675
+ % (lhs_size, rhs_size))
676
+ output.append([lhs,rhs])
677
+ continue
678
+ else:
679
+ for targets, expr in zip(lhs_targets, lhs.args):
680
+ targets.append(expr)
681
+
682
+ if complete_assignments:
683
+ complete_assignments.append(rhs)
684
+ output.append(complete_assignments)
685
+
686
+ # recursively flatten partial assignments
687
+ for cascade, rhs in zip(lhs_targets, rhs_args):
688
+ if cascade:
689
+ cascade.append(rhs)
690
+ flatten_parallel_assignments(cascade, output)
691
+
692
+ # recursively flatten starred assignments
693
+ for cascade in starred_assignments:
694
+ if cascade[0].is_sequence_constructor:
695
+ flatten_parallel_assignments(cascade, output)
696
+ else:
697
+ output.append(cascade)
698
+
699
+
700
+ @cython.cfunc
701
+ def map_starred_assignment(lhs_targets: list, starred_assignments: list, lhs_args: list, rhs_args: list):
702
+ # Appends the fixed-position LHS targets to the target list that
703
+ # appear left and right of the starred argument.
704
+ #
705
+ # The starred_assignments list receives a new tuple
706
+ # (lhs_target, rhs_values_list) that maps the remaining arguments
707
+ # (those that match the starred target) to a list.
708
+
709
+ # left side of the starred target
710
+ i: cython.Py_ssize_t
711
+ starred: cython.Py_ssize_t
712
+ lhs_remaining: cython.Py_ssize_t
713
+ for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
714
+ if expr.is_starred:
715
+ starred = i
716
+ lhs_remaining = len(lhs_args) - i - 1
717
+ break
718
+ targets.append(expr)
719
+ else:
720
+ raise InternalError("no starred arg found when splitting starred assignment")
721
+
722
+ # right side of the starred target
723
+ for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
724
+ lhs_args[starred + 1:])):
725
+ targets.append(expr)
726
+
727
+ # the starred target itself, must be assigned a (potentially empty) list
728
+ target = lhs_args[starred].target # unpack starred node
729
+ starred_rhs = rhs_args[starred:]
730
+ if lhs_remaining:
731
+ starred_rhs = starred_rhs[:-lhs_remaining]
732
+ if starred_rhs:
733
+ pos = starred_rhs[0].pos
734
+ else:
735
+ pos = target.pos
736
+ starred_assignments.append([
737
+ target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
738
+
739
+
740
+ class PxdPostParse(CythonTransform, SkipDeclarations):
741
+ """
742
+ Basic interpretation/validity checking that should only be
743
+ done on pxd trees.
744
+
745
+ A lot of this checking currently happens in the parser; but
746
+ what is listed below happens here.
747
+
748
+ - "def" functions are let through only if they fill the
749
+ getbuffer/releasebuffer slots
750
+
751
+ - cdef functions are let through only if they are on the
752
+ top level and are declared "inline"
753
+ """
754
+ ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
755
+ ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
756
+
757
+ def __call__(self, node):
758
+ self.scope_type = 'pxd'
759
+ return super().__call__(node)
760
+
761
+ def visit_CClassDefNode(self, node):
762
+ old = self.scope_type
763
+ self.scope_type = 'cclass'
764
+ self.visitchildren(node)
765
+ self.scope_type = old
766
+ return node
767
+
768
+ def visit_FuncDefNode(self, node):
769
+ # FuncDefNode always come with an implementation (without
770
+ # an imp they are CVarDefNodes..)
771
+ err = self.ERR_INLINE_ONLY
772
+
773
+ if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
774
+ and node.name in ('__getbuffer__', '__releasebuffer__')):
775
+ err = None # allow these slots
776
+
777
+ if isinstance(node, Nodes.CFuncDefNode):
778
+ if ('inline' in node.modifiers and
779
+ self.scope_type in ('pxd', 'cclass')):
780
+ node.inline_in_pxd = True
781
+ if node.visibility != 'private':
782
+ err = self.ERR_NOGO_WITH_INLINE % node.visibility
783
+ elif node.api:
784
+ err = self.ERR_NOGO_WITH_INLINE % 'api'
785
+ else:
786
+ err = None # allow inline function
787
+ else:
788
+ err = self.ERR_INLINE_ONLY
789
+
790
+ if err:
791
+ self.context.nonfatal_error(PostParseError(node.pos, err))
792
+ return None
793
+ else:
794
+ return node
795
+
796
+
797
+ class TrackNumpyAttributes(VisitorTransform, SkipDeclarations):
798
+ # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two.
799
+ def __init__(self):
800
+ super().__init__()
801
+ self.numpy_module_names = set()
802
+
803
+ def visit_CImportStatNode(self, node):
804
+ if node.module_name == "numpy":
805
+ self.numpy_module_names.add(node.as_name or "numpy")
806
+ return node
807
+
808
+ def visit_AttributeNode(self, node):
809
+ self.visitchildren(node)
810
+ obj = node.obj
811
+ if (obj.is_name and obj.name in self.numpy_module_names) or obj.is_numpy_attribute:
812
+ node.is_numpy_attribute = True
813
+ return node
814
+
815
+ visit_Node = VisitorTransform.recurse_to_children
816
+
817
+
818
+ class InterpretCompilerDirectives(CythonTransform):
819
+ """
820
+ After parsing, directives can be stored in a number of places:
821
+ - #cython-comments at the top of the file (stored in ModuleNode)
822
+ - Command-line arguments overriding these
823
+ - @cython.directivename decorators
824
+ - with cython.directivename: statements
825
+ - replaces "cython.compiled" with BoolNode(value=True)
826
+ allowing unreachable blocks to be removed at a fairly early stage
827
+ before cython typing rules are forced on applied
828
+
829
+ This transform is responsible for interpreting these various sources
830
+ and store the directive in two ways:
831
+ - Set the directives attribute of the ModuleNode for global directives.
832
+ - Use a CompilerDirectivesNode to override directives for a subtree.
833
+
834
+ (The first one is primarily to not have to modify with the tree
835
+ structure, so that ModuleNode stay on top.)
836
+
837
+ The directives are stored in dictionaries from name to value in effect.
838
+ Each such dictionary is always filled in for all possible directives,
839
+ using default values where no value is given by the user.
840
+
841
+ The available directives are controlled in Options.py.
842
+
843
+ Note that we have to run this prior to analysis, and so some minor
844
+ duplication of functionality has to occur: We manually track cimports
845
+ and which names the "cython" module may have been imported to.
846
+ """
847
+ unop_method_nodes = {
848
+ 'typeof': ExprNodes.TypeofNode,
849
+
850
+ 'operator.address': ExprNodes.AmpersandNode,
851
+ 'operator.dereference': ExprNodes.DereferenceNode,
852
+ 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
853
+ 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
854
+ 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
855
+ 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
856
+ 'operator.typeid' : ExprNodes.TypeidNode,
857
+
858
+ # For backwards compatibility.
859
+ 'address': ExprNodes.AmpersandNode,
860
+ }
861
+
862
+ binop_method_nodes = {
863
+ 'operator.comma' : ExprNodes.c_binop_constructor(','),
864
+ }
865
+
866
+ special_methods = {
867
+ 'declare', 'union', 'struct', 'typedef',
868
+ 'sizeof', 'cast', 'pointer', 'compiled',
869
+ 'NULL', 'fused_type', 'parallel',
870
+ }
871
+ special_methods.update(unop_method_nodes)
872
+
873
+ valid_cython_submodules = {
874
+ 'cimports',
875
+ 'dataclasses',
876
+ 'operator',
877
+ 'parallel',
878
+ 'view',
879
+ }
880
+
881
+ valid_parallel_directives = {
882
+ "parallel",
883
+ "prange",
884
+ "threadid",
885
+ #"threadsavailable",
886
+ }
887
+
888
+ def __init__(self, context, compilation_directive_defaults):
889
+ super().__init__(context)
890
+ self.cython_module_names = set()
891
+ self.directive_names = {'staticmethod': 'staticmethod'}
892
+ self.parallel_directives = {}
893
+ directives = copy.deepcopy(Options.get_directive_defaults())
894
+ for key, value in compilation_directive_defaults.items():
895
+ directives[str(key)] = copy.deepcopy(value)
896
+ self.directives = directives
897
+
898
+ def check_directive_scope(self, pos, directive, scope):
899
+ legal_scopes = Options.directive_scopes.get(directive, None)
900
+ if legal_scopes and scope not in legal_scopes:
901
+ self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
902
+ 'is not allowed in %s scope' % (directive, scope)))
903
+ return False
904
+ else:
905
+ if directive not in Options.directive_types:
906
+ error(pos, "Invalid directive: '%s'." % (directive,))
907
+ return True
908
+
909
+ def _check_valid_cython_module(self, pos, module_name):
910
+ if not module_name.startswith("cython."):
911
+ return
912
+ submodule = module_name.split('.', 2)[1]
913
+ if submodule in self.valid_cython_submodules:
914
+ return
915
+
916
+ extra = ""
917
+ # This is very rarely used, so don't waste space on static tuples.
918
+ hints = [
919
+ line.split() for line in """\
920
+ imp cimports
921
+ cimp cimports
922
+ para parallel
923
+ parra parallel
924
+ dataclass dataclasses
925
+ """.splitlines()[:-1]
926
+ ]
927
+ for wrong, correct in hints:
928
+ if module_name.startswith("cython." + wrong):
929
+ extra = "Did you mean 'cython.%s' ?" % correct
930
+ break
931
+ if not extra:
932
+ is_simple_cython_name = submodule in Options.directive_types
933
+ if not is_simple_cython_name and not submodule.startswith("_"):
934
+ # Try to find it in the Shadow module (i.e. the pure Python namespace of cython.*).
935
+ # FIXME: use an internal reference of "cython.*" names instead of Shadow.py
936
+ from .. import Shadow
937
+ is_simple_cython_name = hasattr(Shadow, submodule)
938
+ if is_simple_cython_name:
939
+ extra = "Instead, use 'import cython' and then 'cython.%s'." % submodule
940
+
941
+ error(pos, "'%s' is not a valid cython.* module%s%s" % (
942
+ module_name,
943
+ ". " if extra else "",
944
+ extra,
945
+ ))
946
+
947
+ # Set up processing and handle the cython: comments.
948
+ def visit_ModuleNode(self, node):
949
+ for key in sorted(node.directive_comments):
950
+ if not self.check_directive_scope(node.pos, key, 'module'):
951
+ self.wrong_scope_error(node.pos, key, 'module')
952
+ del node.directive_comments[key]
953
+
954
+ self.module_scope = node.scope
955
+
956
+ self.directives.update(node.directive_comments)
957
+ node.directives = self.directives
958
+ node.parallel_directives = self.parallel_directives
959
+ self.visitchildren(node)
960
+ node.cython_module_names = self.cython_module_names
961
+ return node
962
+
963
+ def visit_CompilerDirectivesNode(self, node):
964
+ old_directives, self.directives = self.directives, node.directives
965
+ self.visitchildren(node)
966
+ self.directives = old_directives
967
+ return node
968
+
969
+ # The following four functions track imports and cimports that
970
+ # begin with "cython"
971
+ def is_cython_directive(self, name):
972
+ return (name in Options.directive_types or
973
+ name in self.special_methods or
974
+ PyrexTypes.parse_basic_type(name))
975
+
976
+ def is_parallel_directive(self, full_name, pos):
977
+ """
978
+ Checks to see if fullname (e.g. cython.parallel.prange) is a valid
979
+ parallel directive. If it is a star import it also updates the
980
+ parallel_directives.
981
+ """
982
+ result = (full_name + ".").startswith("cython.parallel.")
983
+
984
+ if result:
985
+ directive = full_name.split('.')
986
+ if full_name == "cython.parallel":
987
+ self.parallel_directives["parallel"] = "cython.parallel"
988
+ elif full_name == "cython.parallel.*":
989
+ for name in self.valid_parallel_directives:
990
+ self.parallel_directives[name] = "cython.parallel.%s" % name
991
+ elif (len(directive) != 3 or
992
+ directive[-1] not in self.valid_parallel_directives):
993
+ error(pos, "No such directive: %s" % full_name)
994
+
995
+ return result
996
+
997
+ def visit_CImportStatNode(self, node):
998
+ module_name = node.module_name
999
+ if module_name == "cython.cimports":
1000
+ error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.")
1001
+ if module_name.startswith("cython.cimports."):
1002
+ if node.as_name and node.as_name != 'cython':
1003
+ node.module_name = module_name[len("cython.cimports."):]
1004
+ return node
1005
+ error(node.pos,
1006
+ "Python cimports must use 'from cython.cimports... import ...'"
1007
+ " or 'import ... as ...', not just 'import ...'")
1008
+
1009
+ if module_name == "cython":
1010
+ self.cython_module_names.add(node.as_name or "cython")
1011
+ elif module_name.startswith("cython."):
1012
+ if module_name.startswith("cython.parallel."):
1013
+ error(node.pos, node.module_name + " is not a module")
1014
+ else:
1015
+ self._check_valid_cython_module(node.pos, module_name)
1016
+
1017
+ if module_name == "cython.parallel":
1018
+ if node.as_name and node.as_name != "cython":
1019
+ self.parallel_directives[node.as_name] = module_name
1020
+ else:
1021
+ self.cython_module_names.add("cython")
1022
+ self.parallel_directives[
1023
+ "cython.parallel"] = module_name
1024
+ elif node.as_name:
1025
+ self.directive_names[node.as_name] = module_name[7:]
1026
+ else:
1027
+ self.cython_module_names.add("cython")
1028
+ # if this cimport was a compiler directive, we don't
1029
+ # want to leave the cimport node sitting in the tree
1030
+ return None
1031
+ return node
1032
+
1033
+ def visit_FromCImportStatNode(self, node):
1034
+ module_name = node.module_name
1035
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1036
+ # only supported for convenience
1037
+ return self._create_cimport_from_import(
1038
+ node.pos, module_name, node.relative_level, node.imported_names)
1039
+ elif not node.relative_level and (
1040
+ module_name == "cython" or module_name.startswith("cython.")):
1041
+ self._check_valid_cython_module(node.pos, module_name)
1042
+ submodule = (module_name + ".")[7:]
1043
+ newimp = []
1044
+ for pos, name, as_name in node.imported_names:
1045
+ full_name = submodule + name
1046
+ qualified_name = "cython." + full_name
1047
+ if self.is_parallel_directive(qualified_name, node.pos):
1048
+ # from cython cimport parallel, or
1049
+ # from cython.parallel cimport parallel, prange, ...
1050
+ self.parallel_directives[as_name or name] = qualified_name
1051
+ elif self.is_cython_directive(full_name):
1052
+ self.directive_names[as_name or name] = full_name
1053
+ elif full_name in ['dataclasses', 'typing']:
1054
+ self.directive_names[as_name or name] = full_name
1055
+ # unlike many directives, still treat it as a regular module
1056
+ newimp.append((pos, name, as_name))
1057
+ else:
1058
+ newimp.append((pos, name, as_name))
1059
+
1060
+ if not newimp:
1061
+ return None
1062
+
1063
+ node.imported_names = newimp
1064
+ return node
1065
+
1066
+ def visit_FromImportStatNode(self, node):
1067
+ import_node = node.module
1068
+ module_name = import_node.module_name.value
1069
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1070
+ imported_names = []
1071
+ for name, name_node in node.items:
1072
+ imported_names.append(
1073
+ (name_node.pos, name, None if name == name_node.name else name_node.name))
1074
+ return self._create_cimport_from_import(
1075
+ node.pos, module_name, import_node.level, imported_names)
1076
+ elif module_name == "cython" or module_name.startswith("cython."):
1077
+ self._check_valid_cython_module(import_node.module_name.pos, module_name)
1078
+ submodule = (module_name + ".")[7:]
1079
+ newimp = []
1080
+ for name, name_node in node.items:
1081
+ full_name = submodule + name
1082
+ qualified_name = "cython." + full_name
1083
+ if self.is_parallel_directive(qualified_name, node.pos):
1084
+ self.parallel_directives[name_node.name] = qualified_name
1085
+ elif self.is_cython_directive(full_name):
1086
+ self.directive_names[name_node.name] = full_name
1087
+ else:
1088
+ newimp.append((name, name_node))
1089
+ if not newimp:
1090
+ return None
1091
+ node.items = newimp
1092
+ return node
1093
+
1094
+ def _create_cimport_from_import(self, node_pos, module_name, level, imported_names):
1095
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1096
+ module_name = EncodedString(module_name[len("cython.cimports."):]) # may be empty
1097
+
1098
+ if module_name:
1099
+ # from cython.cimports.a.b import x, y, z => from a.b cimport x, y, z
1100
+ return Nodes.FromCImportStatNode(
1101
+ node_pos, module_name=module_name,
1102
+ relative_level=level,
1103
+ imported_names=imported_names)
1104
+ else:
1105
+ # from cython.cimports import x, y, z => cimport x; cimport y; cimport z
1106
+ return [
1107
+ Nodes.CImportStatNode(
1108
+ pos,
1109
+ module_name=dotted_name,
1110
+ as_name=as_name,
1111
+ is_absolute=level == 0)
1112
+ for pos, dotted_name, as_name in imported_names
1113
+ ]
1114
+
1115
+ def visit_SingleAssignmentNode(self, node):
1116
+ if isinstance(node.rhs, ExprNodes.ImportNode):
1117
+ module_name = node.rhs.module_name.value
1118
+ if module_name != "cython" and not module_name.startswith("cython."):
1119
+ return node
1120
+
1121
+ node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name)
1122
+ node = self.visit_CImportStatNode(node)
1123
+ else:
1124
+ self.visitchildren(node)
1125
+
1126
+ return node
1127
+
1128
+ def visit_NameNode(self, node):
1129
+ if node.annotation:
1130
+ self.visitchild(node, 'annotation')
1131
+ if node.name in self.cython_module_names:
1132
+ node.is_cython_module = True
1133
+ else:
1134
+ directive = self.directive_names.get(node.name)
1135
+ if directive is not None:
1136
+ node.cython_attribute = directive
1137
+ if node.as_cython_attribute() == "compiled":
1138
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1139
+ # before they have a chance to cause compile-errors
1140
+ return node
1141
+
1142
+ def visit_AttributeNode(self, node):
1143
+ self.visitchildren(node)
1144
+ if node.as_cython_attribute() == "compiled":
1145
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1146
+ # before they have a chance to cause compile-errors
1147
+ return node
1148
+
1149
+ def visit_AnnotationNode(self, node):
1150
+ # for most transforms annotations are left unvisited (because they're unevaluated)
1151
+ # however, it is important to pick up compiler directives from them
1152
+ if node.expr:
1153
+ self.visit(node.expr)
1154
+ return node
1155
+
1156
+ def visit_NewExprNode(self, node):
1157
+ self.visitchild(node, 'cppclass')
1158
+ self.visitchildren(node)
1159
+ return node
1160
+
1161
+ def try_to_parse_directives(self, node):
1162
+ # If node is the contents of an directive (in a with statement or
1163
+ # decorator), returns a list of (directivename, value) pairs.
1164
+ # Otherwise, returns None
1165
+ if isinstance(node, ExprNodes.CallNode):
1166
+ self.visitchild(node, 'function')
1167
+ optname = node.function.as_cython_attribute()
1168
+ if optname:
1169
+ directivetype = Options.directive_types.get(optname)
1170
+ if directivetype:
1171
+ args, kwds = node.explicit_args_kwds()
1172
+ directives = []
1173
+ key_value_pairs = []
1174
+ if kwds is not None and directivetype is not dict:
1175
+ for keyvalue in kwds.key_value_pairs:
1176
+ key, value = keyvalue
1177
+ sub_optname = "%s.%s" % (optname, key.value)
1178
+ if Options.directive_types.get(sub_optname):
1179
+ directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
1180
+ else:
1181
+ key_value_pairs.append(keyvalue)
1182
+ if not key_value_pairs:
1183
+ kwds = None
1184
+ else:
1185
+ kwds.key_value_pairs = key_value_pairs
1186
+ if directives and not kwds and not args:
1187
+ return directives
1188
+ directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
1189
+ return directives
1190
+ elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
1191
+ self.visit(node)
1192
+ optname = node.as_cython_attribute()
1193
+ if optname:
1194
+ directivetype = Options.directive_types.get(optname)
1195
+ if directivetype is bool:
1196
+ arg = ExprNodes.BoolNode(node.pos, value=True)
1197
+ return [self.try_to_parse_directive(optname, [arg], None, node.pos)]
1198
+ elif directivetype is None or directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1199
+ return [(optname, None)]
1200
+ else:
1201
+ raise PostParseError(
1202
+ node.pos, "The '%s' directive should be used as a function call." % optname)
1203
+ return None
1204
+
1205
+ def try_to_parse_directive(self, optname, args, kwds, pos):
1206
+ if optname == 'np_pythran' and not self.context.cpp:
1207
+ raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname)
1208
+ elif optname == 'exceptval':
1209
+ # default: exceptval(None, check=True)
1210
+ arg_error = len(args) > 1
1211
+ check = True
1212
+ if kwds and kwds.key_value_pairs:
1213
+ kw = kwds.key_value_pairs[0]
1214
+ if (len(kwds.key_value_pairs) == 1 and
1215
+ kw.key.is_string_literal and kw.key.value == 'check' and
1216
+ isinstance(kw.value, ExprNodes.BoolNode)):
1217
+ check = kw.value.value
1218
+ else:
1219
+ arg_error = True
1220
+ if arg_error:
1221
+ raise PostParseError(
1222
+ pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"')
1223
+ return ('exceptval', (args[0] if args else None, check))
1224
+
1225
+ directivetype = Options.directive_types.get(optname)
1226
+ if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
1227
+ return optname, Options.get_directive_defaults()[optname]
1228
+ elif directivetype is bool:
1229
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1230
+ raise PostParseError(pos,
1231
+ 'The %s directive takes one compile-time boolean argument' % optname)
1232
+ return (optname, args[0].value)
1233
+ elif directivetype is int:
1234
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
1235
+ raise PostParseError(pos,
1236
+ 'The %s directive takes one compile-time integer argument' % optname)
1237
+ return (optname, int(args[0].value))
1238
+ elif directivetype is str:
1239
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1240
+ raise PostParseError(pos,
1241
+ 'The %s directive takes one compile-time string argument' % optname)
1242
+ return (optname, str(args[0].value))
1243
+ elif directivetype is type:
1244
+ if kwds is not None or len(args) != 1:
1245
+ raise PostParseError(pos,
1246
+ 'The %s directive takes one type argument' % optname)
1247
+ return (optname, args[0])
1248
+ elif directivetype is dict:
1249
+ if len(args) != 0:
1250
+ raise PostParseError(pos,
1251
+ 'The %s directive takes no prepositional arguments' % optname)
1252
+ return optname, kwds.as_python_dict()
1253
+ elif directivetype is list:
1254
+ if kwds and len(kwds.key_value_pairs) != 0:
1255
+ raise PostParseError(pos,
1256
+ 'The %s directive takes no keyword arguments' % optname)
1257
+ return optname, [ str(arg.value) for arg in args ]
1258
+ elif callable(directivetype):
1259
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1260
+ raise PostParseError(pos,
1261
+ 'The %s directive takes one compile-time string argument' % optname)
1262
+ return (optname, directivetype(optname, str(args[0].value)))
1263
+ elif directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1264
+ # signal to pass things on without processing
1265
+ return (optname, (args, kwds.as_python_dict() if kwds else {}))
1266
+ else:
1267
+ assert False
1268
+
1269
+ def visit_with_directives(self, node, directives, contents_directives):
1270
+ # contents_directives may be None
1271
+ if not directives:
1272
+ assert not contents_directives
1273
+ return self.visit_Node(node)
1274
+
1275
+ old_directives = self.directives
1276
+ new_directives = Options.copy_inherited_directives(old_directives, **directives)
1277
+ if contents_directives is not None:
1278
+ new_contents_directives = Options.copy_inherited_directives(
1279
+ old_directives, **contents_directives)
1280
+ else:
1281
+ new_contents_directives = new_directives
1282
+
1283
+ if new_directives == old_directives:
1284
+ return self.visit_Node(node)
1285
+
1286
+ self.directives = new_directives
1287
+ if (contents_directives is not None and
1288
+ new_contents_directives != new_directives):
1289
+ # we need to wrap the node body in a compiler directives node
1290
+ node.body = Nodes.StatListNode(
1291
+ node.body.pos,
1292
+ stats=[
1293
+ Nodes.CompilerDirectivesNode(
1294
+ node.body.pos,
1295
+ directives=new_contents_directives,
1296
+ body=node.body)
1297
+ ]
1298
+ )
1299
+ retbody = self.visit_Node(node)
1300
+ self.directives = old_directives
1301
+
1302
+ if not isinstance(retbody, Nodes.StatListNode):
1303
+ retbody = Nodes.StatListNode(node.pos, stats=[retbody])
1304
+ return Nodes.CompilerDirectivesNode(
1305
+ pos=retbody.pos, body=retbody, directives=new_directives)
1306
+
1307
+
1308
+ # Handle decorators
1309
+ def visit_FuncDefNode(self, node):
1310
+ directives, contents_directives = self._extract_directives(node, 'function')
1311
+ return self.visit_with_directives(node, directives, contents_directives)
1312
+
1313
+ def visit_CVarDefNode(self, node):
1314
+ directives, _ = self._extract_directives(node, 'function')
1315
+ for name, value in directives.items():
1316
+ if name == 'locals':
1317
+ node.directive_locals = value
1318
+ elif name not in ('final', 'staticmethod'):
1319
+ self.context.nonfatal_error(PostParseError(
1320
+ node.pos,
1321
+ "Cdef functions can only take cython.locals(), "
1322
+ "staticmethod, or final decorators, got %s." % name))
1323
+ return self.visit_with_directives(node, directives, contents_directives=None)
1324
+
1325
+ def visit_CClassDefNode(self, node):
1326
+ directives, contents_directives = self._extract_directives(node, 'cclass')
1327
+ return self.visit_with_directives(node, directives, contents_directives)
1328
+
1329
+ def visit_CppClassNode(self, node):
1330
+ directives, contents_directives = self._extract_directives(node, 'cppclass')
1331
+ return self.visit_with_directives(node, directives, contents_directives)
1332
+
1333
+ def visit_PyClassDefNode(self, node):
1334
+ directives, contents_directives = self._extract_directives(node, 'class')
1335
+ return self.visit_with_directives(node, directives, contents_directives)
1336
+
1337
+ def _extract_directives(self, node, scope_name):
1338
+ """
1339
+ Returns two dicts - directives applied to this function/class
1340
+ and directives applied to its contents. They aren't always the
1341
+ same (since e.g. cfunc should not be applied to inner functions)
1342
+ """
1343
+ if not node.decorators:
1344
+ return {}, {}
1345
+ # Split the decorators into two lists -- real decorators and directives
1346
+ directives = []
1347
+ realdecs = []
1348
+ both = []
1349
+ current_opt_dict = dict(self.directives)
1350
+ missing = object()
1351
+ # Decorators coming first take precedence.
1352
+ for dec in node.decorators[::-1]:
1353
+ new_directives = self.try_to_parse_directives(dec.decorator)
1354
+ if new_directives is not None:
1355
+ for directive in new_directives:
1356
+ if self.check_directive_scope(node.pos, directive[0], scope_name):
1357
+ name, value = directive
1358
+ if name in ('nogil', 'with_gil'):
1359
+ if value is None:
1360
+ value = True
1361
+ else:
1362
+ args, kwds = value
1363
+ if kwds or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1364
+ raise PostParseError(dec.pos, 'The %s directive takes one compile-time boolean argument' % name)
1365
+ value = args[0].value
1366
+ directive = (name, value)
1367
+ if current_opt_dict.get(name, missing) != value:
1368
+ if name == 'cfunc' and 'ufunc' in current_opt_dict:
1369
+ error(dec.pos, "Cannot apply @cfunc to @ufunc, please reverse the decorators.")
1370
+ directives.append(directive)
1371
+ current_opt_dict[name] = value
1372
+ else:
1373
+ warning(dec.pos, "Directive does not change previous value (%s%s)" % (
1374
+ name, '=%r' % value if value is not None else ''))
1375
+ if directive[0] == 'staticmethod':
1376
+ both.append(dec)
1377
+ # Adapt scope type based on decorators that change it.
1378
+ if directive[0] == 'cclass' and scope_name == 'class':
1379
+ scope_name = 'cclass'
1380
+ else:
1381
+ realdecs.append(dec)
1382
+ node.decorators = realdecs[::-1] + both[::-1]
1383
+ # merge or override repeated directives
1384
+ optdict = {}
1385
+ contents_optdict = {}
1386
+ for name, value in directives:
1387
+ if name in optdict:
1388
+ old_value = optdict[name]
1389
+ # keywords and arg lists can be merged, everything
1390
+ # else overrides completely
1391
+ if isinstance(old_value, dict):
1392
+ old_value.update(value)
1393
+ elif isinstance(old_value, list):
1394
+ old_value.extend(value)
1395
+ else:
1396
+ optdict[name] = value
1397
+ else:
1398
+ optdict[name] = value
1399
+ if name not in Options.immediate_decorator_directives:
1400
+ contents_optdict[name] = value
1401
+ return optdict, contents_optdict
1402
+
1403
+ # Handle with-statements
1404
+ def visit_WithStatNode(self, node):
1405
+ directive_dict = {}
1406
+ for directive in self.try_to_parse_directives(node.manager) or []:
1407
+ if directive is not None:
1408
+ if node.target is not None:
1409
+ self.context.nonfatal_error(
1410
+ PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
1411
+ else:
1412
+ name, value = directive
1413
+ if name in ('nogil', 'gil'):
1414
+ # special case: in pure mode, "with nogil" spells "with cython.nogil"
1415
+ condition = None
1416
+ if isinstance(node.manager, ExprNodes.SimpleCallNode) and len(node.manager.args) > 0:
1417
+ if len(node.manager.args) == 1:
1418
+ condition = node.manager.args[0]
1419
+ else:
1420
+ self.context.nonfatal_error(
1421
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % name))
1422
+ elif isinstance(node.manager, ExprNodes.GeneralCallNode):
1423
+ self.context.nonfatal_error(
1424
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % name))
1425
+ node = Nodes.GILStatNode(node.pos, state=name, body=node.body, condition=condition)
1426
+ return self.visit_Node(node)
1427
+ if self.check_directive_scope(node.pos, name, 'with statement'):
1428
+ directive_dict[name] = value
1429
+ if directive_dict:
1430
+ return self.visit_with_directives(node.body, directive_dict, contents_directives=None)
1431
+ return self.visit_Node(node)
1432
+
1433
+
1434
+ class ParallelRangeTransform(CythonTransform, SkipDeclarations):
1435
+ """
1436
+ Transform cython.parallel stuff. The parallel_directives come from the
1437
+ module node, set there by InterpretCompilerDirectives.
1438
+
1439
+ x = cython.parallel.threadavailable() -> ParallelThreadAvailableNode
1440
+ with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
1441
+ print cython.parallel.threadid() -> ParallelThreadIdNode
1442
+ for i in cython.parallel.prange(...): -> ParallelRangeNode
1443
+ ...
1444
+ """
1445
+
1446
+ # a list of names, maps 'cython.parallel.prange' in the code to
1447
+ # ['cython', 'parallel', 'prange']
1448
+ parallel_directive = None
1449
+
1450
+ # Indicates whether a namenode in an expression is the cython module
1451
+ namenode_is_cython_module = False
1452
+
1453
+ # Keep track of whether we are the context manager of a 'with' statement
1454
+ in_context_manager_section = False
1455
+
1456
+ # One of 'prange' or 'with parallel'. This is used to disallow closely
1457
+ # nested 'with parallel:' blocks
1458
+ state = None
1459
+
1460
+ directive_to_node = {
1461
+ "cython.parallel.parallel": Nodes.ParallelWithBlockNode,
1462
+ # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
1463
+ "cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
1464
+ "cython.parallel.prange": Nodes.ParallelRangeNode,
1465
+ }
1466
+
1467
+ def node_is_parallel_directive(self, node):
1468
+ return node.name in self.parallel_directives or node.is_cython_module
1469
+
1470
+ def get_directive_class_node(self, node):
1471
+ """
1472
+ Figure out which parallel directive was used and return the associated
1473
+ Node class.
1474
+
1475
+ E.g. for a cython.parallel.prange() call we return ParallelRangeNode
1476
+ """
1477
+ if self.namenode_is_cython_module:
1478
+ directive = '.'.join(self.parallel_directive)
1479
+ else:
1480
+ directive = self.parallel_directives[self.parallel_directive[0]]
1481
+ directive = '%s.%s' % (directive,
1482
+ '.'.join(self.parallel_directive[1:]))
1483
+ directive = directive.rstrip('.')
1484
+
1485
+ cls = self.directive_to_node.get(directive)
1486
+ if cls is None and not (self.namenode_is_cython_module and
1487
+ self.parallel_directive[0] != 'parallel'):
1488
+ error(node.pos, "Invalid directive: %s" % directive)
1489
+
1490
+ self.namenode_is_cython_module = False
1491
+ self.parallel_directive = None
1492
+
1493
+ return cls
1494
+
1495
+ def visit_ModuleNode(self, node):
1496
+ """
1497
+ If any parallel directives were imported, copy them over and visit
1498
+ the AST
1499
+ """
1500
+ if node.parallel_directives:
1501
+ self.parallel_directives = node.parallel_directives
1502
+ return self.visit_Node(node)
1503
+
1504
+ # No parallel directives were imported, so they can't be used :)
1505
+ return node
1506
+
1507
+ def visit_NameNode(self, node):
1508
+ if self.node_is_parallel_directive(node):
1509
+ self.parallel_directive = [node.name]
1510
+ self.namenode_is_cython_module = node.is_cython_module
1511
+ return node
1512
+
1513
+ def visit_AttributeNode(self, node):
1514
+ self.visitchildren(node)
1515
+ if self.parallel_directive:
1516
+ self.parallel_directive.append(node.attribute)
1517
+ return node
1518
+
1519
+ def visit_CallNode(self, node):
1520
+ self.visitchild(node, 'function')
1521
+ if not self.parallel_directive:
1522
+ self.visitchildren(node, exclude=('function',))
1523
+ return node
1524
+
1525
+ # We are a parallel directive, replace this node with the
1526
+ # corresponding ParallelSomethingSomething node
1527
+
1528
+ if isinstance(node, ExprNodes.GeneralCallNode):
1529
+ args = node.positional_args.args
1530
+ kwargs = node.keyword_args
1531
+ else:
1532
+ args = node.args
1533
+ kwargs = {}
1534
+
1535
+ parallel_directive_class = self.get_directive_class_node(node)
1536
+ if parallel_directive_class:
1537
+ # Note: in case of a parallel() the body is set by
1538
+ # visit_WithStatNode
1539
+ node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)
1540
+
1541
+ return node
1542
+
1543
+ def visit_WithStatNode(self, node):
1544
+ "Rewrite with cython.parallel.parallel() blocks"
1545
+ newnode = self.visit(node.manager)
1546
+
1547
+ if isinstance(newnode, Nodes.ParallelWithBlockNode):
1548
+ if self.state == 'parallel with':
1549
+ error(node.manager.pos,
1550
+ "Nested parallel with blocks are disallowed")
1551
+
1552
+ self.state = 'parallel with'
1553
+ body = self.visitchild(node, 'body')
1554
+ self.state = None
1555
+
1556
+ newnode.body = body
1557
+ return newnode
1558
+ elif self.parallel_directive:
1559
+ parallel_directive_class = self.get_directive_class_node(node)
1560
+
1561
+ if not parallel_directive_class:
1562
+ # There was an error, stop here and now
1563
+ return None
1564
+
1565
+ if parallel_directive_class is Nodes.ParallelWithBlockNode:
1566
+ error(node.pos, "The parallel directive must be called")
1567
+ return None
1568
+
1569
+ self.visitchild(node, 'body')
1570
+ return node
1571
+
1572
+ def visit_ForInStatNode(self, node):
1573
+ "Rewrite 'for i in cython.parallel.prange(...):'"
1574
+ self.visitchild(node, 'iterator')
1575
+ self.visitchild(node, 'target')
1576
+
1577
+ in_prange = isinstance(node.iterator.sequence,
1578
+ Nodes.ParallelRangeNode)
1579
+ previous_state = self.state
1580
+
1581
+ if in_prange:
1582
+ # This will replace the entire ForInStatNode, so copy the
1583
+ # attributes
1584
+ parallel_range_node = node.iterator.sequence
1585
+
1586
+ parallel_range_node.target = node.target
1587
+ parallel_range_node.body = node.body
1588
+ parallel_range_node.else_clause = node.else_clause
1589
+
1590
+ node = parallel_range_node
1591
+
1592
+ if not isinstance(node.target, ExprNodes.NameNode):
1593
+ error(node.target.pos,
1594
+ "Can only iterate over an iteration variable")
1595
+
1596
+ self.state = 'prange'
1597
+
1598
+ self.visitchild(node, 'body')
1599
+ self.state = previous_state
1600
+ self.visitchild(node, 'else_clause')
1601
+ return node
1602
+
1603
+ def visit(self, node):
1604
+ "Visit a node that may be None"
1605
+ if node is not None:
1606
+ return super().visit(node)
1607
+
1608
+
1609
+ class WithTransform(VisitorTransform, SkipDeclarations):
1610
+ def visit_WithStatNode(self, node):
1611
+ self.visitchildren(node, 'body')
1612
+ pos = node.pos
1613
+ is_async = node.is_async
1614
+ body, target, manager = node.body, node.target, node.manager
1615
+ manager = node.manager = ExprNodes.ProxyNode(manager)
1616
+ node.enter_call = ExprNodes.SimpleCallNode(
1617
+ pos, function=ExprNodes.AttributeNode(
1618
+ pos, obj=ExprNodes.CloneNode(manager),
1619
+ attribute=EncodedString('__aenter__' if is_async else '__enter__'),
1620
+ is_special_lookup=True),
1621
+ args=[],
1622
+ is_temp=True)
1623
+
1624
+ if is_async:
1625
+ node.enter_call = ExprNodes.AwaitExprNode(pos, arg=node.enter_call)
1626
+
1627
+ if target is not None:
1628
+ body = Nodes.StatListNode(
1629
+ pos, stats=[
1630
+ Nodes.WithTargetAssignmentStatNode(
1631
+ pos, lhs=target, with_node=node),
1632
+ body])
1633
+
1634
+ excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
1635
+ ExprNodes.ExcValueNode(pos) for _ in range(3)])
1636
+ except_clause = Nodes.ExceptClauseNode(
1637
+ pos, body=Nodes.IfStatNode(
1638
+ pos, if_clauses=[
1639
+ Nodes.IfClauseNode(
1640
+ pos, condition=ExprNodes.NotNode(
1641
+ pos, operand=ExprNodes.WithExitCallNode(
1642
+ pos, with_stat=node,
1643
+ test_if_run=False,
1644
+ args=excinfo_target,
1645
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1646
+ body=Nodes.ReraiseStatNode(pos),
1647
+ ),
1648
+ ],
1649
+ else_clause=None),
1650
+ pattern=None,
1651
+ target=None,
1652
+ excinfo_target=excinfo_target,
1653
+ )
1654
+
1655
+ node.body = Nodes.TryFinallyStatNode(
1656
+ pos, body=Nodes.TryExceptStatNode(
1657
+ pos, body=body,
1658
+ except_clauses=[except_clause],
1659
+ else_clause=None,
1660
+ ),
1661
+ finally_clause=Nodes.ExprStatNode(
1662
+ pos, expr=ExprNodes.WithExitCallNode(
1663
+ pos, with_stat=node,
1664
+ test_if_run=True,
1665
+ args=ExprNodes.TupleNode(
1666
+ pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]),
1667
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1668
+ handle_error_case=False,
1669
+ )
1670
+ return node
1671
+
1672
+ def visit_ExprNode(self, node):
1673
+ # With statements are never inside expressions.
1674
+ return node
1675
+
1676
+ visit_Node = VisitorTransform.recurse_to_children
1677
+
1678
+
1679
+ class _GeneratorExpressionArgumentsMarker(TreeVisitor, SkipDeclarations):
1680
+ # called from "MarkClosureVisitor"
1681
+ def __init__(self, gen_expr):
1682
+ super().__init__()
1683
+ self.gen_expr = gen_expr
1684
+
1685
+ def visit_ExprNode(self, node):
1686
+ if not node.is_literal:
1687
+ # Don't bother tagging literal nodes
1688
+ assert (not node.generator_arg_tag) # nobody has tagged this first
1689
+ node.generator_arg_tag = self.gen_expr
1690
+ self.visitchildren(node)
1691
+
1692
+ def visit_Node(self, node):
1693
+ # We're only interested in the expressions that make up the iterator sequence,
1694
+ # so don't go beyond ExprNodes (e.g. into ForFromStatNode).
1695
+ return
1696
+
1697
+ def visit_GeneratorExpressionNode(self, node):
1698
+ node.generator_arg_tag = self.gen_expr
1699
+ # don't visit children, can't handle overlapping tags
1700
+ # (and assume generator expressions don't end up optimized out in a way
1701
+ # that would require overlapping tags)
1702
+
1703
+
1704
+ class _HandleGeneratorArguments(VisitorTransform, SkipDeclarations):
1705
+ # used from within CreateClosureClasses
1706
+
1707
+ def __call__(self, node):
1708
+ from . import Visitor
1709
+ assert isinstance(node, ExprNodes.GeneratorExpressionNode)
1710
+ self.gen_node = node
1711
+
1712
+ self.args = list(node.def_node.args)
1713
+ self.call_parameters = list(node.call_parameters)
1714
+ self.tag_count = 0
1715
+ self.substitutions = {}
1716
+
1717
+ self.visitchildren(node)
1718
+
1719
+ for k, v in self.substitutions.items():
1720
+ # doing another search for replacements here (at the end) allows us to sweep up
1721
+ # CloneNodes too (which are often generated by the optimizer)
1722
+ # (it could arguably be done more efficiently with a single traversal though)
1723
+ Visitor.recursively_replace_node(node, k, v)
1724
+
1725
+ node.def_node.args = self.args
1726
+ node.call_parameters = self.call_parameters
1727
+ return node
1728
+
1729
+ def visit_GeneratorExpressionNode(self, node):
1730
+ # a generator can also be substituted itself, so handle that case
1731
+ new_node = self._handle_ExprNode(node, do_visit_children=False)
1732
+ # However do not traverse into it. A new _HandleGeneratorArguments visitor will be used
1733
+ # elsewhere to do that.
1734
+ return node
1735
+
1736
+ def _handle_ExprNode(self, node, do_visit_children):
1737
+ if (node.generator_arg_tag is not None and self.gen_node is not None and
1738
+ self.gen_node == node.generator_arg_tag):
1739
+ pos = node.pos
1740
+ # The reason for using ".x" as the name is that this is how CPython
1741
+ # tracks internal variables in loops (e.g.
1742
+ # { locals() for v in range(10) }
1743
+ # will produce "v" and ".0"). We don't replicate this behaviour completely
1744
+ # but use it as a starting point
1745
+ name_source = self.tag_count
1746
+ self.tag_count += 1
1747
+ name = EncodedString(".{}".format(name_source))
1748
+ def_node = self.gen_node.def_node
1749
+ if not def_node.local_scope.lookup_here(name):
1750
+ from . import Symtab
1751
+ cname = EncodedString(Naming.genexpr_arg_prefix + Symtab.punycodify_name(str(name_source)))
1752
+ name_decl = Nodes.CNameDeclaratorNode(pos=pos, name=name)
1753
+ type = node.type
1754
+
1755
+ # strip away cv types - they shouldn't be applied to the
1756
+ # function argument or to the closure struct.
1757
+ # It isn't obvious whether the right thing to do would be to capture by reference or by
1758
+ # value (C++ itself doesn't know either for lambda functions and forces a choice).
1759
+ # However, capture by reference involves converting to FakeReference which would require
1760
+ # re-analysing AttributeNodes. Therefore I've picked capture-by-value out of convenience
1761
+ # TODO - could probably be optimized by making the arg a reference but the closure not
1762
+ # (see https://github.com/cython/cython/issues/2468)
1763
+ type = PyrexTypes.remove_cv_ref(type, remove_fakeref=False)
1764
+
1765
+ name_decl.type = type
1766
+ new_arg = Nodes.CArgDeclNode(pos=pos, declarator=name_decl,
1767
+ base_type=None, default=None, annotation=None)
1768
+ new_arg.name = name_decl.name
1769
+ new_arg.type = type
1770
+
1771
+ self.args.append(new_arg)
1772
+ node.generator_arg_tag = None # avoid the possibility of this being caught again
1773
+ self.call_parameters.append(node)
1774
+ new_arg.entry = def_node.declare_argument(def_node.local_scope, new_arg)
1775
+ new_arg.entry.cname = cname
1776
+ new_arg.entry.in_closure = True
1777
+
1778
+ if do_visit_children:
1779
+ # now visit the Nodes's children (but remove self.gen_node to not to further
1780
+ # argument substitution)
1781
+ gen_node, self.gen_node = self.gen_node, None
1782
+ self.visitchildren(node)
1783
+ self.gen_node = gen_node
1784
+
1785
+ # replace the node inside the generator with a looked-up name
1786
+ # (initialized_check can safely be False because the source variable will be checked
1787
+ # before it is captured if the check is required)
1788
+ name_node = ExprNodes.NameNode(pos, name=name, initialized_check=False)
1789
+ name_node.entry = self.gen_node.def_node.gbody.local_scope.lookup(name_node.name)
1790
+ name_node.type = name_node.entry.type
1791
+ self.substitutions[node] = name_node
1792
+ return name_node
1793
+ if do_visit_children:
1794
+ self.visitchildren(node)
1795
+ return node
1796
+
1797
+ def visit_ExprNode(self, node):
1798
+ return self._handle_ExprNode(node, True)
1799
+
1800
+ visit_Node = VisitorTransform.recurse_to_children
1801
+
1802
+
1803
+ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
1804
+ """
1805
+ Transforms method decorators in cdef classes into nested calls or properties.
1806
+
1807
+ Python-style decorator properties are transformed into a PropertyNode
1808
+ with up to the three getter, setter and deleter DefNodes.
1809
+ The functional style isn't supported yet.
1810
+ """
1811
+ _properties = None
1812
+
1813
+ _map_property_attribute = {
1814
+ 'getter': EncodedString('__get__'),
1815
+ 'setter': EncodedString('__set__'),
1816
+ 'deleter': EncodedString('__del__'),
1817
+ }.get
1818
+
1819
+ def visit_CClassDefNode(self, node):
1820
+ if self._properties is None:
1821
+ self._properties = []
1822
+ self._properties.append({})
1823
+ node = super().visit_CClassDefNode(node)
1824
+ self._properties.pop()
1825
+ return node
1826
+
1827
+ def visit_PropertyNode(self, node):
1828
+ # Low-level warning for other code until we can convert all our uses over.
1829
+ level = 2 if isinstance(node.pos[0], str) else 0
1830
+ warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level)
1831
+ return node
1832
+
1833
+ def visit_CFuncDefNode(self, node):
1834
+ node = self.visit_FuncDefNode(node)
1835
+ if not node.decorators:
1836
+ return node
1837
+ elif self.scope_type != 'cclass' or self.scope_node.visibility != "extern":
1838
+ # at the moment cdef functions are very restricted in what decorators they can take
1839
+ # so it's simple to test for the small number of allowed decorators....
1840
+ if not (len(node.decorators) == 1 and node.decorators[0].decorator.is_name and
1841
+ node.decorators[0].decorator.name == "staticmethod"):
1842
+ error(node.decorators[0].pos, "Cdef functions cannot take arbitrary decorators.")
1843
+ return node
1844
+
1845
+ ret_node = node
1846
+ decorator_node = self._find_property_decorator(node)
1847
+ if decorator_node:
1848
+ if decorator_node.decorator.is_name:
1849
+ name = node.declared_name()
1850
+ if name:
1851
+ ret_node = self._add_property(node, name, decorator_node)
1852
+ else:
1853
+ error(decorator_node.pos, "C property decorator can only be @property")
1854
+
1855
+ if node.decorators:
1856
+ return self._reject_decorated_property(node, node.decorators[0])
1857
+ return ret_node
1858
+
1859
+ def visit_DefNode(self, node):
1860
+ scope_type = self.scope_type
1861
+ node = self.visit_FuncDefNode(node)
1862
+ if scope_type != 'cclass' or not node.decorators:
1863
+ return node
1864
+
1865
+ # transform @property decorators
1866
+ decorator_node = self._find_property_decorator(node)
1867
+ if decorator_node is not None:
1868
+ decorator = decorator_node.decorator
1869
+ if decorator.is_name:
1870
+ return self._add_property(node, node.name, decorator_node)
1871
+ else:
1872
+ handler_name = self._map_property_attribute(decorator.attribute)
1873
+ if handler_name:
1874
+ if decorator.obj.name != node.name:
1875
+ # CPython does not generate an error or warning, but not something useful either.
1876
+ error(decorator_node.pos,
1877
+ "Mismatching property names, expected '%s', got '%s'" % (
1878
+ decorator.obj.name, node.name))
1879
+ elif len(node.decorators) > 1:
1880
+ return self._reject_decorated_property(node, decorator_node)
1881
+ else:
1882
+ return self._add_to_property(node, handler_name, decorator_node)
1883
+
1884
+ # we clear node.decorators, so we need to set the
1885
+ # is_staticmethod/is_classmethod attributes now
1886
+ for decorator in node.decorators:
1887
+ func = decorator.decorator
1888
+ if func.is_name:
1889
+ node.is_classmethod |= func.name == 'classmethod'
1890
+ node.is_staticmethod |= func.name == 'staticmethod'
1891
+
1892
+ # transform normal decorators
1893
+ decs = node.decorators
1894
+ node.decorators = None
1895
+ return self.chain_decorators(node, decs, node.name)
1896
+
1897
+ def _find_property_decorator(self, node):
1898
+ properties = self._properties[-1]
1899
+ for decorator_node in node.decorators[::-1]:
1900
+ decorator = decorator_node.decorator
1901
+ if decorator.is_name and decorator.name == 'property':
1902
+ # @property
1903
+ return decorator_node
1904
+ elif decorator.is_attribute and decorator.obj.name in properties:
1905
+ # @prop.setter etc.
1906
+ return decorator_node
1907
+ return None
1908
+
1909
+ @staticmethod
1910
+ def _reject_decorated_property(node, decorator_node):
1911
+ # restrict transformation to outermost decorator as wrapped properties will probably not work
1912
+ for deco in node.decorators:
1913
+ if deco != decorator_node:
1914
+ error(deco.pos, "Property methods with additional decorators are not supported")
1915
+ return node
1916
+
1917
+ def _add_property(self, node, name, decorator_node):
1918
+ if len(node.decorators) > 1:
1919
+ return self._reject_decorated_property(node, decorator_node)
1920
+ node.decorators.remove(decorator_node)
1921
+ properties = self._properties[-1]
1922
+ is_cproperty = isinstance(node, Nodes.CFuncDefNode)
1923
+ body = Nodes.StatListNode(node.pos, stats=[node])
1924
+ if is_cproperty:
1925
+ if name in properties:
1926
+ error(node.pos, "C property redeclared")
1927
+ if 'inline' not in node.modifiers:
1928
+ error(node.pos, "C property method must be declared 'inline'")
1929
+ prop = Nodes.CPropertyNode(node.pos, doc=node.doc, name=name, body=body)
1930
+ elif name in properties:
1931
+ prop = properties[name]
1932
+ if prop.is_cproperty:
1933
+ error(node.pos, "C property redeclared")
1934
+ else:
1935
+ node.name = EncodedString("__get__")
1936
+ prop.pos = node.pos
1937
+ prop.doc = node.doc
1938
+ prop.body.stats = [node]
1939
+ return None
1940
+ else:
1941
+ node.name = EncodedString("__get__")
1942
+ prop = Nodes.PropertyNode(
1943
+ node.pos, name=name, doc=node.doc, body=body)
1944
+ properties[name] = prop
1945
+ return prop
1946
+
1947
+ def _add_to_property(self, node, name, decorator):
1948
+ properties = self._properties[-1]
1949
+ prop = properties[node.name]
1950
+ if prop.is_cproperty:
1951
+ error(node.pos, "C property redeclared")
1952
+ return None
1953
+ node.name = name
1954
+ node.decorators.remove(decorator)
1955
+ stats = prop.body.stats
1956
+ for i, stat in enumerate(stats):
1957
+ if stat.name == name:
1958
+ stats[i] = node
1959
+ break
1960
+ else:
1961
+ stats.append(node)
1962
+ return None
1963
+
1964
+ @staticmethod
1965
+ def chain_decorators(node, decorators, name):
1966
+ """
1967
+ Decorators are applied directly in DefNode and PyClassDefNode to avoid
1968
+ reassignments to the function/class name - except for cdef class methods.
1969
+ For those, the reassignment is required as methods are originally
1970
+ defined in the PyMethodDef struct.
1971
+
1972
+ The IndirectionNode allows DefNode to override the decorator.
1973
+ """
1974
+ decorator_result = ExprNodes.NameNode(node.pos, name=name)
1975
+ for decorator in decorators[::-1]:
1976
+ decorator_result = ExprNodes.SimpleCallNode(
1977
+ decorator.pos,
1978
+ function=decorator.decorator,
1979
+ args=[decorator_result])
1980
+
1981
+ name_node = ExprNodes.NameNode(node.pos, name=name)
1982
+ reassignment = Nodes.SingleAssignmentNode(
1983
+ node.pos,
1984
+ lhs=name_node,
1985
+ rhs=decorator_result)
1986
+
1987
+ reassignment = Nodes.IndirectionNode([reassignment])
1988
+ node.decorator_indirection = reassignment
1989
+ return [node, reassignment]
1990
+
1991
+
1992
+ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
1993
+ """
1994
+ Only part of the CythonUtilityCode pipeline. Must be run before
1995
+ DecoratorTransform in case this is a decorator for a cdef class.
1996
+ It filters out @cname('my_cname') decorators and rewrites them to
1997
+ CnameDecoratorNodes.
1998
+ """
1999
+
2000
+ def handle_function(self, node):
2001
+ if not getattr(node, 'decorators', None):
2002
+ return self.visit_Node(node)
2003
+
2004
+ for i, decorator in enumerate(node.decorators):
2005
+ decorator = decorator.decorator
2006
+
2007
+ if (isinstance(decorator, ExprNodes.CallNode) and
2008
+ decorator.function.is_name and
2009
+ decorator.function.name == 'cname'):
2010
+ args, kwargs = decorator.explicit_args_kwds()
2011
+
2012
+ if kwargs:
2013
+ raise AssertionError(
2014
+ "cname decorator does not take keyword arguments")
2015
+
2016
+ if len(args) != 1:
2017
+ raise AssertionError(
2018
+ "cname decorator takes exactly one argument")
2019
+
2020
+ if not (args[0].is_literal and args[0].type is Builtin.unicode_type):
2021
+ raise AssertionError(
2022
+ "argument to cname decorator must be a string literal")
2023
+
2024
+ cname = args[0].compile_time_value(None)
2025
+ del node.decorators[i]
2026
+ node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
2027
+ cname=cname)
2028
+ break
2029
+
2030
+ return self.visit_Node(node)
2031
+
2032
+ visit_FuncDefNode = handle_function
2033
+ visit_CClassDefNode = handle_function
2034
+ visit_CEnumDefNode = handle_function
2035
+ visit_CStructOrUnionDefNode = handle_function
2036
+
2037
+
2038
+ class ForwardDeclareTypes(CythonTransform):
2039
+ """
2040
+ Declare all global cdef names that we allow referencing in other places,
2041
+ before declaring everything (else) in source code order.
2042
+ """
2043
+
2044
+ def visit_CompilerDirectivesNode(self, node):
2045
+ env = self.module_scope
2046
+ old = env.directives
2047
+ env.directives = node.directives
2048
+ self.visitchildren(node)
2049
+ env.directives = old
2050
+ return node
2051
+
2052
+ def visit_ModuleNode(self, node):
2053
+ self.module_scope = node.scope
2054
+ self.module_scope.directives = node.directives
2055
+ self.visitchildren(node)
2056
+ return node
2057
+
2058
+ def visit_CDefExternNode(self, node):
2059
+ old_cinclude_flag = self.module_scope.in_cinclude
2060
+ self.module_scope.in_cinclude = 1
2061
+ self.visitchildren(node)
2062
+ self.module_scope.in_cinclude = old_cinclude_flag
2063
+ return node
2064
+
2065
+ def visit_CEnumDefNode(self, node):
2066
+ node.declare(self.module_scope)
2067
+ return node
2068
+
2069
+ def visit_CStructOrUnionDefNode(self, node):
2070
+ if node.name not in self.module_scope.entries:
2071
+ node.declare(self.module_scope)
2072
+ return node
2073
+
2074
+ def visit_CClassDefNode(self, node):
2075
+ if node.class_name not in self.module_scope.entries:
2076
+ node.declare(self.module_scope)
2077
+ # Expand fused methods of .pxd declared types to construct the final vtable order.
2078
+ type = self.module_scope.entries[node.class_name].type
2079
+ if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope:
2080
+ scope = type.scope
2081
+ for entry in scope.cfunc_entries:
2082
+ if entry.type and entry.type.is_fused:
2083
+ entry.type.get_all_specialized_function_types()
2084
+ return node
2085
+
2086
+ def visit_FuncDefNode(self, node):
2087
+ # no traversal needed
2088
+ return node
2089
+
2090
+ def visit_PyClassDefNode(self, node):
2091
+ # no traversal needed
2092
+ return node
2093
+
2094
+
2095
+ class AnalyseDeclarationsTransform(EnvTransform):
2096
+
2097
+ basic_property = TreeFragment("""
2098
+ property NAME:
2099
+ def __get__(self):
2100
+ return ATTR
2101
+ def __set__(self, value):
2102
+ ATTR = value
2103
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2104
+ basic_pyobject_property = TreeFragment("""
2105
+ property NAME:
2106
+ def __get__(self):
2107
+ return ATTR
2108
+ def __set__(self, value):
2109
+ ATTR = value
2110
+ def __del__(self):
2111
+ ATTR = None
2112
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2113
+ basic_property_ro = TreeFragment("""
2114
+ property NAME:
2115
+ def __get__(self):
2116
+ return ATTR
2117
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2118
+
2119
+ struct_or_union_wrapper = TreeFragment("""
2120
+ cdef class NAME:
2121
+ cdef TYPE value
2122
+ def __init__(self, MEMBER=None):
2123
+ cdef int count
2124
+ count = 0
2125
+ INIT_ASSIGNMENTS
2126
+ if IS_UNION and count > 1:
2127
+ raise ValueError, "At most one union member should be specified."
2128
+ def __str__(self):
2129
+ return STR_FORMAT % MEMBER_TUPLE
2130
+ def __repr__(self):
2131
+ return REPR_FORMAT % MEMBER_TUPLE
2132
+ """, pipeline=[NormalizeTree(None)])
2133
+
2134
+ init_assignment = TreeFragment("""
2135
+ if VALUE is not None:
2136
+ ATTR = VALUE
2137
+ count += 1
2138
+ """, pipeline=[NormalizeTree(None)])
2139
+
2140
+ fused_function = None
2141
+ in_lambda = 0
2142
+
2143
+ def __call__(self, root):
2144
+ # needed to determine if a cdef var is declared after it's used.
2145
+ self.seen_vars_stack = []
2146
+ self.fused_error_funcs = set()
2147
+ super_class = super()
2148
+ self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
2149
+ return super_class.__call__(root)
2150
+
2151
+ def visit_NameNode(self, node):
2152
+ self.seen_vars_stack[-1].add(node.name)
2153
+ return node
2154
+
2155
+ def visit_ModuleNode(self, node):
2156
+ # Pickling support requires injecting module-level nodes.
2157
+ self.extra_module_declarations = []
2158
+ self.seen_vars_stack.append(set())
2159
+ node.analyse_declarations(self.current_env())
2160
+ self.visitchildren(node)
2161
+ self.seen_vars_stack.pop()
2162
+ node.body.stats.extend(self.extra_module_declarations)
2163
+ return node
2164
+
2165
+ def visit_LambdaNode(self, node):
2166
+ self.in_lambda += 1
2167
+ node.analyse_declarations(self.current_env())
2168
+ self.visitchildren(node)
2169
+ self.in_lambda -= 1
2170
+ return node
2171
+
2172
+ def visit_CClassDefNode(self, node):
2173
+ node = self.visit_ClassDefNode(node)
2174
+ if node.scope and 'dataclasses.dataclass' in node.scope.directives:
2175
+ from .Dataclass import handle_cclass_dataclass
2176
+ handle_cclass_dataclass(node, node.scope.directives['dataclasses.dataclass'], self)
2177
+ if node.scope and node.scope.implemented and node.body:
2178
+ stats = []
2179
+ for entry in node.scope.var_entries:
2180
+ if entry.needs_property:
2181
+ property = self.create_Property(entry)
2182
+ property.analyse_declarations(node.scope)
2183
+ self.visit(property)
2184
+ stats.append(property)
2185
+ if stats:
2186
+ node.body.stats += stats
2187
+ if (node.visibility != 'extern'
2188
+ and not node.scope.lookup('__reduce__')
2189
+ and not node.scope.lookup('__reduce_ex__')):
2190
+ self._inject_pickle_methods(node)
2191
+ return node
2192
+
2193
+ def _inject_pickle_methods(self, node):
2194
+ env = self.current_env()
2195
+ if node.scope.directives['auto_pickle'] is False: # None means attempt it.
2196
+ # Old behavior of not doing anything.
2197
+ return
2198
+ auto_pickle_forced = node.scope.directives['auto_pickle'] is True
2199
+
2200
+ all_members = []
2201
+ cls = node.entry.type
2202
+ cinit = None
2203
+ inherited_reduce = None
2204
+ while cls is not None:
2205
+ all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__'))
2206
+ cinit = cinit or cls.scope.lookup('__cinit__')
2207
+ inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__')
2208
+ cls = cls.base_type
2209
+ all_members.sort(key=lambda e: e.name)
2210
+
2211
+ if inherited_reduce:
2212
+ # This is not failsafe, as we may not know whether a cimported class defines a __reduce__.
2213
+ # This is why we define __reduce_cython__ and only replace __reduce__
2214
+ # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation.
2215
+ return
2216
+
2217
+ non_py = [
2218
+ e for e in all_members
2219
+ if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env)
2220
+ or not e.type.can_coerce_from_pyobject(env))
2221
+ ]
2222
+
2223
+ structs = [e for e in all_members if e.type.is_struct_or_union]
2224
+
2225
+ if cinit or non_py or (structs and not auto_pickle_forced):
2226
+ if cinit:
2227
+ # TODO(robertwb): We could allow this if __cinit__ has no require arguments.
2228
+ msg = 'no default __reduce__ due to non-trivial __cinit__'
2229
+ elif non_py:
2230
+ msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py)
2231
+ else:
2232
+ # Extern structs may be only partially defined.
2233
+ # TODO(robertwb): Limit the restriction to extern
2234
+ # (and recursively extern-containing) structs.
2235
+ msg = ("Pickling of struct members such as %s must be explicitly requested "
2236
+ "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs))
2237
+
2238
+ if auto_pickle_forced:
2239
+ error(node.pos, msg)
2240
+
2241
+ pickle_func = TreeFragment("""
2242
+ def __reduce_cython__(self):
2243
+ raise TypeError, "%(msg)s"
2244
+ def __setstate_cython__(self, __pyx_state):
2245
+ raise TypeError, "%(msg)s"
2246
+ """ % {'msg': msg},
2247
+ level='c_class', pipeline=[NormalizeTree(None)]).substitute({})
2248
+ pickle_func.analyse_declarations(node.scope)
2249
+ self.visit(pickle_func)
2250
+ node.body.stats.append(pickle_func)
2251
+
2252
+ else:
2253
+ for e in all_members:
2254
+ if not e.type.is_pyobject:
2255
+ e.type.create_to_py_utility_code(env)
2256
+ e.type.create_from_py_utility_code(env)
2257
+
2258
+ all_members_names = [e.name for e in all_members]
2259
+ checksums = _calculate_pickle_checksums(all_members_names)
2260
+
2261
+ unpickle_func_name = '__pyx_unpickle_%s' % node.punycode_class_name
2262
+
2263
+ # TODO(robertwb): Move the state into the third argument
2264
+ # so it can be pickled *after* self is memoized.
2265
+ unpickle_func = TreeFragment("""
2266
+ def %(unpickle_func_name)s(__pyx_type, long __pyx_checksum, __pyx_state):
2267
+ cdef object __pyx_PickleError
2268
+ cdef object __pyx_result
2269
+ if __pyx_checksum not in %(checksums)s:
2270
+ from pickle import PickleError as __pyx_PickleError
2271
+ raise __pyx_PickleError, "Incompatible checksums (0x%%x vs %(checksums)s = (%(members)s))" %% __pyx_checksum
2272
+ __pyx_result = %(class_name)s.__new__(__pyx_type)
2273
+ if __pyx_state is not None:
2274
+ %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state)
2275
+ return __pyx_result
2276
+
2277
+ cdef %(unpickle_func_name)s__set_state(%(class_name)s __pyx_result, tuple __pyx_state):
2278
+ %(assignments)s
2279
+ if len(__pyx_state) > %(num_members)d and hasattr(__pyx_result, '__dict__'):
2280
+ __pyx_result.__dict__.update(__pyx_state[%(num_members)d])
2281
+ """ % {
2282
+ 'unpickle_func_name': unpickle_func_name,
2283
+ 'checksums': "(%s)" % ', '.join(checksums),
2284
+ 'members': ', '.join(all_members_names),
2285
+ 'class_name': node.class_name,
2286
+ 'assignments': '; '.join(
2287
+ '__pyx_result.%s = __pyx_state[%s]' % (v, ix)
2288
+ for ix, v in enumerate(all_members_names)),
2289
+ 'num_members': len(all_members_names),
2290
+ }, level='module', pipeline=[NormalizeTree(None)]).substitute({})
2291
+ unpickle_func.analyse_declarations(node.entry.scope)
2292
+ self.visit(unpickle_func)
2293
+ self.extra_module_declarations.append(unpickle_func)
2294
+
2295
+ pickle_func = TreeFragment("""
2296
+ def __reduce_cython__(self):
2297
+ cdef tuple state
2298
+ cdef object _dict
2299
+ cdef bint use_setstate
2300
+ state = (%(members)s)
2301
+ _dict = getattr(self, '__dict__', None)
2302
+ if _dict is not None:
2303
+ state += (_dict,)
2304
+ use_setstate = True
2305
+ else:
2306
+ use_setstate = %(any_notnone_members)s
2307
+ if use_setstate:
2308
+ return %(unpickle_func_name)s, (type(self), %(checksum)s, None), state
2309
+ else:
2310
+ return %(unpickle_func_name)s, (type(self), %(checksum)s, state)
2311
+
2312
+ def __setstate_cython__(self, __pyx_state):
2313
+ %(unpickle_func_name)s__set_state(self, __pyx_state)
2314
+ """ % {
2315
+ 'unpickle_func_name': unpickle_func_name,
2316
+ 'checksum': checksums[0],
2317
+ 'members': ', '.join('self.%s' % v for v in all_members_names) + (',' if len(all_members_names) == 1 else ''),
2318
+ # Even better, we could check PyType_IS_GC.
2319
+ 'any_notnone_members' : ' or '.join(['self.%s is not None' % e.name for e in all_members if e.type.is_pyobject] or ['False']),
2320
+ },
2321
+ level='c_class', pipeline=[NormalizeTree(None)]).substitute({})
2322
+ pickle_func.analyse_declarations(node.scope)
2323
+ self.enter_scope(node, node.scope) # functions should be visited in the class scope
2324
+ self.visit(pickle_func)
2325
+ self.exit_scope()
2326
+ node.body.stats.append(pickle_func)
2327
+
2328
+ def _handle_fused_def_decorators(self, old_decorators, env, node):
2329
+ """
2330
+ Create function calls to the decorators and reassignments to
2331
+ the function.
2332
+ """
2333
+ # Delete staticmethod and classmethod decorators, this is
2334
+ # handled directly by the fused function object.
2335
+ decorators = []
2336
+ for decorator in old_decorators:
2337
+ func = decorator.decorator
2338
+ if (not func.is_name or
2339
+ func.name not in ('staticmethod', 'classmethod') or
2340
+ env.lookup_here(func.name)):
2341
+ # not a static or classmethod
2342
+ decorators.append(decorator)
2343
+
2344
+ if decorators:
2345
+ transform = DecoratorTransform(self.context)
2346
+ def_node = node.node
2347
+ _, reassignments = transform.chain_decorators(
2348
+ def_node, decorators, def_node.name)
2349
+ reassignments.analyse_declarations(env)
2350
+ node = [node, reassignments]
2351
+
2352
+ return node
2353
+
2354
+ def _handle_def(self, decorators, env, node):
2355
+ "Handle def or cpdef fused functions"
2356
+ # Create PyCFunction nodes for each specialization
2357
+ node.stats.insert(0, node.py_func)
2358
+ self.visitchild(node, 'py_func')
2359
+ node.update_fused_defnode_entry(env)
2360
+ # For the moment, fused functions do not support METH_FASTCALL
2361
+ node.py_func.entry.signature.use_fastcall = False
2362
+ pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True)
2363
+ pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
2364
+ node.resulting_fused_function = pycfunc
2365
+ # Create assignment node for our def function
2366
+ node.fused_func_assignment = self._create_assignment(
2367
+ node.py_func, ExprNodes.CloneNode(pycfunc), env)
2368
+
2369
+ if decorators:
2370
+ node = self._handle_fused_def_decorators(decorators, env, node)
2371
+
2372
+ return node
2373
+
2374
+ def _create_fused_function(self, env, node):
2375
+ "Create a fused function for a DefNode with fused arguments"
2376
+ from . import FusedNode
2377
+
2378
+ if self.fused_function or self.in_lambda:
2379
+ if self.fused_function not in self.fused_error_funcs:
2380
+ if self.in_lambda:
2381
+ error(node.pos, "Fused lambdas not allowed")
2382
+ else:
2383
+ error(node.pos, "Cannot nest fused functions")
2384
+
2385
+ self.fused_error_funcs.add(self.fused_function)
2386
+
2387
+ node.body = Nodes.PassStatNode(node.pos)
2388
+ for arg in node.args:
2389
+ if arg.type.is_fused:
2390
+ arg.type = arg.type.get_fused_types()[0]
2391
+
2392
+ return node
2393
+
2394
+ decorators = getattr(node, 'decorators', None)
2395
+ node = FusedNode.FusedCFuncDefNode(node, env)
2396
+ self.fused_function = node
2397
+ self.visitchildren(node)
2398
+ self.fused_function = None
2399
+ if node.py_func:
2400
+ node = self._handle_def(decorators, env, node)
2401
+
2402
+ return node
2403
+
2404
+ def _handle_fused(self, node):
2405
+ if node.is_generator and node.has_fused_arguments:
2406
+ error(node.pos, "Fused generators not supported")
2407
+ node.has_fused_arguments = False
2408
+ node.gbody.body = Nodes.StatListNode(node.pos, stats=[])
2409
+
2410
+ return node.has_fused_arguments
2411
+
2412
+ def visit_FuncDefNode(self, node):
2413
+ """
2414
+ Analyse a function and its body, as that hasn't happened yet. Also
2415
+ analyse the directive_locals set by @cython.locals().
2416
+
2417
+ Then, if we are a function with fused arguments, replace the function
2418
+ (after it has declared itself in the symbol table!) with a
2419
+ FusedCFuncDefNode, and analyse its children (which are in turn normal
2420
+ functions). If we're a normal function, just analyse the body of the
2421
+ function.
2422
+ """
2423
+ env = self.current_env()
2424
+
2425
+ self.seen_vars_stack.append(set())
2426
+ lenv = node.local_scope
2427
+ node.declare_arguments(lenv)
2428
+
2429
+ # @cython.locals(...)
2430
+ for var, type_node in node.directive_locals.items():
2431
+ if not lenv.lookup_here(var): # don't redeclare args
2432
+ type = type_node.analyse_as_type(lenv)
2433
+ if type and type.is_fused and lenv.fused_to_specific:
2434
+ type = type.specialize(lenv.fused_to_specific)
2435
+ if type:
2436
+ lenv.declare_var(var, type, type_node.pos)
2437
+ else:
2438
+ error(type_node.pos, "Not a type")
2439
+
2440
+ if self._handle_fused(node):
2441
+ node = self._create_fused_function(env, node)
2442
+ else:
2443
+ node.body.analyse_declarations(lenv)
2444
+ node = self._super_visit_FuncDefNode(node)
2445
+
2446
+ self.seen_vars_stack.pop()
2447
+
2448
+ if "ufunc" in lenv.directives:
2449
+ from . import UFuncs
2450
+ return UFuncs.convert_to_ufunc(node)
2451
+ return node
2452
+
2453
+ def visit_DefNode(self, node):
2454
+ node = self.visit_FuncDefNode(node)
2455
+ if not isinstance(node, Nodes.DefNode):
2456
+ return node
2457
+ env = self.current_env()
2458
+ if node.code_object is None:
2459
+ node.code_object = ExprNodes.CodeObjectNode(node)
2460
+ node.code_object.analyse_declarations(env)
2461
+ if node.fused_py_func or node.is_generator_body:
2462
+ return node
2463
+ if not node.needs_assignment_synthesis(env):
2464
+ return node
2465
+ return [node, self._synthesize_assignment(node, env)]
2466
+
2467
+ def visit_CFuncDefNode(self, node):
2468
+ if node.code_object is None and node.py_func is None:
2469
+ node.code_object = ExprNodes.CodeObjectNode.for_cfunc(node)
2470
+ node.code_object.analyse_declarations(self.current_env())
2471
+ return self.visit_FuncDefNode(node)
2472
+
2473
+ def visit_GeneratorBodyDefNode(self, node):
2474
+ return self.visit_FuncDefNode(node)
2475
+
2476
+ def visit_GeneratorDefNode(self, node):
2477
+ # The generator body should use the same code object as the (user facing) generator function that creates it.
2478
+ result = self.visit_DefNode(node)
2479
+ # 'result' will usually be a list of statements, but we still have the original node.
2480
+ node.gbody.code_object = node.code_object
2481
+ return result
2482
+
2483
+ def _synthesize_assignment(self, node, env):
2484
+ # Synthesize assignment node and put it right after defnode
2485
+ genv = env
2486
+ while genv.is_py_class_scope or genv.is_c_class_scope:
2487
+ genv = genv.outer_scope
2488
+
2489
+ binding = env.is_py_class_scope or self.current_directives.get('binding')
2490
+ if genv.is_closure_scope:
2491
+ rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode.from_defnode(node, binding)
2492
+ else:
2493
+ rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
2494
+
2495
+ node.is_cyfunction = rhs.binding
2496
+ return self._create_assignment(node, rhs, env)
2497
+
2498
+ def _create_assignment(self, def_node, rhs, env):
2499
+ if def_node.decorators:
2500
+ for decorator in def_node.decorators[::-1]:
2501
+ rhs = ExprNodes.SimpleCallNode(
2502
+ decorator.pos,
2503
+ function = decorator.decorator,
2504
+ args = [rhs])
2505
+ def_node.decorators = None
2506
+
2507
+ assmt = Nodes.SingleAssignmentNode(
2508
+ def_node.pos,
2509
+ lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
2510
+ rhs=rhs)
2511
+ assmt.analyse_declarations(env)
2512
+ return assmt
2513
+
2514
+ def visit_func_outer_attrs(self, node):
2515
+ # any names in the outer attrs should not be looked up in the function "seen_vars_stack"
2516
+ stack = self.seen_vars_stack.pop()
2517
+ super().visit_func_outer_attrs(node)
2518
+ self.seen_vars_stack.append(stack)
2519
+
2520
+ def visit_ScopedExprNode(self, node):
2521
+ env = self.current_env()
2522
+ node.analyse_declarations(env)
2523
+ # the node may or may not have a local scope
2524
+ if node.expr_scope:
2525
+ self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
2526
+ self.enter_scope(node, node.expr_scope)
2527
+ node.analyse_scoped_declarations(node.expr_scope)
2528
+ self.visitchildren(node)
2529
+ self.exit_scope()
2530
+ self.seen_vars_stack.pop()
2531
+ else:
2532
+
2533
+ node.analyse_scoped_declarations(env)
2534
+ self.visitchildren(node)
2535
+ return node
2536
+
2537
+ def visit_TempResultFromStatNode(self, node):
2538
+ self.visitchildren(node)
2539
+ node.analyse_declarations(self.current_env())
2540
+ return node
2541
+
2542
+ def visit_CppClassNode(self, node):
2543
+ if node.visibility == 'extern':
2544
+ return None
2545
+ else:
2546
+ return self.visit_ClassDefNode(node)
2547
+
2548
+ def visit_CStructOrUnionDefNode(self, node):
2549
+ # Create a wrapper node if needed.
2550
+ # We want to use the struct type information (so it can't happen
2551
+ # before this phase) but also create new objects to be declared
2552
+ # (so it can't happen later).
2553
+ # Note that we don't return the original node, as it is
2554
+ # never used after this phase.
2555
+ if True: # private (default)
2556
+ return None
2557
+
2558
+ self_value = ExprNodes.AttributeNode(
2559
+ pos = node.pos,
2560
+ obj = ExprNodes.NameNode(pos=node.pos, name="self"),
2561
+ attribute = EncodedString("value"))
2562
+ var_entries = node.entry.type.scope.var_entries
2563
+ attributes = []
2564
+ for entry in var_entries:
2565
+ attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
2566
+ obj = self_value,
2567
+ attribute = entry.name))
2568
+ # __init__ assignments
2569
+ init_assignments = []
2570
+ for entry, attr in zip(var_entries, attributes):
2571
+ # TODO: branch on visibility
2572
+ init_assignments.append(self.init_assignment.substitute({
2573
+ "VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
2574
+ "ATTR": attr,
2575
+ }, pos = entry.pos))
2576
+
2577
+ # create the class
2578
+ str_format = "%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
2579
+ wrapper_class = self.struct_or_union_wrapper.substitute({
2580
+ "INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
2581
+ "IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
2582
+ "MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
2583
+ "STR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format)),
2584
+ "REPR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
2585
+ }, pos = node.pos).stats[0]
2586
+ wrapper_class.class_name = node.name
2587
+ wrapper_class.shadow = True
2588
+ class_body = wrapper_class.body.stats
2589
+
2590
+ # fix value type
2591
+ assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
2592
+ class_body[0].base_type.name = node.name
2593
+
2594
+ # fix __init__ arguments
2595
+ init_method = class_body[1]
2596
+ assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
2597
+ arg_template = init_method.args[1]
2598
+ if not node.entry.type.is_struct:
2599
+ arg_template.kw_only = True
2600
+ del init_method.args[1]
2601
+ for entry, attr in zip(var_entries, attributes):
2602
+ arg = copy.deepcopy(arg_template)
2603
+ arg.declarator.name = entry.name
2604
+ init_method.args.append(arg)
2605
+
2606
+ # setters/getters
2607
+ for entry, attr in zip(var_entries, attributes):
2608
+ # TODO: branch on visibility
2609
+ if entry.type.is_pyobject:
2610
+ template = self.basic_pyobject_property
2611
+ else:
2612
+ template = self.basic_property
2613
+ property = template.substitute({
2614
+ "ATTR": attr,
2615
+ }, pos = entry.pos).stats[0]
2616
+ property.name = entry.name
2617
+ wrapper_class.body.stats.append(property)
2618
+
2619
+ wrapper_class.analyse_declarations(self.current_env())
2620
+ return self.visit_CClassDefNode(wrapper_class)
2621
+
2622
+ # Some nodes are no longer needed after declaration
2623
+ # analysis and can be dropped. The analysis was performed
2624
+ # on these nodes in a separate recursive process from the
2625
+ # enclosing function or module, so we can simply drop them.
2626
+ def visit_CDeclaratorNode(self, node):
2627
+ # necessary to ensure that all CNameDeclaratorNodes are visited.
2628
+ self.visitchildren(node)
2629
+ return node
2630
+
2631
+ def visit_CTypeDefNode(self, node):
2632
+ return node
2633
+
2634
+ def visit_CBaseTypeNode(self, node):
2635
+ return None
2636
+
2637
+ def visit_CEnumDefNode(self, node):
2638
+ if node.visibility == 'public':
2639
+ return node
2640
+ else:
2641
+ return None
2642
+
2643
+ def visit_CNameDeclaratorNode(self, node):
2644
+ if node.name in self.seen_vars_stack[-1]:
2645
+ entry = self.current_env().lookup(node.name)
2646
+ if (entry is None or entry.visibility != 'extern'
2647
+ and not entry.scope.is_c_class_scope):
2648
+ error(node.pos, "cdef variable '%s' declared after it is used" % node.name)
2649
+ self.visitchildren(node)
2650
+ return node
2651
+
2652
+ def visit_CVarDefNode(self, node):
2653
+ # to ensure all CNameDeclaratorNodes are visited.
2654
+ self.visitchildren(node)
2655
+ return None
2656
+
2657
+ def visit_CnameDecoratorNode(self, node):
2658
+ child_node = self.visitchild(node, 'node')
2659
+ if not child_node:
2660
+ return None
2661
+ if type(child_node) is list: # Assignment synthesized
2662
+ node.node = child_node[0]
2663
+ return [node] + child_node[1:]
2664
+ return node
2665
+
2666
+ def create_Property(self, entry):
2667
+ if entry.visibility == 'public':
2668
+ if entry.type.is_pyobject:
2669
+ template = self.basic_pyobject_property
2670
+ else:
2671
+ template = self.basic_property
2672
+ elif entry.visibility == 'readonly':
2673
+ template = self.basic_property_ro
2674
+ property = template.substitute({
2675
+ "ATTR": ExprNodes.AttributeNode(pos=entry.pos,
2676
+ obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
2677
+ attribute=entry.name),
2678
+ }, pos=entry.pos).stats[0]
2679
+ property.name = entry.name
2680
+ property.doc = entry.doc
2681
+ return property
2682
+
2683
+ def visit_AssignmentExpressionNode(self, node):
2684
+ self.visitchildren(node)
2685
+ node.analyse_declarations(self.current_env())
2686
+ return node
2687
+
2688
+
2689
+ def _calculate_pickle_checksums(member_names):
2690
+ # Cython 0.x used MD5 for the checksum, which a few Python installations remove for security reasons.
2691
+ # SHA-256 should be ok for years to come, but early Cython 3.0 alpha releases used SHA-1,
2692
+ # which may not be.
2693
+ member_names_string = ' '.join(member_names).encode('utf-8')
2694
+ hash_kwargs = {'usedforsecurity': False} if sys.version_info >= (3, 9) else {}
2695
+ checksums = []
2696
+ for algo_name in ['sha256', 'sha1', 'md5']:
2697
+ try:
2698
+ mkchecksum = getattr(hashlib, algo_name)
2699
+ checksum = mkchecksum(member_names_string, **hash_kwargs).hexdigest()
2700
+ except (AttributeError, ValueError):
2701
+ # The algorithm (i.e. MD5) might not be there at all, or might be blocked at runtime.
2702
+ continue
2703
+ checksums.append('0x' + checksum[:7])
2704
+ return checksums
2705
+
2706
+
2707
+ class CalculateQualifiedNamesTransform(EnvTransform):
2708
+ """
2709
+ Calculate and store the '__qualname__' and the global
2710
+ module name on some nodes.
2711
+ """
2712
+ needs_qualname_assignment = False
2713
+ needs_module_assignment = False
2714
+
2715
+ def visit_ModuleNode(self, node):
2716
+ self.module_name = self.global_scope().qualified_name
2717
+ self.qualified_name = []
2718
+ _super = super()
2719
+ self._super_visit_FuncDefNode = _super.visit_FuncDefNode
2720
+ self._super_visit_ClassDefNode = _super.visit_ClassDefNode
2721
+ self.visitchildren(node)
2722
+ return node
2723
+
2724
+ def _set_qualname(self, node, name=None):
2725
+ if name:
2726
+ qualname = self.qualified_name[:]
2727
+ qualname.append(name)
2728
+ else:
2729
+ qualname = self.qualified_name
2730
+ node.qualname = EncodedString('.'.join(qualname))
2731
+ node.module_name = self.module_name
2732
+
2733
+ def _append_entry(self, entry):
2734
+ if entry.is_pyglobal and not entry.is_pyclass_attr:
2735
+ self.qualified_name = [entry.name]
2736
+ else:
2737
+ self.qualified_name.append(entry.name)
2738
+
2739
+ def visit_ClassNode(self, node):
2740
+ self._set_qualname(node, node.name)
2741
+ self.visitchildren(node)
2742
+ return node
2743
+
2744
+ def visit_PyClassNamespaceNode(self, node):
2745
+ # class name was already added by parent node
2746
+ self._set_qualname(node)
2747
+ self.visitchildren(node)
2748
+ return node
2749
+
2750
+ def visit_PyCFunctionNode(self, node):
2751
+ orig_qualified_name = self.qualified_name[:]
2752
+ if node.def_node.is_wrapper and self.qualified_name and self.qualified_name[-1] == '<locals>':
2753
+ self.qualified_name.pop()
2754
+ self._set_qualname(node)
2755
+ else:
2756
+ self._set_qualname(node, node.def_node.name)
2757
+ self.visitchildren(node)
2758
+ self.qualified_name = orig_qualified_name
2759
+ return node
2760
+
2761
+ def visit_DefNode(self, node):
2762
+ if node.is_wrapper and self.qualified_name:
2763
+ assert self.qualified_name[-1] == '<locals>', self.qualified_name
2764
+ orig_qualified_name = self.qualified_name[:]
2765
+ self.qualified_name.pop()
2766
+ self._set_qualname(node)
2767
+ self._super_visit_FuncDefNode(node)
2768
+ self.qualified_name = orig_qualified_name
2769
+ else:
2770
+ self._set_qualname(node, node.name)
2771
+ self.visit_FuncDefNode(node)
2772
+ return node
2773
+
2774
+ def visit_FuncDefNode(self, node):
2775
+ orig_qualified_name = self.qualified_name[:]
2776
+ if getattr(node, 'name', None) == '<lambda>':
2777
+ self.qualified_name.append('<lambda>')
2778
+ else:
2779
+ self._append_entry(node.entry)
2780
+ self.qualified_name.append('<locals>')
2781
+ self._super_visit_FuncDefNode(node)
2782
+ self.qualified_name = orig_qualified_name
2783
+ return node
2784
+
2785
+ def generate_assignment(self, node, name, value):
2786
+ entry = node.scope.lookup_here(name)
2787
+ lhs = ExprNodes.NameNode(
2788
+ node.pos,
2789
+ name=EncodedString(name),
2790
+ entry=entry,
2791
+ is_target=True)
2792
+ rhs = ExprNodes.UnicodeNode(node.pos, value=value)
2793
+ node.body.stats.insert(0, Nodes.SingleAssignmentNode(
2794
+ node.pos,
2795
+ lhs=lhs,
2796
+ rhs=rhs,
2797
+ ).analyse_expressions(self.current_env()))
2798
+
2799
+ def visit_ClassDefNode(self, node):
2800
+ orig_needs_qualname_assignment = self.needs_qualname_assignment
2801
+ self.needs_qualname_assignment = False
2802
+ orig_needs_module_assignment = self.needs_module_assignment
2803
+ self.needs_module_assignment = False
2804
+ orig_qualified_name = self.qualified_name[:]
2805
+ entry = (getattr(node, 'entry', None) or # PyClass
2806
+ self.current_env().lookup_here(node.target.name)) # CClass
2807
+ self._append_entry(entry)
2808
+ self._super_visit_ClassDefNode(node)
2809
+ if self.needs_qualname_assignment:
2810
+ self.generate_assignment(node, "__qualname__",
2811
+ EncodedString(".".join(self.qualified_name)))
2812
+ if self.needs_module_assignment:
2813
+ self.generate_assignment(node, "__module__",
2814
+ EncodedString(self.module_name))
2815
+ self.qualified_name = orig_qualified_name
2816
+ self.needs_qualname_assignment = orig_needs_qualname_assignment
2817
+ self.needs_module_assignment = orig_needs_module_assignment
2818
+ return node
2819
+
2820
+ def visit_NameNode(self, node):
2821
+ scope = self.current_env()
2822
+ if scope.is_c_class_scope:
2823
+ # unlike for a PyClass scope, these attributes aren't defined in the
2824
+ # dictionary when the class definition is executed, therefore we ask
2825
+ # the compiler to generate an assignment to them at the start of the
2826
+ # body.
2827
+ # NOTE: this doesn't put them in locals()
2828
+ if node.name == "__qualname__":
2829
+ self.needs_qualname_assignment = True
2830
+ elif node.name == "__module__":
2831
+ self.needs_module_assignment = True
2832
+ return node
2833
+
2834
+
2835
+ class AnalyseExpressionsTransform(CythonTransform):
2836
+
2837
+ def visit_ModuleNode(self, node):
2838
+ node.scope.infer_types()
2839
+ node.body = node.body.analyse_expressions(node.scope)
2840
+ self.positions = [{node.pos}]
2841
+ self.visitchildren(node)
2842
+ self._build_positions(node)
2843
+ return node
2844
+
2845
+ def visit_FuncDefNode(self, node):
2846
+ node.local_scope.infer_types()
2847
+ node.body = node.body.analyse_expressions(node.local_scope)
2848
+ self.positions[-1].add(node.pos)
2849
+
2850
+ if node.is_wrapper:
2851
+ # Share positions between function and Python wrapper.
2852
+ local_positions = self.positions[-1]
2853
+ else:
2854
+ local_positions = {node.pos}
2855
+ self.positions.append(local_positions)
2856
+
2857
+ self.visitchildren(node)
2858
+ self._build_positions(node)
2859
+ return node
2860
+
2861
+ def visit_ScopedExprNode(self, node):
2862
+ if node.has_local_scope:
2863
+ node.expr_scope.infer_types()
2864
+ node = node.analyse_scoped_expressions(node.expr_scope)
2865
+ self.visit_ExprNode(node)
2866
+ return node
2867
+
2868
+ def visit_IndexNode(self, node):
2869
+ """
2870
+ Replace index nodes used to specialize cdef functions with fused
2871
+ argument types with the Attribute- or NameNode referring to the
2872
+ function. We then need to copy over the specialization properties to
2873
+ the attribute or name node.
2874
+
2875
+ Because the indexing might be a Python indexing operation on a fused
2876
+ function, or (usually) a Cython indexing operation, we need to
2877
+ re-analyse the types.
2878
+ """
2879
+ self.visit_ExprNode(node)
2880
+ if node.is_fused_index and not node.type.is_error:
2881
+ node = node.base
2882
+ return node
2883
+
2884
+ # Build the line table according to PEP-626.
2885
+ # We mostly just do this here to avoid yet another transform traversal.
2886
+
2887
+ def visit_ExprNode(self, node):
2888
+ self.positions[-1].add(node.pos)
2889
+ self.visitchildren(node)
2890
+ return node
2891
+
2892
+ def visit_StatNode(self, node):
2893
+ self.positions[-1].add(node.pos)
2894
+ self.visitchildren(node)
2895
+ return node
2896
+
2897
+ def _build_positions(self, func_node):
2898
+ """
2899
+ Build the PEP-626 line table and "bytecode-to-position" mapping used for CodeObjects.
2900
+ """
2901
+ # Code can originate from different source files and string code fragments, even within a single function.
2902
+ # Thus, it's not completely correct to just ignore the source files when sorting the line numbers,
2903
+ # but it also doesn't hurt much for the moment. Eventually, we might need different CodeObjects
2904
+ # even within a single function if it uses code from different sources / line number ranges.
2905
+ positions: list = sorted(
2906
+ self.positions.pop(),
2907
+ key=itemgetter(1, 2), # (line, column)
2908
+ # Build ranges backwards to know the end column before we see the start column in the same line.
2909
+ reverse=True,
2910
+ )
2911
+
2912
+ next_line = -1
2913
+ next_column_in_line = 0
2914
+
2915
+ ranges = []
2916
+ for _, line, start_column in positions:
2917
+ ranges.append((line, line, start_column, next_column_in_line if line == next_line else start_column + 1))
2918
+ next_line, next_column_in_line = line, start_column
2919
+
2920
+ ranges.reverse()
2921
+ func_node.node_positions = ranges
2922
+
2923
+ positions.reverse()
2924
+ i: cython.Py_ssize_t
2925
+ func_node.local_scope.node_positions_to_offset = {
2926
+ position: i
2927
+ for i, position in enumerate(positions)
2928
+ }
2929
+
2930
+
2931
+ class FindInvalidUseOfFusedTypes(TreeVisitor):
2932
+
2933
+ def __call__(self, tree):
2934
+ self._in_fused_function = False
2935
+ self.visit(tree)
2936
+ return tree
2937
+
2938
+ def visit_Node(self, node):
2939
+ self.visitchildren(node)
2940
+
2941
+ def visit_FuncDefNode(self, node):
2942
+ outer_status = self._in_fused_function
2943
+ self._in_fused_function = node.has_fused_arguments
2944
+
2945
+ if not self._in_fused_function:
2946
+ # Errors related to use in functions with fused args will already
2947
+ # have been detected.
2948
+ if not node.is_generator_body and node.return_type.is_fused:
2949
+ error(node.pos, "Return type is not specified as argument type")
2950
+
2951
+ self.visitchildren(node)
2952
+ self._in_fused_function = outer_status
2953
+
2954
+ def visit_ExprNode(self, node):
2955
+ if not self._in_fused_function and node.type and node.type.is_fused:
2956
+ error(node.pos, "Invalid use of fused types, type cannot be specialized")
2957
+ # Errors in subtrees are likely related, so do not recurse.
2958
+ else:
2959
+ self.visitchildren(node)
2960
+
2961
+
2962
+ class ExpandInplaceOperators(EnvTransform):
2963
+
2964
+ def visit_InPlaceAssignmentNode(self, node):
2965
+ lhs = node.lhs
2966
+ rhs = node.rhs
2967
+ if lhs.type.is_cpp_class:
2968
+ # No getting around this exact operator here.
2969
+ return node
2970
+ if isinstance(lhs, ExprNodes.BufferIndexNode):
2971
+ # There is code to handle this case in InPlaceAssignmentNode
2972
+ return node
2973
+
2974
+ env = self.current_env()
2975
+ def side_effect_free_reference(node, setting=False):
2976
+ if node.is_name:
2977
+ return node, []
2978
+ elif node.type.is_pyobject and not setting:
2979
+ node = LetRefNode(node)
2980
+ return node, [node]
2981
+ elif node.is_subscript:
2982
+ base, temps = side_effect_free_reference(node.base)
2983
+ index = LetRefNode(node.index)
2984
+ return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
2985
+ elif node.is_attribute:
2986
+ obj, temps = side_effect_free_reference(node.obj, setting=setting)
2987
+ return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
2988
+ elif isinstance(node, ExprNodes.BufferIndexNode):
2989
+ raise ValueError("Don't allow things like attributes of buffer indexing operations")
2990
+ else:
2991
+ node = LetRefNode(node)
2992
+ return node, [node]
2993
+ try:
2994
+ lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
2995
+ except ValueError:
2996
+ return node
2997
+ dup = lhs.__class__(**lhs.__dict__)
2998
+ binop = ExprNodes.binop_node(node.pos,
2999
+ operator = node.operator,
3000
+ operand1 = dup,
3001
+ operand2 = rhs,
3002
+ inplace=True)
3003
+ # Manually analyse types for new node.
3004
+ lhs.is_target = True
3005
+ lhs = lhs.analyse_target_types(env)
3006
+ dup.analyse_types(env) # FIXME: no need to reanalyse the copy, right?
3007
+ binop.analyse_operation(env)
3008
+ node = Nodes.SingleAssignmentNode(
3009
+ node.pos,
3010
+ lhs = lhs,
3011
+ rhs=binop.coerce_to(lhs.type, env))
3012
+ # Use LetRefNode to avoid side effects.
3013
+ let_ref_nodes.reverse()
3014
+ for t in let_ref_nodes:
3015
+ node = LetNode(t, node)
3016
+ return node
3017
+
3018
+ def visit_ExprNode(self, node):
3019
+ # In-place assignments can't happen within an expression.
3020
+ return node
3021
+
3022
+
3023
+ class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
3024
+ """
3025
+ Adjust function and class definitions by the decorator directives:
3026
+
3027
+ @cython.cfunc
3028
+ @cython.cclass
3029
+ @cython.ccall
3030
+ @cython.inline
3031
+ @cython.nogil
3032
+ """
3033
+ # list of directives that cause conversion to cclass
3034
+ converts_to_cclass = ('cclass', 'total_ordering', 'dataclasses.dataclass')
3035
+
3036
+ def visit_ModuleNode(self, node):
3037
+ self.directives = node.directives
3038
+ self.in_py_class = False
3039
+ self.visitchildren(node)
3040
+ return node
3041
+
3042
+ def visit_CompilerDirectivesNode(self, node):
3043
+ old_directives = self.directives
3044
+ self.directives = node.directives
3045
+ self.visitchildren(node)
3046
+ self.directives = old_directives
3047
+ return node
3048
+
3049
+ def visit_DefNode(self, node):
3050
+ modifiers = []
3051
+ if 'inline' in self.directives:
3052
+ modifiers.append('inline')
3053
+ nogil = self.directives.get('nogil')
3054
+ with_gil = self.directives.get('with_gil')
3055
+ except_val = self.directives.get('exceptval')
3056
+ has_explicit_exc_clause = False if except_val is None else True
3057
+ return_type_node = self.directives.get('returns')
3058
+ if return_type_node is None and self.directives['annotation_typing']:
3059
+ return_type_node = node.return_type_annotation
3060
+ # for Python annotations, prefer safe exception handling by default
3061
+ if return_type_node is not None and except_val is None:
3062
+ except_val = (None, True) # except *
3063
+ elif except_val is None:
3064
+ # backward compatible default: no exception check, unless there's also a "@returns" declaration
3065
+ except_val = (None, True if return_type_node else False)
3066
+ if 'ccall' in self.directives:
3067
+ if 'cfunc' in self.directives:
3068
+ error(node.pos, "cfunc and ccall directives cannot be combined")
3069
+ if with_gil:
3070
+ error(node.pos, "ccall functions cannot be declared 'with_gil'")
3071
+ node = node.as_cfunction(
3072
+ overridable=True, modifiers=modifiers, nogil=nogil,
3073
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3074
+ return self.visit(node)
3075
+ if 'cfunc' in self.directives:
3076
+ if self.in_py_class:
3077
+ error(node.pos, "cfunc directive is not allowed here")
3078
+ else:
3079
+ node = node.as_cfunction(
3080
+ overridable=False, modifiers=modifiers, nogil=nogil, with_gil=with_gil,
3081
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3082
+ return self.visit(node)
3083
+ if 'inline' in modifiers:
3084
+ error(node.pos, "Python functions cannot be declared 'inline'")
3085
+ if nogil:
3086
+ # TODO: turn this into a "with gil" declaration.
3087
+ error(node.pos, "Python functions cannot be declared 'nogil'")
3088
+ if with_gil:
3089
+ error(node.pos, "Python functions cannot be declared 'with_gil'")
3090
+ self.visitchildren(node)
3091
+ return node
3092
+
3093
+ def visit_LambdaNode(self, node):
3094
+ # No directives should modify lambdas or generator expressions (and also nothing in them).
3095
+ return node
3096
+
3097
+ def visit_PyClassDefNode(self, node):
3098
+ if any(directive in self.directives for directive in self.converts_to_cclass):
3099
+ node = node.as_cclass()
3100
+ return self.visit(node)
3101
+ else:
3102
+ old_in_pyclass = self.in_py_class
3103
+ self.in_py_class = True
3104
+ self.visitchildren(node)
3105
+ self.in_py_class = old_in_pyclass
3106
+ return node
3107
+
3108
+ def visit_CClassDefNode(self, node):
3109
+ old_in_pyclass = self.in_py_class
3110
+ self.in_py_class = False
3111
+ self.visitchildren(node)
3112
+ self.in_py_class = old_in_pyclass
3113
+ return node
3114
+
3115
+
3116
+ class AlignFunctionDefinitions(CythonTransform):
3117
+ """
3118
+ This class takes the signatures from a .pxd file and applies them to
3119
+ the def methods in a .py file.
3120
+ """
3121
+
3122
+ def visit_ModuleNode(self, node):
3123
+ self.scope = node.scope
3124
+ self.visitchildren(node)
3125
+ return node
3126
+
3127
+ def visit_PyClassDefNode(self, node):
3128
+ pxd_def = self.scope.lookup(node.name)
3129
+ if pxd_def:
3130
+ if pxd_def.is_cclass:
3131
+ return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
3132
+ elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
3133
+ error(node.pos, "'%s' redeclared" % node.name)
3134
+ if pxd_def.pos:
3135
+ error(pxd_def.pos, "previous declaration here")
3136
+ return None
3137
+ return node
3138
+
3139
+ def visit_CClassDefNode(self, node, pxd_def=None):
3140
+ if pxd_def is None:
3141
+ pxd_def = self.scope.lookup(node.class_name)
3142
+ if pxd_def:
3143
+ if not pxd_def.defined_in_pxd:
3144
+ return node
3145
+ outer_scope = self.scope
3146
+ self.scope = pxd_def.type.scope
3147
+ self.visitchildren(node)
3148
+ if pxd_def:
3149
+ self.scope = outer_scope
3150
+ return node
3151
+
3152
+ def visit_DefNode(self, node):
3153
+ pxd_def = self.scope.lookup(node.name)
3154
+ if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
3155
+ if not pxd_def.is_cfunction:
3156
+ error(node.pos, "'%s' redeclared" % node.name)
3157
+ if pxd_def.pos:
3158
+ error(pxd_def.pos, "previous declaration here")
3159
+ return None
3160
+ node = node.as_cfunction(pxd_def)
3161
+ # Enable this when nested cdef functions are allowed.
3162
+ # self.visitchildren(node)
3163
+ return node
3164
+
3165
+ def visit_ExprNode(self, node):
3166
+ # ignore lambdas and everything else that appears in expressions
3167
+ return node
3168
+
3169
+
3170
+ class AutoCpdefFunctionDefinitions(CythonTransform):
3171
+
3172
+ def visit_ModuleNode(self, node):
3173
+ self.directives = node.directives
3174
+ self.imported_names = set() # hack, see visit_FromImportStatNode()
3175
+ self.scope = node.scope
3176
+ self.visitchildren(node)
3177
+ return node
3178
+
3179
+ def visit_DefNode(self, node):
3180
+ if (self.scope.is_module_scope and self.directives['auto_cpdef']
3181
+ and node.name not in self.imported_names
3182
+ and node.is_cdef_func_compatible()):
3183
+ # FIXME: cpdef-ing should be done in analyse_declarations()
3184
+ node = node.as_cfunction(scope=self.scope)
3185
+ return node
3186
+
3187
+ def visit_CClassDefNode(self, node, pxd_def=None):
3188
+ if pxd_def is None:
3189
+ pxd_def = self.scope.lookup(node.class_name)
3190
+ if pxd_def:
3191
+ if not pxd_def.defined_in_pxd:
3192
+ return node
3193
+ outer_scope = self.scope
3194
+ self.scope = pxd_def.type.scope
3195
+ self.visitchildren(node)
3196
+ if pxd_def:
3197
+ self.scope = outer_scope
3198
+ return node
3199
+
3200
+ def visit_FromImportStatNode(self, node):
3201
+ # hack to prevent conditional import fallback functions from
3202
+ # being cdpef-ed (global Python variables currently conflict
3203
+ # with imports)
3204
+ if self.scope.is_module_scope:
3205
+ for name, _ in node.items:
3206
+ self.imported_names.add(name)
3207
+ return node
3208
+
3209
+ def visit_ExprNode(self, node):
3210
+ # ignore lambdas and everything else that appears in expressions
3211
+ return node
3212
+
3213
+
3214
+ class RemoveUnreachableCode(CythonTransform):
3215
+
3216
+ def visit_StatListNode(self, node):
3217
+ if not self.current_directives['remove_unreachable']:
3218
+ return node
3219
+ self.visitchildren(node)
3220
+ if len(node.stats) == 1 and isinstance(node.stats[0], Nodes.StatListNode) and not node.stats[0].stats:
3221
+ del node.stats[:]
3222
+ for idx, stat in enumerate(node.stats, 1):
3223
+ if stat.is_terminator:
3224
+ if idx < len(node.stats):
3225
+ if self.current_directives['warn.unreachable']:
3226
+ warning(node.stats[idx].pos, "Unreachable code", 2)
3227
+ node.stats = node.stats[:idx]
3228
+ node.is_terminator = True
3229
+ break
3230
+ return node
3231
+
3232
+ def visit_IfClauseNode(self, node):
3233
+ self.visitchildren(node)
3234
+ if node.body.is_terminator:
3235
+ node.is_terminator = True
3236
+ return node
3237
+
3238
+ def visit_IfStatNode(self, node):
3239
+ self.visitchildren(node)
3240
+ if node.else_clause and node.else_clause.is_terminator:
3241
+ for clause in node.if_clauses:
3242
+ if not clause.is_terminator:
3243
+ break
3244
+ else:
3245
+ node.is_terminator = True
3246
+ return node
3247
+
3248
+ def visit_TryExceptStatNode(self, node):
3249
+ self.visitchildren(node)
3250
+ if node.body.is_terminator and node.else_clause:
3251
+ if self.current_directives['warn.unreachable']:
3252
+ warning(node.else_clause.pos, "Unreachable code", 2)
3253
+ node.else_clause = None
3254
+ return node
3255
+
3256
+ def visit_TryFinallyStatNode(self, node):
3257
+ self.visitchildren(node)
3258
+ if node.finally_clause.is_terminator:
3259
+ node.is_terminator = True
3260
+ return node
3261
+
3262
+ def visit_PassStatNode(self, node):
3263
+ """Eliminate useless PassStatNode"""
3264
+ # 'pass' statements often appear in a separate line and must be traced.
3265
+ if not self.current_directives['linetrace']:
3266
+ node = Nodes.StatListNode(pos=node.pos, stats=[])
3267
+ return node
3268
+
3269
+
3270
+ class YieldNodeCollector(TreeVisitor):
3271
+
3272
+ def __init__(self, excludes=[]):
3273
+ super().__init__()
3274
+ self.yields = []
3275
+ self.returns = []
3276
+ self.finallys = []
3277
+ self.excepts = []
3278
+ self.has_return_value = False
3279
+ self.has_yield = False
3280
+ self.has_await = False
3281
+ self.excludes = excludes
3282
+
3283
+ def visit_Node(self, node):
3284
+ if node not in self.excludes:
3285
+ self.visitchildren(node)
3286
+
3287
+ def visit_YieldExprNode(self, node):
3288
+ self.yields.append(node)
3289
+ self.has_yield = True
3290
+ self.visitchildren(node)
3291
+
3292
+ def visit_AwaitExprNode(self, node):
3293
+ self.yields.append(node)
3294
+ self.has_await = True
3295
+ self.visitchildren(node)
3296
+
3297
+ def visit_ReturnStatNode(self, node):
3298
+ self.visitchildren(node)
3299
+ if node.value:
3300
+ self.has_return_value = True
3301
+ self.returns.append(node)
3302
+
3303
+ def visit_TryFinallyStatNode(self, node):
3304
+ self.visitchildren(node)
3305
+ self.finallys.append(node)
3306
+
3307
+ def visit_TryExceptStatNode(self, node):
3308
+ self.visitchildren(node)
3309
+ self.excepts.append(node)
3310
+
3311
+ def visit_ClassDefNode(self, node):
3312
+ pass
3313
+
3314
+ def visit_FuncDefNode(self, node):
3315
+ pass
3316
+
3317
+ def visit_LambdaNode(self, node):
3318
+ pass
3319
+
3320
+ def visit_GeneratorExpressionNode(self, node):
3321
+ # node.loop iterator is evaluated outside the generator expression
3322
+ if isinstance(node.loop, Nodes._ForInStatNode):
3323
+ # Possibly should handle ForFromStatNode
3324
+ # but for now do nothing
3325
+ self.visit(node.loop.iterator)
3326
+
3327
+ def visit_CArgDeclNode(self, node):
3328
+ # do not look into annotations
3329
+ # FIXME: support (yield) in default arguments (currently crashes)
3330
+ pass
3331
+
3332
+
3333
+ class MarkClosureVisitor(CythonTransform):
3334
+ # In addition to marking closures this is also responsible to finding parts of the
3335
+ # generator iterable and marking them
3336
+
3337
+ def visit_ModuleNode(self, node):
3338
+ self.needs_closure = False
3339
+ self.excludes = []
3340
+ self.visitchildren(node)
3341
+ return node
3342
+
3343
+ def visit_FuncDefNode(self, node):
3344
+ self.needs_closure = False
3345
+ self.visitchildren(node)
3346
+ node.needs_closure = self.needs_closure
3347
+ self.needs_closure = True
3348
+
3349
+ collector = YieldNodeCollector(self.excludes)
3350
+ collector.visitchildren(node)
3351
+
3352
+ if node.is_async_def:
3353
+ coroutine_type = Nodes.AsyncDefNode
3354
+ if collector.has_yield:
3355
+ coroutine_type = Nodes.AsyncGenNode
3356
+ for yield_expr in collector.yields + collector.returns:
3357
+ yield_expr.in_async_gen = True
3358
+ elif self.current_directives['iterable_coroutine']:
3359
+ coroutine_type = Nodes.IterableAsyncDefNode
3360
+ elif collector.has_await:
3361
+ found = next(y for y in collector.yields if y.is_await)
3362
+ error(found.pos, "'await' not allowed in generators (use 'yield')")
3363
+ return node
3364
+ elif collector.has_yield:
3365
+ coroutine_type = Nodes.GeneratorDefNode
3366
+ else:
3367
+ return node
3368
+
3369
+ for i, yield_expr in enumerate(collector.yields, 1):
3370
+ yield_expr.label_num = i
3371
+ for retnode in collector.returns + collector.finallys + collector.excepts:
3372
+ retnode.in_generator = True
3373
+
3374
+ gbody = Nodes.GeneratorBodyDefNode(
3375
+ pos=node.pos, name=node.name, body=node.body,
3376
+ is_coroutine_body=node.is_async_def,
3377
+ is_async_gen_body=node.is_async_def and collector.has_yield)
3378
+ coroutine = coroutine_type(
3379
+ pos=node.pos, name=node.name, args=node.args,
3380
+ star_arg=node.star_arg, starstar_arg=node.starstar_arg,
3381
+ doc=node.doc, decorators=node.decorators,
3382
+ gbody=gbody, lambda_name=node.lambda_name,
3383
+ return_type_annotation=node.return_type_annotation,
3384
+ is_generator_expression=node.is_generator_expression)
3385
+ return coroutine
3386
+
3387
+ def visit_CFuncDefNode(self, node):
3388
+ self.needs_closure = False
3389
+ self.visitchildren(node)
3390
+ node.needs_closure = self.needs_closure
3391
+ self.needs_closure = True
3392
+ if node.needs_closure and node.overridable:
3393
+ error(node.pos, "closures inside cpdef functions not yet supported")
3394
+ return node
3395
+
3396
+ def visit_LambdaNode(self, node):
3397
+ self.needs_closure = False
3398
+ self.visitchildren(node)
3399
+ node.needs_closure = self.needs_closure
3400
+ self.needs_closure = True
3401
+ return node
3402
+
3403
+ def visit_ClassDefNode(self, node):
3404
+ self.visitchildren(node)
3405
+ self.needs_closure = True
3406
+ return node
3407
+
3408
+ def visit_GeneratorExpressionNode(self, node):
3409
+ excludes = self.excludes
3410
+ if isinstance(node.loop, Nodes._ForInStatNode):
3411
+ self.excludes = [node.loop.iterator]
3412
+ node = self.visit_LambdaNode(node)
3413
+ self.excludes = excludes
3414
+ if not isinstance(node.loop, Nodes._ForInStatNode):
3415
+ # Possibly should handle ForFromStatNode
3416
+ # but for now do nothing
3417
+ return node
3418
+ itseq = node.loop.iterator.sequence
3419
+ # literals do not need replacing with an argument
3420
+ if itseq.is_literal:
3421
+ return node
3422
+ _GeneratorExpressionArgumentsMarker(node).visit(itseq)
3423
+ return node
3424
+
3425
+
3426
+ class CreateClosureClasses(CythonTransform):
3427
+ # Output closure classes in module scope for all functions
3428
+ # that really need it.
3429
+
3430
+ def __init__(self, context):
3431
+ super().__init__(context)
3432
+ self.path = []
3433
+ self.in_lambda = False
3434
+
3435
+ def visit_ModuleNode(self, node):
3436
+ self.module_scope = node.scope
3437
+ self.visitchildren(node)
3438
+ return node
3439
+
3440
+ def find_entries_used_in_closures(self, node):
3441
+ from_closure = []
3442
+ in_closure = []
3443
+ for scope in node.local_scope.iter_local_scopes():
3444
+ for name, entry in scope.entries.items():
3445
+ if not name:
3446
+ continue
3447
+ if entry.from_closure:
3448
+ from_closure.append((name, entry))
3449
+ elif entry.in_closure:
3450
+ in_closure.append((name, entry))
3451
+ return from_closure, in_closure
3452
+
3453
+ def create_class_from_scope(self, node, target_module_scope, inner_node=None):
3454
+ # move local variables into closure
3455
+ if node.is_generator:
3456
+ for scope in node.local_scope.iter_local_scopes():
3457
+ for entry in scope.entries.values():
3458
+ if not (entry.from_closure or entry.is_pyglobal or entry.is_cglobal):
3459
+ entry.in_closure = True
3460
+
3461
+ from_closure, in_closure = self.find_entries_used_in_closures(node)
3462
+ in_closure.sort()
3463
+
3464
+ # Now from the beginning
3465
+ node.needs_closure = False
3466
+ node.needs_outer_scope = False
3467
+
3468
+ func_scope = node.local_scope
3469
+ cscope = node.entry.scope
3470
+ while cscope.is_py_class_scope or cscope.is_c_class_scope:
3471
+ cscope = cscope.outer_scope
3472
+
3473
+ if not from_closure and (self.path or inner_node):
3474
+ if not inner_node:
3475
+ if not node.py_cfunc_node:
3476
+ raise InternalError("DefNode does not have assignment node")
3477
+ inner_node = node.py_cfunc_node
3478
+ inner_node.needs_closure_code = False
3479
+ node.needs_outer_scope = False
3480
+
3481
+ if node.is_generator:
3482
+ pass
3483
+ elif not in_closure and not from_closure:
3484
+ return
3485
+ elif not in_closure:
3486
+ func_scope.is_passthrough = True
3487
+ func_scope.scope_class = cscope.scope_class
3488
+ node.needs_outer_scope = True
3489
+ return
3490
+
3491
+ # entry.cname can contain periods (eg. a derived C method of a class).
3492
+ # We want to use the cname as part of a C struct name, so we replace
3493
+ # periods with double underscores.
3494
+ as_name = '%s_%s' % (
3495
+ target_module_scope.next_id(Naming.closure_class_prefix),
3496
+ node.entry.cname.replace('.','__'))
3497
+ as_name = EncodedString(as_name)
3498
+
3499
+ entry = target_module_scope.declare_c_class(
3500
+ name=as_name, pos=node.pos, defining=True,
3501
+ implementing=True)
3502
+ entry.type.is_final_type = True
3503
+
3504
+ func_scope.scope_class = entry
3505
+ class_scope = entry.type.scope
3506
+ class_scope.is_internal = True
3507
+ class_scope.is_closure_class_scope = True
3508
+ if node.is_async_def or node.is_generator:
3509
+ # Generators need their closure intact during cleanup as they resume to handle GeneratorExit
3510
+ class_scope.directives['no_gc_clear'] = True
3511
+ if Options.closure_freelist_size:
3512
+ class_scope.directives['freelist'] = Options.closure_freelist_size
3513
+
3514
+ if from_closure:
3515
+ assert cscope.is_closure_scope
3516
+ class_scope.declare_var(pos=node.pos,
3517
+ name=Naming.outer_scope_cname,
3518
+ cname=Naming.outer_scope_cname,
3519
+ type=cscope.scope_class.type,
3520
+ is_cdef=True)
3521
+ node.needs_outer_scope = True
3522
+ for name, entry in in_closure:
3523
+ closure_entry = class_scope.declare_var(
3524
+ pos=entry.pos,
3525
+ name=entry.name if not entry.in_subscope else None,
3526
+ cname=entry.cname,
3527
+ type=entry.type,
3528
+ is_cdef=True)
3529
+ if entry.is_declared_generic:
3530
+ closure_entry.is_declared_generic = 1
3531
+ node.needs_closure = True
3532
+ # Do it here because other classes are already checked
3533
+ target_module_scope.check_c_class(func_scope.scope_class)
3534
+
3535
+ def visit_LambdaNode(self, node):
3536
+ if not isinstance(node.def_node, Nodes.DefNode):
3537
+ # fused function, an error has been previously issued
3538
+ return node
3539
+
3540
+ was_in_lambda = self.in_lambda
3541
+ self.in_lambda = True
3542
+ self.create_class_from_scope(node.def_node, self.module_scope, node)
3543
+ self.visitchildren(node)
3544
+ self.in_lambda = was_in_lambda
3545
+ return node
3546
+
3547
+ def visit_FuncDefNode(self, node):
3548
+ if self.in_lambda:
3549
+ self.visitchildren(node)
3550
+ return node
3551
+ if node.needs_closure or self.path:
3552
+ self.create_class_from_scope(node, self.module_scope)
3553
+ self.path.append(node)
3554
+ self.visitchildren(node)
3555
+ self.path.pop()
3556
+ return node
3557
+
3558
+ def visit_GeneratorBodyDefNode(self, node):
3559
+ self.visitchildren(node)
3560
+ return node
3561
+
3562
+ def visit_CFuncDefNode(self, node):
3563
+ if not node.overridable:
3564
+ return self.visit_FuncDefNode(node)
3565
+ else:
3566
+ self.visitchildren(node)
3567
+ return node
3568
+
3569
+ def visit_GeneratorExpressionNode(self, node):
3570
+ node = _HandleGeneratorArguments()(node)
3571
+ return self.visit_LambdaNode(node)
3572
+
3573
+
3574
+ class InjectGilHandling(VisitorTransform, SkipDeclarations):
3575
+ """
3576
+ Allow certain Python operations inside of nogil blocks by implicitly acquiring the GIL.
3577
+
3578
+ Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get
3579
+ set up, parallel sections know that the GIL is acquired inside of them, etc.
3580
+ """
3581
+ nogil = False
3582
+
3583
+ # special node handling
3584
+
3585
+ def _inject_gil_in_nogil(self, node):
3586
+ """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block."""
3587
+ if self.nogil:
3588
+ node = Nodes.GILStatNode(node.pos, state='gil', body=node)
3589
+ return node
3590
+
3591
+ visit_RaiseStatNode = _inject_gil_in_nogil
3592
+ visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function
3593
+
3594
+ # further candidates:
3595
+ # def visit_ReraiseStatNode(self, node):
3596
+
3597
+ # nogil tracking
3598
+
3599
+ def visit_GILStatNode(self, node):
3600
+ was_nogil = self.nogil
3601
+ self.nogil = (node.state == 'nogil')
3602
+ self.visitchildren(node)
3603
+ self.nogil = was_nogil
3604
+ return node
3605
+
3606
+ def visit_CFuncDefNode(self, node):
3607
+ was_nogil = self.nogil
3608
+ if isinstance(node.declarator, Nodes.CFuncDeclaratorNode):
3609
+ self.nogil = node.declarator.nogil and not node.declarator.with_gil
3610
+ self.visitchildren(node)
3611
+ self.nogil = was_nogil
3612
+ return node
3613
+
3614
+ def visit_ParallelRangeNode(self, node):
3615
+ was_nogil = self.nogil
3616
+ self.nogil = node.nogil
3617
+ self.visitchildren(node)
3618
+ self.nogil = was_nogil
3619
+ return node
3620
+
3621
+ def visit_ExprNode(self, node):
3622
+ # No special GIL handling inside of expressions for now.
3623
+ return node
3624
+
3625
+ visit_Node = VisitorTransform.recurse_to_children
3626
+
3627
+
3628
+ class GilCheck(VisitorTransform):
3629
+ """
3630
+ Call `node.gil_check(env)` on each node to make sure we hold the
3631
+ GIL when we need it. Raise an error when on Python operations
3632
+ inside a `nogil` environment.
3633
+
3634
+ Additionally, raise exceptions for closely nested with gil or with nogil
3635
+ statements. The latter would abort Python.
3636
+ """
3637
+
3638
+ def __call__(self, root):
3639
+ self.env_stack = [root.scope]
3640
+ self.nogil = False
3641
+
3642
+ # True for 'cdef func() nogil:' functions, as the GIL may be held while
3643
+ # calling this function (thus contained 'nogil' blocks may be valid).
3644
+ self.nogil_declarator_only = False
3645
+
3646
+ self.current_gilstat_node_knows_gil_state = False
3647
+ return super().__call__(root)
3648
+
3649
+ def _visit_scoped_children(self, node, gil_state):
3650
+ was_nogil = self.nogil
3651
+ outer_attrs = node.outer_attrs
3652
+ if outer_attrs and len(self.env_stack) > 1:
3653
+ self.nogil = self.env_stack[-2].nogil
3654
+ self.visitchildren(node, outer_attrs)
3655
+
3656
+ self.nogil = gil_state
3657
+ self.visitchildren(node, attrs=None, exclude=outer_attrs)
3658
+ self.nogil = was_nogil
3659
+
3660
+ def visit_FuncDefNode(self, node):
3661
+ self.env_stack.append(node.local_scope)
3662
+ inner_nogil = node.local_scope.nogil
3663
+
3664
+ nogil_declarator_only = self.nogil_declarator_only
3665
+ if inner_nogil:
3666
+ self.nogil_declarator_only = True
3667
+
3668
+ if inner_nogil and node.nogil_check:
3669
+ node.nogil_check(node.local_scope)
3670
+
3671
+ self._visit_scoped_children(node, inner_nogil)
3672
+
3673
+ # FuncDefNodes can be nested, because a cpdef function contains a def function
3674
+ # inside it. Therefore restore to previous state
3675
+ self.nogil_declarator_only = nogil_declarator_only
3676
+
3677
+ self.env_stack.pop()
3678
+ return node
3679
+
3680
+ def visit_GILStatNode(self, node):
3681
+ if node.condition is not None:
3682
+ error(node.condition.pos,
3683
+ "Non-constant condition in a "
3684
+ "`with %s(<condition>)` statement" % node.state)
3685
+ return node
3686
+
3687
+ if self.nogil and node.nogil_check:
3688
+ node.nogil_check()
3689
+
3690
+ was_nogil = self.nogil
3691
+ is_nogil = (node.state == 'nogil')
3692
+
3693
+ if was_nogil == is_nogil and not self.nogil_declarator_only:
3694
+ if not was_nogil:
3695
+ error(node.pos, "Trying to acquire the GIL while it is "
3696
+ "already held.")
3697
+ else:
3698
+ error(node.pos, "Trying to release the GIL while it was "
3699
+ "previously released.")
3700
+ if self.nogil_declarator_only:
3701
+ node.scope_gil_state_known = False
3702
+
3703
+ if isinstance(node.finally_clause, Nodes.StatListNode):
3704
+ # The finally clause of the GILStatNode is a GILExitNode,
3705
+ # which is wrapped in a StatListNode. Just unpack that.
3706
+ node.finally_clause, = node.finally_clause.stats
3707
+
3708
+ nogil_declarator_only = self.nogil_declarator_only
3709
+ self.nogil_declarator_only = False
3710
+ current_gilstat_node_knows_gil_state = self.current_gilstat_node_knows_gil_state
3711
+ self.current_gilstat_node_knows_gil_state = node.scope_gil_state_known
3712
+ self._visit_scoped_children(node, is_nogil)
3713
+ self.nogil_declarator_only = nogil_declarator_only
3714
+ self.current_gilstat_node_knows_gil_state = current_gilstat_node_knows_gil_state
3715
+ return node
3716
+
3717
+ def visit_ParallelRangeNode(self, node):
3718
+ if node.nogil or self.nogil_declarator_only:
3719
+ node_was_nogil, node.nogil = node.nogil, False
3720
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
3721
+ if not node_was_nogil and self.nogil_declarator_only:
3722
+ # We're in a "nogil" function, but that doesn't prove we
3723
+ # didn't have the gil
3724
+ node.scope_gil_state_known = False
3725
+ return self.visit_GILStatNode(node)
3726
+
3727
+ if not self.nogil:
3728
+ error(node.pos, "prange() can only be used without the GIL")
3729
+ # Forget about any GIL-related errors that may occur in the body
3730
+ return None
3731
+
3732
+ node.nogil_check(self.env_stack[-1])
3733
+ self.visitchildren(node)
3734
+ return node
3735
+
3736
+ def visit_ParallelWithBlockNode(self, node):
3737
+ if not self.nogil:
3738
+ error(node.pos, "The parallel section may only be used without "
3739
+ "the GIL")
3740
+ return None
3741
+ if self.nogil_declarator_only:
3742
+ # We're in a "nogil" function but that doesn't prove we didn't
3743
+ # have the gil, so release it
3744
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
3745
+ node.scope_gil_state_known = False
3746
+ return self.visit_GILStatNode(node)
3747
+
3748
+ if node.nogil_check:
3749
+ # It does not currently implement this, but test for it anyway to
3750
+ # avoid potential future surprises
3751
+ node.nogil_check(self.env_stack[-1])
3752
+
3753
+ self.visitchildren(node)
3754
+ return node
3755
+
3756
+ def visit_TryFinallyStatNode(self, node):
3757
+ """
3758
+ Take care of try/finally statements in nogil code sections.
3759
+ """
3760
+ if not self.nogil or isinstance(node, Nodes.GILStatNode):
3761
+ return self.visit_Node(node)
3762
+
3763
+ node.nogil_check = None
3764
+ node.is_try_finally_in_nogil = True
3765
+ self.visitchildren(node)
3766
+ return node
3767
+
3768
+ def visit_GILExitNode(self, node):
3769
+ if not self.current_gilstat_node_knows_gil_state:
3770
+ node.scope_gil_state_known = False
3771
+ self.visitchildren(node)
3772
+ return node
3773
+
3774
+ def visit_Node(self, node):
3775
+ if self.env_stack and self.nogil and node.nogil_check:
3776
+ node.nogil_check(self.env_stack[-1])
3777
+ if node.outer_attrs:
3778
+ self._visit_scoped_children(node, self.nogil)
3779
+ else:
3780
+ self.visitchildren(node)
3781
+ if self.nogil:
3782
+ node.in_nogil_context = True
3783
+ return node
3784
+
3785
+
3786
+ class CoerceCppTemps(EnvTransform, SkipDeclarations):
3787
+ """
3788
+ For temporary expression that are implemented using std::optional it's necessary the temps are
3789
+ assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform
3790
+ inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be
3791
+ inserted into the tree)
3792
+
3793
+ TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_rhs() and ExprNode.lhs_rhs()???
3794
+ """
3795
+ def visit_ModuleNode(self, node):
3796
+ if self.current_env().cpp:
3797
+ # skipping this makes it essentially free for C files
3798
+ self.visitchildren(node)
3799
+ return node
3800
+
3801
+ def visit_ExprNode(self, node):
3802
+ self.visitchildren(node)
3803
+ if (self.current_env().directives['cpp_locals'] and
3804
+ node.is_temp and node.type.is_cpp_class and
3805
+ # Fake references are not replaced with "std::optional()".
3806
+ not node.type.is_fake_reference):
3807
+ node = ExprNodes.CppOptionalTempCoercion(node)
3808
+
3809
+ return node
3810
+
3811
+ def visit_ExprStatNode(self, node):
3812
+ # Deliberately skip `expr` in ExprStatNode - we don't need to access it.
3813
+ self.visitchildren(node.expr)
3814
+ return node
3815
+
3816
+
3817
+ class TransformBuiltinMethods(EnvTransform):
3818
+ """
3819
+ Replace Cython's own cython.* builtins by the corresponding tree nodes.
3820
+ Also handle some Python special builtin functions (e.g. super()/locals())
3821
+ that require introspection by the compiler.
3822
+ """
3823
+ def __init__(self, *args, **kwds):
3824
+ super().__init__(*args, **kwds)
3825
+ self.def_node_body_insertions = {}
3826
+
3827
+ def visit_SingleAssignmentNode(self, node):
3828
+ if node.declaration_only:
3829
+ return None
3830
+ else:
3831
+ self.visitchildren(node)
3832
+ return node
3833
+
3834
+ def visit_AttributeNode(self, node):
3835
+ self.visitchildren(node)
3836
+ return self.visit_cython_attribute(node)
3837
+
3838
+ def visit_NameNode(self, node):
3839
+ if node.name == u'__class__':
3840
+ lenv = self.current_env()
3841
+ entry = lenv.lookup_here(u'__class__')
3842
+ if not entry:
3843
+ node = self._inject_class(node)
3844
+ return self.visit_cython_attribute(node)
3845
+
3846
+ def visit_cython_attribute(self, node):
3847
+ attribute = node.as_cython_attribute()
3848
+ if attribute:
3849
+ if attribute == '__version__':
3850
+ from .. import __version__ as version
3851
+ node = ExprNodes.UnicodeNode(node.pos, value=EncodedString(version))
3852
+ elif attribute == 'NULL':
3853
+ node = ExprNodes.NullNode(node.pos)
3854
+ elif attribute in ('set', 'frozenset', 'staticmethod'):
3855
+ node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
3856
+ entry=self.current_env().builtin_scope().lookup_here(attribute))
3857
+ elif PyrexTypes.parse_basic_type(attribute):
3858
+ pass
3859
+ elif self.context.cython_scope.lookup_qualified_name(attribute):
3860
+ pass
3861
+ else:
3862
+ error(node.pos, "'%s' not a valid cython attribute or is being used incorrectly" % attribute)
3863
+ return node
3864
+
3865
+ def visit_ExecStatNode(self, node):
3866
+ lenv = self.current_env()
3867
+ self.visitchildren(node)
3868
+ if len(node.args) == 1:
3869
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
3870
+ if not lenv.is_module_scope:
3871
+ node.args.append(
3872
+ ExprNodes.LocalsExprNode(
3873
+ node.pos, self.current_scope_node(), lenv))
3874
+ return node
3875
+
3876
+ def _inject_locals(self, node, func_name):
3877
+ # locals()/dir()/vars() builtins
3878
+ lenv = self.current_env()
3879
+ entry = lenv.lookup_here(func_name)
3880
+ if entry:
3881
+ # not the builtin
3882
+ return node
3883
+ pos = node.pos
3884
+ if func_name in ('locals', 'vars'):
3885
+ if func_name == 'locals' and len(node.args) > 0:
3886
+ error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
3887
+ % len(node.args))
3888
+ return node
3889
+ elif func_name == 'vars':
3890
+ if len(node.args) > 1:
3891
+ error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
3892
+ % len(node.args))
3893
+ if len(node.args) > 0:
3894
+ return node # nothing to do
3895
+ return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
3896
+ else: # dir()
3897
+ if len(node.args) > 1:
3898
+ error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
3899
+ % len(node.args))
3900
+ if len(node.args) > 0:
3901
+ # optimised in Builtin.py
3902
+ return node
3903
+ if lenv.is_py_class_scope or lenv.is_module_scope:
3904
+ if lenv.is_py_class_scope:
3905
+ pyclass = self.current_scope_node()
3906
+ locals_dict = ExprNodes.CloneNode(pyclass.dict)
3907
+ else:
3908
+ locals_dict = ExprNodes.GlobalsExprNode(pos)
3909
+ return ExprNodes.SortedDictKeysNode(locals_dict)
3910
+ local_names = sorted(var.name for var in lenv.entries.values() if var.name)
3911
+ items = [ExprNodes.IdentifierStringNode(pos, value=var)
3912
+ for var in local_names]
3913
+ return ExprNodes.ListNode(pos, args=items)
3914
+
3915
+ def visit_PrimaryCmpNode(self, node):
3916
+ # special case: for in/not-in test, we do not need to sort locals()
3917
+ self.visitchildren(node)
3918
+ if node.operator in 'not_in': # in/not_in
3919
+ if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
3920
+ arg = node.operand2.arg
3921
+ if isinstance(arg, ExprNodes.NoneCheckNode):
3922
+ arg = arg.arg
3923
+ node.operand2 = arg
3924
+ return node
3925
+
3926
+ def visit_CascadedCmpNode(self, node):
3927
+ return self.visit_PrimaryCmpNode(node)
3928
+
3929
+ def _inject_eval(self, node, func_name):
3930
+ lenv = self.current_env()
3931
+ entry = lenv.lookup(func_name)
3932
+ if len(node.args) != 1 or (entry and not entry.is_builtin):
3933
+ return node
3934
+ # Inject globals and locals
3935
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
3936
+ if not lenv.is_module_scope:
3937
+ node.args.append(
3938
+ ExprNodes.LocalsExprNode(
3939
+ node.pos, self.current_scope_node(), lenv))
3940
+ return node
3941
+
3942
+ def _inject_class(self, node):
3943
+ # bare __class__ reference inside function
3944
+ current_def_node = self.current_scope_node()
3945
+
3946
+ if not isinstance(current_def_node, Nodes.FuncDefNode):
3947
+ return node
3948
+
3949
+ # Go up the stack, find the first class node and its direct method (i.e. function).
3950
+ fdef_node = class_node = generator_node = None
3951
+ for stack_node, stack_scope in reversed(self.env_stack):
3952
+ if isinstance(stack_node, Nodes.ClassDefNode):
3953
+ class_node = stack_node
3954
+ class_scope = stack_scope
3955
+ break
3956
+ elif isinstance(stack_node, Nodes.GeneratorDefNode):
3957
+ generator_node = stack_node
3958
+ fdef_node = stack_node.gbody
3959
+ fdef_scope = stack_scope
3960
+ elif isinstance(stack_node, Nodes.FuncDefNode):
3961
+ fdef_node = stack_node
3962
+ fdef_scope = stack_scope
3963
+
3964
+ if not fdef_node or not class_node:
3965
+ # failed to find a class or function
3966
+ return node
3967
+
3968
+ # now we arrange to inject:
3969
+ # __class__ = ... at the start of the def_node body
3970
+ # The advantage of doing it like this is that it automatically appears in locals()
3971
+ # and it can be captured by inner functions
3972
+ if fdef_node not in self.def_node_body_insertions:
3973
+ pos = fdef_node.body.pos
3974
+ if class_scope.is_c_class_scope:
3975
+ # c-classes can be resolved at compile-time, so they have a simpler
3976
+ # implementation
3977
+ rhs = ExprNodes.NameNode(
3978
+ pos, name=class_node.scope.name,
3979
+ entry=class_node.entry)
3980
+ elif class_scope.is_py_class_scope:
3981
+ rhs = ExprNodes.ClassCellNode(pos, is_generator=generator_node is not None)
3982
+ if generator_node:
3983
+ generator_node.requires_classobj = True
3984
+ else:
3985
+ fdef_node.requires_classobj = True
3986
+ class_node.class_cell.is_active = True
3987
+ else:
3988
+ return node # should never happen
3989
+
3990
+ assign_node = Nodes.SingleAssignmentNode(pos,
3991
+ lhs=ExprNodes.NameNode(pos, name=EncodedString("__class__")),
3992
+ rhs=rhs)
3993
+
3994
+ assign_node.analyse_declarations(fdef_scope)
3995
+
3996
+ assert fdef_node not in self.def_node_body_insertions
3997
+ self.def_node_body_insertions[fdef_node] = assign_node
3998
+
3999
+ return node
4000
+
4001
+ def _inject_super(self, node, func_name):
4002
+ lenv = self.current_env()
4003
+ entry = lenv.lookup_here(func_name)
4004
+ if entry or node.args:
4005
+ return node
4006
+ # Inject no-args super
4007
+ def_node = self.current_scope_node()
4008
+ if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2:
4009
+ return node
4010
+ class_node, class_scope = self.env_stack[-2]
4011
+ if class_scope.is_py_class_scope:
4012
+ def_node.requires_classobj = True
4013
+ class_node.class_cell.is_active = True
4014
+ node.args = [
4015
+ ExprNodes.ClassCellNode(
4016
+ node.pos, is_generator=def_node.is_generator),
4017
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4018
+ ]
4019
+ elif class_scope.is_c_class_scope:
4020
+ node.args = [
4021
+ ExprNodes.NameNode(
4022
+ node.pos, name=class_node.scope.name,
4023
+ entry=class_node.entry),
4024
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4025
+ ]
4026
+ return node
4027
+
4028
+ def _do_body_insertion(self, node):
4029
+ body_insertion = self.def_node_body_insertions.pop(node, None)
4030
+ if body_insertion:
4031
+ if isinstance(node.body, Nodes.StatListNode):
4032
+ node.body.stats.insert(0, body_insertion)
4033
+ else:
4034
+ node.body = Nodes.StatListNode(node.body.pos,
4035
+ stats=[body_insertion, node.body])
4036
+
4037
+ def visit_FuncDefNode(self, node):
4038
+ node = super().visit_FuncDefNode(node)
4039
+ self._do_body_insertion(node)
4040
+ return node
4041
+
4042
+ def visit_GeneratorBodyDefNode(self, node):
4043
+ node = super().visit_GeneratorBodyDefNode(node)
4044
+ self._do_body_insertion(node)
4045
+ return node
4046
+
4047
+ def visit_SimpleCallNode(self, node):
4048
+ # cython.foo
4049
+ function = node.function.as_cython_attribute()
4050
+ if function:
4051
+ if function in InterpretCompilerDirectives.unop_method_nodes:
4052
+ if len(node.args) != 1:
4053
+ error(node.function.pos, "%s() takes exactly one argument" % function)
4054
+ else:
4055
+ node = InterpretCompilerDirectives.unop_method_nodes[function](
4056
+ node.function.pos, operand=node.args[0])
4057
+ elif function in InterpretCompilerDirectives.binop_method_nodes:
4058
+ if len(node.args) != 2:
4059
+ error(node.function.pos, "%s() takes exactly two arguments" % function)
4060
+ else:
4061
+ node = InterpretCompilerDirectives.binop_method_nodes[function](
4062
+ node.function.pos, operand1=node.args[0], operand2=node.args[1])
4063
+ elif function == 'cast':
4064
+ if len(node.args) != 2:
4065
+ error(node.function.pos,
4066
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4067
+ else:
4068
+ type = node.args[0].analyse_as_type(self.current_env())
4069
+ if type:
4070
+ node = ExprNodes.TypecastNode(
4071
+ node.function.pos, type=type, operand=node.args[1], typecheck=False)
4072
+ else:
4073
+ error(node.args[0].pos, "Not a type")
4074
+ elif function == 'sizeof':
4075
+ if len(node.args) != 1:
4076
+ error(node.function.pos, "sizeof() takes exactly one argument")
4077
+ else:
4078
+ type = node.args[0].analyse_as_type(self.current_env())
4079
+ if type:
4080
+ node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
4081
+ else:
4082
+ node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
4083
+ elif function == 'cmod':
4084
+ if len(node.args) != 2:
4085
+ error(node.function.pos, "cmod() takes exactly two arguments")
4086
+ else:
4087
+ node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
4088
+ node.cdivision = True
4089
+ elif function == 'cdiv':
4090
+ if len(node.args) != 2:
4091
+ error(node.function.pos, "cdiv() takes exactly two arguments")
4092
+ else:
4093
+ node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
4094
+ node.cdivision = True
4095
+ elif function == 'set':
4096
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
4097
+ elif function == 'staticmethod':
4098
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod'))
4099
+ elif self.context.cython_scope.lookup_qualified_name(function):
4100
+ pass
4101
+ else:
4102
+ error(node.function.pos,
4103
+ "'%s' not a valid cython language construct" % function)
4104
+
4105
+ self.visitchildren(node)
4106
+
4107
+ if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
4108
+ func_name = node.function.name
4109
+ if func_name in ('dir', 'locals', 'vars'):
4110
+ return self._inject_locals(node, func_name)
4111
+ if func_name == 'eval':
4112
+ return self._inject_eval(node, func_name)
4113
+ if func_name == 'super':
4114
+ return self._inject_super(node, func_name)
4115
+ return node
4116
+
4117
+ def visit_GeneralCallNode(self, node):
4118
+ function = node.function.as_cython_attribute()
4119
+ if function == 'cast':
4120
+ # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args
4121
+ args = node.positional_args.args
4122
+ kwargs = node.keyword_args.compile_time_value(None)
4123
+ if (len(args) != 2 or len(kwargs) > 1 or
4124
+ (len(kwargs) == 1 and 'typecheck' not in kwargs)):
4125
+ error(node.function.pos,
4126
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4127
+ else:
4128
+ type = args[0].analyse_as_type(self.current_env())
4129
+ if type:
4130
+ typecheck = kwargs.get('typecheck', False)
4131
+ node = ExprNodes.TypecastNode(
4132
+ node.function.pos, type=type, operand=args[1], typecheck=typecheck)
4133
+ else:
4134
+ error(args[0].pos, "Not a type")
4135
+
4136
+ self.visitchildren(node)
4137
+ return node
4138
+
4139
+
4140
+ class ReplaceFusedTypeChecks(VisitorTransform):
4141
+ """
4142
+ This is not a transform in the pipeline. It is invoked on the specific
4143
+ versions of a cdef function with fused argument types. It filters out any
4144
+ type branches that don't match. e.g.
4145
+
4146
+ if fused_t is mytype:
4147
+ ...
4148
+ elif fused_t in other_fused_type:
4149
+ ...
4150
+ """
4151
+ def __init__(self, local_scope):
4152
+ super().__init__()
4153
+ self.local_scope = local_scope
4154
+ # defer the import until now to avoid circular import time dependencies
4155
+ from .Optimize import ConstantFolding
4156
+ self.transform = ConstantFolding(reevaluate=True)
4157
+
4158
+ def visit_IfStatNode(self, node):
4159
+ """
4160
+ Filters out any if clauses with false compile time type check
4161
+ expression.
4162
+ """
4163
+ self.visitchildren(node)
4164
+ return self.transform(node)
4165
+
4166
+ def visit_GILStatNode(self, node):
4167
+ """
4168
+ Fold constant condition of GILStatNode.
4169
+ """
4170
+ self.visitchildren(node)
4171
+ return self.transform(node)
4172
+
4173
+ def visit_PrimaryCmpNode(self, node):
4174
+ with Errors.local_errors(ignore=True):
4175
+ type1 = node.operand1.analyse_as_type(self.local_scope)
4176
+ type2 = node.operand2.analyse_as_type(self.local_scope)
4177
+
4178
+ if type1 and type2:
4179
+ false_node = ExprNodes.BoolNode(node.pos, value=False)
4180
+ true_node = ExprNodes.BoolNode(node.pos, value=True)
4181
+
4182
+ type1 = self.specialize_type(type1, node.operand1.pos)
4183
+ op = node.operator
4184
+
4185
+ if op in ('is', 'is_not', '==', '!='):
4186
+ type2 = self.specialize_type(type2, node.operand2.pos)
4187
+
4188
+ is_same = type1.same_as(type2)
4189
+ eq = op in ('is', '==')
4190
+
4191
+ if (is_same and eq) or (not is_same and not eq):
4192
+ return true_node
4193
+
4194
+ elif op in ('in', 'not_in'):
4195
+ # We have to do an instance check directly, as operand2
4196
+ # needs to be a fused type and not a type with a subtype
4197
+ # that is fused. First unpack the typedef
4198
+ if isinstance(type2, PyrexTypes.CTypedefType):
4199
+ type2 = type2.typedef_base_type
4200
+
4201
+ if type1.is_fused:
4202
+ error(node.operand1.pos, "Type is fused")
4203
+ elif not type2.is_fused:
4204
+ error(node.operand2.pos,
4205
+ "Can only use 'in' or 'not in' on a fused type")
4206
+ else:
4207
+ types = PyrexTypes.get_specialized_types(type2)
4208
+
4209
+ for specialized_type in types:
4210
+ if type1.same_as(specialized_type):
4211
+ if op == 'in':
4212
+ return true_node
4213
+ else:
4214
+ return false_node
4215
+
4216
+ if op == 'not_in':
4217
+ return true_node
4218
+
4219
+ return false_node
4220
+
4221
+ return node
4222
+
4223
+ def specialize_type(self, type, pos):
4224
+ try:
4225
+ return type.specialize(self.local_scope.fused_to_specific)
4226
+ except KeyError:
4227
+ error(pos, "Type is not specific")
4228
+ return type
4229
+
4230
+ def visit_Node(self, node):
4231
+ self.visitchildren(node)
4232
+ return node
4233
+
4234
+
4235
+ class DebugTransform(CythonTransform):
4236
+ """
4237
+ Write debug information for this Cython module.
4238
+ """
4239
+
4240
+ def __init__(self, context, options, result):
4241
+ super().__init__(context)
4242
+ self.visited = set()
4243
+ # our treebuilder and debug output writer
4244
+ # (see Cython.Debugger.debug_output.CythonDebugWriter)
4245
+ self.tb = self.context.gdb_debug_outputwriter
4246
+ #self.c_output_file = options.output_file
4247
+ self.c_output_file = result.c_file
4248
+
4249
+ # Closure support, basically treat nested functions as if the AST were
4250
+ # never nested
4251
+ self.nested_funcdefs = []
4252
+
4253
+ # tells visit_NameNode whether it should register step-into functions
4254
+ self.register_stepinto = False
4255
+
4256
+ def visit_ModuleNode(self, node):
4257
+ self.tb.module_name = node.full_module_name
4258
+ attrs = dict(
4259
+ module_name=node.full_module_name,
4260
+ filename=node.pos[0].filename,
4261
+ c_filename=self.c_output_file)
4262
+
4263
+ self.tb.start('Module', attrs)
4264
+
4265
+ # serialize functions
4266
+ self.tb.start('Functions')
4267
+ # First, serialize functions normally...
4268
+ self.visitchildren(node)
4269
+
4270
+ # ... then, serialize nested functions
4271
+ for nested_funcdef in self.nested_funcdefs:
4272
+ self.visit_FuncDefNode(nested_funcdef)
4273
+
4274
+ self.register_stepinto = True
4275
+ self.serialize_modulenode_as_function(node)
4276
+ self.register_stepinto = False
4277
+ self.tb.end('Functions')
4278
+
4279
+ # 2.3 compatibility. Serialize global variables
4280
+ self.tb.start('Globals')
4281
+ entries = {}
4282
+
4283
+ for k, v in node.scope.entries.items():
4284
+ if (v.qualified_name not in self.visited and not
4285
+ v.name.startswith('__pyx_') and not
4286
+ v.type.is_cfunction and not
4287
+ v.type.is_extension_type):
4288
+ entries[k]= v
4289
+
4290
+ self.serialize_local_variables(entries)
4291
+ self.tb.end('Globals')
4292
+ # self.tb.end('Module') # end Module after the line number mapping in
4293
+ # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
4294
+ return node
4295
+
4296
+ def visit_FuncDefNode(self, node):
4297
+ self.visited.add(node.local_scope.qualified_name)
4298
+
4299
+ if getattr(node, 'is_wrapper', False):
4300
+ return node
4301
+
4302
+ if self.register_stepinto:
4303
+ self.nested_funcdefs.append(node)
4304
+ return node
4305
+
4306
+ # node.entry.visibility = 'extern'
4307
+ if node.py_func is None:
4308
+ pf_cname = ''
4309
+ else:
4310
+ pf_cname = node.py_func.entry.func_cname
4311
+
4312
+ # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_*
4313
+ # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_*
4314
+ # In all cases, cname will be the name of the function containing the actual code
4315
+ cname = node.entry.pyfunc_cname or node.entry.func_cname
4316
+
4317
+ attrs = dict(
4318
+ name=node.entry.name or getattr(node, 'name', '<unknown>'),
4319
+ cname=cname,
4320
+ pf_cname=pf_cname,
4321
+ qualified_name=node.local_scope.qualified_name,
4322
+ lineno=str(node.pos[1]))
4323
+
4324
+ self.tb.start('Function', attrs=attrs)
4325
+
4326
+ self.tb.start('Locals')
4327
+ self.serialize_local_variables(node.local_scope.entries)
4328
+ self.tb.end('Locals')
4329
+
4330
+ self.tb.start('Arguments')
4331
+ for arg in node.local_scope.arg_entries:
4332
+ self.tb.start(arg.name)
4333
+ self.tb.end(arg.name)
4334
+ self.tb.end('Arguments')
4335
+
4336
+ self.tb.start('StepIntoFunctions')
4337
+ self.register_stepinto = True
4338
+ self.visitchildren(node)
4339
+ self.register_stepinto = False
4340
+ self.tb.end('StepIntoFunctions')
4341
+ self.tb.end('Function')
4342
+
4343
+ return node
4344
+
4345
+ def visit_NameNode(self, node):
4346
+ if (self.register_stepinto and
4347
+ node.type is not None and
4348
+ node.type.is_cfunction and
4349
+ getattr(node, 'is_called', False) and
4350
+ node.entry.func_cname is not None):
4351
+ # don't check node.entry.in_cinclude, as 'cdef extern: ...'
4352
+ # declared functions are not 'in_cinclude'.
4353
+ # This means we will list called 'cdef' functions as
4354
+ # "step into functions", but this is not an issue as they will be
4355
+ # recognized as Cython functions anyway.
4356
+ attrs = dict(name=node.entry.func_cname)
4357
+ self.tb.start('StepIntoFunction', attrs=attrs)
4358
+ self.tb.end('StepIntoFunction')
4359
+
4360
+ self.visitchildren(node)
4361
+ return node
4362
+
4363
+ def serialize_modulenode_as_function(self, node):
4364
+ """
4365
+ Serialize the module-level code as a function so the debugger will know
4366
+ it's a "relevant frame" and it will know where to set the breakpoint
4367
+ for 'break modulename'.
4368
+ """
4369
+ self._serialize_modulenode_as_function(node, dict(
4370
+ name=node.full_module_name.rpartition('.')[-1],
4371
+ cname=node.module_init_func_cname(),
4372
+ pf_cname='',
4373
+ # Ignore the qualified_name, breakpoints should be set using
4374
+ # `cy break modulename:lineno` for module-level breakpoints.
4375
+ qualified_name='',
4376
+ lineno='1',
4377
+ is_initmodule_function="True",
4378
+ ))
4379
+
4380
+ def _serialize_modulenode_as_function(self, node, attrs):
4381
+ self.tb.start('Function', attrs=attrs)
4382
+
4383
+ self.tb.start('Locals')
4384
+ self.serialize_local_variables(node.scope.entries)
4385
+ self.tb.end('Locals')
4386
+
4387
+ self.tb.start('Arguments')
4388
+ self.tb.end('Arguments')
4389
+
4390
+ self.tb.start('StepIntoFunctions')
4391
+ self.register_stepinto = True
4392
+ self.visitchildren(node)
4393
+ self.register_stepinto = False
4394
+ self.tb.end('StepIntoFunctions')
4395
+
4396
+ self.tb.end('Function')
4397
+
4398
+ def serialize_local_variables(self, entries):
4399
+ for entry in entries.values():
4400
+ if not entry.cname:
4401
+ # not a local variable
4402
+ continue
4403
+ if entry.type.is_pyobject:
4404
+ vartype = 'PythonObject'
4405
+ else:
4406
+ vartype = 'CObject'
4407
+
4408
+ if entry.from_closure:
4409
+ # We're dealing with a closure where a variable from an outer
4410
+ # scope is accessed, get it from the scope object.
4411
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4412
+ entry.outer_entry.cname)
4413
+
4414
+ qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
4415
+ entry.scope.name,
4416
+ entry.name)
4417
+ elif entry.in_closure:
4418
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4419
+ entry.cname)
4420
+ qname = entry.qualified_name
4421
+ else:
4422
+ cname = entry.cname
4423
+ qname = entry.qualified_name
4424
+
4425
+ if not entry.pos:
4426
+ # this happens for variables that are not in the user's code,
4427
+ # e.g. for the global __builtins__, __doc__, etc. We can just
4428
+ # set the lineno to 0 for those.
4429
+ lineno = '0'
4430
+ else:
4431
+ lineno = str(entry.pos[1])
4432
+
4433
+ attrs = dict(
4434
+ name=entry.name,
4435
+ cname=cname,
4436
+ qualified_name=qname,
4437
+ type=vartype,
4438
+ lineno=lineno)
4439
+
4440
+ self.tb.start('LocalVar', attrs)
4441
+ self.tb.end('LocalVar')