Cython 3.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (316) hide show
  1. Cython/Build/BuildExecutable.py +169 -0
  2. Cython/Build/Cache.py +199 -0
  3. Cython/Build/Cythonize.py +323 -0
  4. Cython/Build/Dependencies.py +1306 -0
  5. Cython/Build/Distutils.py +1 -0
  6. Cython/Build/Inline.py +463 -0
  7. Cython/Build/IpythonMagic.py +560 -0
  8. Cython/Build/SharedModule.py +76 -0
  9. Cython/Build/Tests/TestCyCache.py +194 -0
  10. Cython/Build/Tests/TestCythonizeArgsParser.py +481 -0
  11. Cython/Build/Tests/TestDependencies.py +133 -0
  12. Cython/Build/Tests/TestInline.py +177 -0
  13. Cython/Build/Tests/TestIpythonMagic.py +287 -0
  14. Cython/Build/Tests/TestRecythonize.py +212 -0
  15. Cython/Build/Tests/TestStripLiterals.py +155 -0
  16. Cython/Build/Tests/__init__.py +1 -0
  17. Cython/Build/__init__.py +8 -0
  18. Cython/CodeWriter.py +811 -0
  19. Cython/Compiler/AnalysedTreeTransforms.py +97 -0
  20. Cython/Compiler/Annotate.py +326 -0
  21. Cython/Compiler/AutoDocTransforms.py +320 -0
  22. Cython/Compiler/Buffer.py +680 -0
  23. Cython/Compiler/Builtin.py +934 -0
  24. Cython/Compiler/CmdLine.py +259 -0
  25. Cython/Compiler/Code.pxd +148 -0
  26. Cython/Compiler/Code.py +3375 -0
  27. Cython/Compiler/CodeGeneration.py +33 -0
  28. Cython/Compiler/CythonScope.py +187 -0
  29. Cython/Compiler/Dataclass.py +868 -0
  30. Cython/Compiler/DebugFlags.py +24 -0
  31. Cython/Compiler/Errors.py +295 -0
  32. Cython/Compiler/ExprNodes.py +15267 -0
  33. Cython/Compiler/FlowControl.pxd +97 -0
  34. Cython/Compiler/FlowControl.py +1455 -0
  35. Cython/Compiler/FusedNode.py +1002 -0
  36. Cython/Compiler/Future.py +16 -0
  37. Cython/Compiler/Interpreter.py +57 -0
  38. Cython/Compiler/Lexicon.py +340 -0
  39. Cython/Compiler/LineTable.py +114 -0
  40. Cython/Compiler/Main.py +853 -0
  41. Cython/Compiler/MatchCaseNodes.py +259 -0
  42. Cython/Compiler/MemoryView.py +922 -0
  43. Cython/Compiler/ModuleNode.py +4024 -0
  44. Cython/Compiler/Naming.py +374 -0
  45. Cython/Compiler/Nodes.py +10826 -0
  46. Cython/Compiler/Optimize.py +5256 -0
  47. Cython/Compiler/Options.py +835 -0
  48. Cython/Compiler/ParseTreeTransforms.pxd +77 -0
  49. Cython/Compiler/ParseTreeTransforms.py +4509 -0
  50. Cython/Compiler/Parsing.pxd +9 -0
  51. Cython/Compiler/Parsing.py +4789 -0
  52. Cython/Compiler/Pipeline.py +439 -0
  53. Cython/Compiler/PyrexTypes.py +5762 -0
  54. Cython/Compiler/Pythran.py +232 -0
  55. Cython/Compiler/Scanning.pxd +40 -0
  56. Cython/Compiler/Scanning.py +577 -0
  57. Cython/Compiler/StringEncoding.py +347 -0
  58. Cython/Compiler/Symtab.py +3080 -0
  59. Cython/Compiler/Tests/TestBuffer.py +105 -0
  60. Cython/Compiler/Tests/TestBuiltin.py +72 -0
  61. Cython/Compiler/Tests/TestCmdLine.py +586 -0
  62. Cython/Compiler/Tests/TestCode.py +86 -0
  63. Cython/Compiler/Tests/TestFlowControl.py +65 -0
  64. Cython/Compiler/Tests/TestGrammar.py +202 -0
  65. Cython/Compiler/Tests/TestMemView.py +71 -0
  66. Cython/Compiler/Tests/TestParseTreeTransforms.py +285 -0
  67. Cython/Compiler/Tests/TestScanning.py +134 -0
  68. Cython/Compiler/Tests/TestSignatureMatching.py +73 -0
  69. Cython/Compiler/Tests/TestStringEncoding.py +33 -0
  70. Cython/Compiler/Tests/TestTreeFragment.py +63 -0
  71. Cython/Compiler/Tests/TestTreePath.py +103 -0
  72. Cython/Compiler/Tests/TestTypes.py +75 -0
  73. Cython/Compiler/Tests/TestUtilityLoad.py +112 -0
  74. Cython/Compiler/Tests/TestVisitor.py +61 -0
  75. Cython/Compiler/Tests/Utils.py +36 -0
  76. Cython/Compiler/Tests/__init__.py +1 -0
  77. Cython/Compiler/TreeFragment.py +278 -0
  78. Cython/Compiler/TreePath.py +303 -0
  79. Cython/Compiler/TypeInference.py +584 -0
  80. Cython/Compiler/TypeSlots.py +1181 -0
  81. Cython/Compiler/UFuncs.py +311 -0
  82. Cython/Compiler/UtilNodes.py +389 -0
  83. Cython/Compiler/UtilityCode.py +344 -0
  84. Cython/Compiler/Version.py +8 -0
  85. Cython/Compiler/Visitor.pxd +53 -0
  86. Cython/Compiler/Visitor.py +861 -0
  87. Cython/Compiler/__init__.py +1 -0
  88. Cython/Coverage.py +448 -0
  89. Cython/Debugger/Cygdb.py +175 -0
  90. Cython/Debugger/DebugWriter.py +82 -0
  91. Cython/Debugger/Tests/TestLibCython.py +275 -0
  92. Cython/Debugger/Tests/__init__.py +1 -0
  93. Cython/Debugger/Tests/cfuncs.c +8 -0
  94. Cython/Debugger/Tests/codefile +49 -0
  95. Cython/Debugger/Tests/test_libcython_in_gdb.py +578 -0
  96. Cython/Debugger/Tests/test_libpython_in_gdb.py +90 -0
  97. Cython/Debugger/__init__.py +1 -0
  98. Cython/Debugger/libcython.py +1548 -0
  99. Cython/Debugger/libpython.py +2821 -0
  100. Cython/Debugging.py +20 -0
  101. Cython/Distutils/__init__.py +2 -0
  102. Cython/Distutils/build_ext.py +139 -0
  103. Cython/Distutils/extension.py +96 -0
  104. Cython/Distutils/old_build_ext.py +351 -0
  105. Cython/Includes/cpython/__init__.pxd +173 -0
  106. Cython/Includes/cpython/array.pxd +174 -0
  107. Cython/Includes/cpython/bool.pxd +37 -0
  108. Cython/Includes/cpython/buffer.pxd +112 -0
  109. Cython/Includes/cpython/bytearray.pxd +33 -0
  110. Cython/Includes/cpython/bytes.pxd +200 -0
  111. Cython/Includes/cpython/cellobject.pxd +35 -0
  112. Cython/Includes/cpython/ceval.pxd +8 -0
  113. Cython/Includes/cpython/codecs.pxd +121 -0
  114. Cython/Includes/cpython/complex.pxd +60 -0
  115. Cython/Includes/cpython/contextvars.pxd +145 -0
  116. Cython/Includes/cpython/conversion.pxd +36 -0
  117. Cython/Includes/cpython/datetime.pxd +395 -0
  118. Cython/Includes/cpython/descr.pxd +26 -0
  119. Cython/Includes/cpython/dict.pxd +187 -0
  120. Cython/Includes/cpython/exc.pxd +263 -0
  121. Cython/Includes/cpython/fileobject.pxd +57 -0
  122. Cython/Includes/cpython/float.pxd +47 -0
  123. Cython/Includes/cpython/function.pxd +65 -0
  124. Cython/Includes/cpython/genobject.pxd +25 -0
  125. Cython/Includes/cpython/getargs.pxd +12 -0
  126. Cython/Includes/cpython/instance.pxd +25 -0
  127. Cython/Includes/cpython/iterator.pxd +36 -0
  128. Cython/Includes/cpython/iterobject.pxd +24 -0
  129. Cython/Includes/cpython/list.pxd +92 -0
  130. Cython/Includes/cpython/long.pxd +149 -0
  131. Cython/Includes/cpython/longintrepr.pxd +14 -0
  132. Cython/Includes/cpython/mapping.pxd +63 -0
  133. Cython/Includes/cpython/marshal.pxd +66 -0
  134. Cython/Includes/cpython/mem.pxd +120 -0
  135. Cython/Includes/cpython/memoryview.pxd +50 -0
  136. Cython/Includes/cpython/method.pxd +49 -0
  137. Cython/Includes/cpython/module.pxd +208 -0
  138. Cython/Includes/cpython/number.pxd +258 -0
  139. Cython/Includes/cpython/object.pxd +433 -0
  140. Cython/Includes/cpython/pycapsule.pxd +143 -0
  141. Cython/Includes/cpython/pylifecycle.pxd +68 -0
  142. Cython/Includes/cpython/pyport.pxd +8 -0
  143. Cython/Includes/cpython/pystate.pxd +95 -0
  144. Cython/Includes/cpython/pythread.pxd +53 -0
  145. Cython/Includes/cpython/ref.pxd +67 -0
  146. Cython/Includes/cpython/sequence.pxd +134 -0
  147. Cython/Includes/cpython/set.pxd +119 -0
  148. Cython/Includes/cpython/slice.pxd +70 -0
  149. Cython/Includes/cpython/time.pxd +129 -0
  150. Cython/Includes/cpython/tuple.pxd +72 -0
  151. Cython/Includes/cpython/type.pxd +53 -0
  152. Cython/Includes/cpython/unicode.pxd +639 -0
  153. Cython/Includes/cpython/version.pxd +32 -0
  154. Cython/Includes/cpython/weakref.pxd +78 -0
  155. Cython/Includes/libc/__init__.pxd +1 -0
  156. Cython/Includes/libc/complex.pxd +35 -0
  157. Cython/Includes/libc/errno.pxd +127 -0
  158. Cython/Includes/libc/float.pxd +43 -0
  159. Cython/Includes/libc/limits.pxd +28 -0
  160. Cython/Includes/libc/locale.pxd +46 -0
  161. Cython/Includes/libc/math.pxd +209 -0
  162. Cython/Includes/libc/setjmp.pxd +10 -0
  163. Cython/Includes/libc/signal.pxd +64 -0
  164. Cython/Includes/libc/stddef.pxd +9 -0
  165. Cython/Includes/libc/stdint.pxd +105 -0
  166. Cython/Includes/libc/stdio.pxd +80 -0
  167. Cython/Includes/libc/stdlib.pxd +72 -0
  168. Cython/Includes/libc/string.pxd +50 -0
  169. Cython/Includes/libc/threads.pxd +84 -0
  170. Cython/Includes/libc/time.pxd +51 -0
  171. Cython/Includes/libcpp/__init__.pxd +4 -0
  172. Cython/Includes/libcpp/algorithm.pxd +320 -0
  173. Cython/Includes/libcpp/any.pxd +16 -0
  174. Cython/Includes/libcpp/atomic.pxd +59 -0
  175. Cython/Includes/libcpp/barrier.pxd +22 -0
  176. Cython/Includes/libcpp/bit.pxd +29 -0
  177. Cython/Includes/libcpp/cast.pxd +12 -0
  178. Cython/Includes/libcpp/cmath.pxd +518 -0
  179. Cython/Includes/libcpp/complex.pxd +106 -0
  180. Cython/Includes/libcpp/deque.pxd +165 -0
  181. Cython/Includes/libcpp/exception.pxd +86 -0
  182. Cython/Includes/libcpp/execution.pxd +15 -0
  183. Cython/Includes/libcpp/forward_list.pxd +63 -0
  184. Cython/Includes/libcpp/functional.pxd +26 -0
  185. Cython/Includes/libcpp/future.pxd +103 -0
  186. Cython/Includes/libcpp/iterator.pxd +34 -0
  187. Cython/Includes/libcpp/latch.pxd +17 -0
  188. Cython/Includes/libcpp/limits.pxd +61 -0
  189. Cython/Includes/libcpp/list.pxd +117 -0
  190. Cython/Includes/libcpp/map.pxd +252 -0
  191. Cython/Includes/libcpp/memory.pxd +115 -0
  192. Cython/Includes/libcpp/mutex.pxd +130 -0
  193. Cython/Includes/libcpp/numbers.pxd +15 -0
  194. Cython/Includes/libcpp/numeric.pxd +131 -0
  195. Cython/Includes/libcpp/optional.pxd +34 -0
  196. Cython/Includes/libcpp/pair.pxd +1 -0
  197. Cython/Includes/libcpp/queue.pxd +25 -0
  198. Cython/Includes/libcpp/random.pxd +166 -0
  199. Cython/Includes/libcpp/semaphore.pxd +44 -0
  200. Cython/Includes/libcpp/set.pxd +228 -0
  201. Cython/Includes/libcpp/shared_mutex.pxd +72 -0
  202. Cython/Includes/libcpp/span.pxd +87 -0
  203. Cython/Includes/libcpp/stack.pxd +11 -0
  204. Cython/Includes/libcpp/stop_token.pxd +105 -0
  205. Cython/Includes/libcpp/string.pxd +355 -0
  206. Cython/Includes/libcpp/string_view.pxd +181 -0
  207. Cython/Includes/libcpp/typeindex.pxd +15 -0
  208. Cython/Includes/libcpp/typeinfo.pxd +10 -0
  209. Cython/Includes/libcpp/unordered_map.pxd +193 -0
  210. Cython/Includes/libcpp/unordered_set.pxd +152 -0
  211. Cython/Includes/libcpp/utility.pxd +30 -0
  212. Cython/Includes/libcpp/vector.pxd +186 -0
  213. Cython/Includes/openmp.pxd +50 -0
  214. Cython/Includes/posix/__init__.pxd +1 -0
  215. Cython/Includes/posix/dlfcn.pxd +14 -0
  216. Cython/Includes/posix/fcntl.pxd +86 -0
  217. Cython/Includes/posix/ioctl.pxd +4 -0
  218. Cython/Includes/posix/mman.pxd +101 -0
  219. Cython/Includes/posix/resource.pxd +57 -0
  220. Cython/Includes/posix/select.pxd +21 -0
  221. Cython/Includes/posix/signal.pxd +73 -0
  222. Cython/Includes/posix/stat.pxd +98 -0
  223. Cython/Includes/posix/stdio.pxd +37 -0
  224. Cython/Includes/posix/stdlib.pxd +29 -0
  225. Cython/Includes/posix/strings.pxd +9 -0
  226. Cython/Includes/posix/time.pxd +71 -0
  227. Cython/Includes/posix/types.pxd +30 -0
  228. Cython/Includes/posix/uio.pxd +26 -0
  229. Cython/Includes/posix/unistd.pxd +271 -0
  230. Cython/Includes/posix/wait.pxd +38 -0
  231. Cython/Plex/Actions.pxd +24 -0
  232. Cython/Plex/Actions.py +119 -0
  233. Cython/Plex/DFA.pxd +14 -0
  234. Cython/Plex/DFA.py +164 -0
  235. Cython/Plex/Errors.py +48 -0
  236. Cython/Plex/Lexicons.py +178 -0
  237. Cython/Plex/Machines.pxd +36 -0
  238. Cython/Plex/Machines.py +238 -0
  239. Cython/Plex/Regexps.py +539 -0
  240. Cython/Plex/Scanners.pxd +47 -0
  241. Cython/Plex/Scanners.py +360 -0
  242. Cython/Plex/Transitions.pxd +14 -0
  243. Cython/Plex/Transitions.py +239 -0
  244. Cython/Plex/__init__.py +34 -0
  245. Cython/Runtime/__init__.py +1 -0
  246. Cython/Runtime/refnanny.pyx +237 -0
  247. Cython/Shadow.py +690 -0
  248. Cython/Shadow.pyi +521 -0
  249. Cython/StringIOTree.py +170 -0
  250. Cython/Tempita/__init__.py +4 -0
  251. Cython/Tempita/_looper.py +154 -0
  252. Cython/Tempita/_tempita.py +1091 -0
  253. Cython/TestUtils.py +410 -0
  254. Cython/Tests/TestCodeWriter.py +128 -0
  255. Cython/Tests/TestCythonUtils.py +202 -0
  256. Cython/Tests/TestJediTyper.py +223 -0
  257. Cython/Tests/TestShadow.py +114 -0
  258. Cython/Tests/TestStringIOTree.py +67 -0
  259. Cython/Tests/TestTestUtils.py +90 -0
  260. Cython/Tests/__init__.py +1 -0
  261. Cython/Tests/xmlrunner.py +390 -0
  262. Cython/Utility/AsyncGen.c +1002 -0
  263. Cython/Utility/Buffer.c +875 -0
  264. Cython/Utility/BufferFormatFromTypeInfo.pxd +2 -0
  265. Cython/Utility/Builtins.c +776 -0
  266. Cython/Utility/CConvert.pyx +134 -0
  267. Cython/Utility/CMath.c +104 -0
  268. Cython/Utility/CommonStructures.c +118 -0
  269. Cython/Utility/Complex.c +378 -0
  270. Cython/Utility/Coroutine.c +2206 -0
  271. Cython/Utility/CpdefEnums.pyx +103 -0
  272. Cython/Utility/CppConvert.pyx +279 -0
  273. Cython/Utility/CppSupport.cpp +143 -0
  274. Cython/Utility/CythonFunction.c +1794 -0
  275. Cython/Utility/Dataclasses.c +185 -0
  276. Cython/Utility/Dataclasses.py +112 -0
  277. Cython/Utility/Embed.c +125 -0
  278. Cython/Utility/Exceptions.c +1012 -0
  279. Cython/Utility/ExtensionTypes.c +809 -0
  280. Cython/Utility/FunctionArguments.c +965 -0
  281. Cython/Utility/ImportExport.c +987 -0
  282. Cython/Utility/Lock.c +136 -0
  283. Cython/Utility/MemoryView.pxd +187 -0
  284. Cython/Utility/MemoryView.pyx +1481 -0
  285. Cython/Utility/MemoryView_C.c +1046 -0
  286. Cython/Utility/ModuleSetupCode.c +3059 -0
  287. Cython/Utility/NumpyImportArray.c +46 -0
  288. Cython/Utility/ObjectHandling.c +3342 -0
  289. Cython/Utility/Optimize.c +1589 -0
  290. Cython/Utility/Overflow.c +404 -0
  291. Cython/Utility/Printing.c +86 -0
  292. Cython/Utility/Profile.c +709 -0
  293. Cython/Utility/StringTools.c +1259 -0
  294. Cython/Utility/TestCyUtilityLoader.pyx +8 -0
  295. Cython/Utility/TestCythonScope.pyx +75 -0
  296. Cython/Utility/TestUtilityLoader.c +12 -0
  297. Cython/Utility/TypeConversion.c +1284 -0
  298. Cython/Utility/UFuncs.pyx +50 -0
  299. Cython/Utility/UFuncs_C.c +89 -0
  300. Cython/Utility/__init__.py +28 -0
  301. Cython/Utility/arrayarray.h +148 -0
  302. Cython/Utils.py +687 -0
  303. Cython/__init__.py +10 -0
  304. Cython/__init__.pyi +7 -0
  305. Cython/py.typed +0 -0
  306. cython-3.1.0.dist-info/COPYING.txt +19 -0
  307. cython-3.1.0.dist-info/LICENSE.txt +176 -0
  308. cython-3.1.0.dist-info/METADATA +636 -0
  309. cython-3.1.0.dist-info/RECORD +316 -0
  310. cython-3.1.0.dist-info/WHEEL +5 -0
  311. cython-3.1.0.dist-info/entry_points.txt +4 -0
  312. cython-3.1.0.dist-info/top_level.txt +3 -0
  313. cython.py +29 -0
  314. pyximport/__init__.py +4 -0
  315. pyximport/pyxbuild.py +160 -0
  316. pyximport/pyximport.py +482 -0
@@ -0,0 +1,4509 @@
1
+ import cython
2
+ cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
3
+ Options=object, UtilNodes=object, LetNode=object,
4
+ LetRefNode=object, TreeFragment=object, EncodedString=object,
5
+ error=object, warning=object, copy=object, hashlib=object, sys=object,
6
+ itemgetter=object)
7
+
8
+ import copy
9
+ import hashlib
10
+ import sys
11
+ from operator import itemgetter
12
+
13
+ from . import PyrexTypes
14
+ from . import Naming
15
+ from . import ExprNodes
16
+ from . import Nodes
17
+ from . import Options
18
+ from . import Builtin
19
+ from . import Errors
20
+
21
+ from .Visitor import VisitorTransform, TreeVisitor
22
+ from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
23
+ from .UtilNodes import LetNode, LetRefNode
24
+ from .TreeFragment import TreeFragment
25
+ from .StringEncoding import EncodedString
26
+ from .Errors import error, warning, CompileError, InternalError
27
+
28
+
29
+ class SkipDeclarations:
30
+ """
31
+ Variable and function declarations can often have a deep tree structure,
32
+ and yet most transformations don't need to descend to this depth.
33
+
34
+ Declaration nodes are removed after AnalyseDeclarationsTransform, so there
35
+ is no need to use this for transformations after that point.
36
+ """
37
+ def visit_CTypeDefNode(self, node):
38
+ return node
39
+
40
+ def visit_CVarDefNode(self, node):
41
+ return node
42
+
43
+ def visit_CDeclaratorNode(self, node):
44
+ return node
45
+
46
+ def visit_CBaseTypeNode(self, node):
47
+ return node
48
+
49
+ def visit_CEnumDefNode(self, node):
50
+ return node
51
+
52
+ def visit_CStructOrUnionDefNode(self, node):
53
+ return node
54
+
55
+ def visit_CppClassNode(self, node):
56
+ if node.visibility != "extern":
57
+ # Need to traverse methods.
58
+ self.visitchildren(node)
59
+ return node
60
+
61
+
62
+ class NormalizeTree(CythonTransform):
63
+ """
64
+ This transform fixes up a few things after parsing
65
+ in order to make the parse tree more suitable for
66
+ transforms.
67
+
68
+ a) After parsing, blocks with only one statement will
69
+ be represented by that statement, not by a StatListNode.
70
+ When doing transforms this is annoying and inconsistent,
71
+ as one cannot in general remove a statement in a consistent
72
+ way and so on. This transform wraps any single statements
73
+ in a StatListNode containing a single statement.
74
+
75
+ b) The PassStatNode is a noop and serves no purpose beyond
76
+ plugging such one-statement blocks; i.e., once parsed a
77
+ ` "pass" can just as well be represented using an empty
78
+ StatListNode. This means less special cases to worry about
79
+ in subsequent transforms (one always checks to see if a
80
+ StatListNode has no children to see if the block is empty).
81
+ """
82
+
83
+ def __init__(self, context):
84
+ super().__init__(context)
85
+ self.is_in_statlist = False
86
+ self.is_in_expr = False
87
+
88
+ def visit_ModuleNode(self, node):
89
+ self.visitchildren(node)
90
+ if not isinstance(node.body, Nodes.StatListNode):
91
+ # This can happen when the body only consists of a single (unused) declaration and no statements.
92
+ node.body = Nodes.StatListNode(pos=node.pos, stats=[node.body])
93
+ return node
94
+
95
+ def visit_ExprNode(self, node):
96
+ stacktmp = self.is_in_expr
97
+ self.is_in_expr = True
98
+ self.visitchildren(node)
99
+ self.is_in_expr = stacktmp
100
+ return node
101
+
102
+ def visit_StatNode(self, node, is_listcontainer=False):
103
+ stacktmp = self.is_in_statlist
104
+ self.is_in_statlist = is_listcontainer
105
+ self.visitchildren(node)
106
+ self.is_in_statlist = stacktmp
107
+ if not self.is_in_statlist and not self.is_in_expr:
108
+ return Nodes.StatListNode(pos=node.pos, stats=[node])
109
+ else:
110
+ return node
111
+
112
+ def visit_StatListNode(self, node):
113
+ self.is_in_statlist = True
114
+ self.visitchildren(node)
115
+ self.is_in_statlist = False
116
+ return node
117
+
118
+ def visit_ParallelAssignmentNode(self, node):
119
+ return self.visit_StatNode(node, True)
120
+
121
+ def visit_CEnumDefNode(self, node):
122
+ return self.visit_StatNode(node, True)
123
+
124
+ def visit_CStructOrUnionDefNode(self, node):
125
+ return self.visit_StatNode(node, True)
126
+
127
+ def visit_ExprStatNode(self, node):
128
+ """Eliminate useless string literals"""
129
+ if node.expr.is_string_literal:
130
+ return Nodes.PassStatNode(node.expr.pos)
131
+ else:
132
+ return self.visit_StatNode(node)
133
+
134
+ def visit_CDeclaratorNode(self, node):
135
+ return node
136
+
137
+
138
+ class PostParseError(CompileError): pass
139
+
140
+ # error strings checked by unit tests, so define them
141
+ ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
142
+ ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
143
+ ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
144
+ class PostParse(ScopeTrackingTransform):
145
+ """
146
+ Basic interpretation of the parse tree, as well as validity
147
+ checking that can be done on a very basic level on the parse
148
+ tree (while still not being a problem with the basic syntax,
149
+ as such).
150
+
151
+ Specifically:
152
+ - Default values to cdef assignments are turned into single
153
+ assignments following the declaration (everywhere but in class
154
+ bodies, where they raise a compile error)
155
+
156
+ - Interpret some node structures into Python runtime values.
157
+ Some nodes take compile-time arguments (currently:
158
+ TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
159
+ which should be interpreted. This happens in a general way
160
+ and other steps should be taken to ensure validity.
161
+
162
+ Type arguments cannot be interpreted in this way.
163
+
164
+ - For __cythonbufferdefaults__ the arguments are checked for
165
+ validity.
166
+
167
+ TemplatedTypeNode has its directives interpreted:
168
+ Any first positional argument goes into the "dtype" attribute,
169
+ any "ndim" keyword argument goes into the "ndim" attribute and
170
+ so on. Also it is checked that the directive combination is valid.
171
+ - __cythonbufferdefaults__ attributes are parsed and put into the
172
+ type information.
173
+
174
+ Note: Currently Parsing.py does a lot of interpretation and
175
+ reorganization that can be refactored into this transform
176
+ if a more pure Abstract Syntax Tree is wanted.
177
+
178
+ - Some invalid uses of := assignment expressions are detected
179
+ """
180
+ def __init__(self, context):
181
+ super().__init__(context)
182
+ self.specialattribute_handlers = {
183
+ '__cythonbufferdefaults__' : self.handle_bufferdefaults
184
+ }
185
+ self.in_pattern_node = False
186
+
187
+ def visit_LambdaNode(self, node):
188
+ # unpack a lambda expression into the corresponding DefNode
189
+ collector = YieldNodeCollector()
190
+ collector.visitchildren(node.result_expr)
191
+ if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode):
192
+ body = Nodes.ExprStatNode(
193
+ node.result_expr.pos, expr=node.result_expr)
194
+ else:
195
+ body = Nodes.ReturnStatNode(
196
+ node.result_expr.pos, value=node.result_expr)
197
+ node.def_node = Nodes.DefNode(
198
+ node.pos, name=node.name,
199
+ args=node.args, star_arg=node.star_arg,
200
+ starstar_arg=node.starstar_arg,
201
+ body=body, doc=None)
202
+ self.visitchildren(node)
203
+ return node
204
+
205
+ def visit_GeneratorExpressionNode(self, node):
206
+ # unpack a generator expression into the corresponding DefNode
207
+ collector = YieldNodeCollector()
208
+ collector.visitchildren(node.loop, attrs=None, exclude=["iterator"])
209
+ node.def_node = Nodes.DefNode(
210
+ node.pos, name=node.name, doc=None,
211
+ args=[], star_arg=None, starstar_arg=None,
212
+ body=node.loop, is_async_def=collector.has_await,
213
+ is_generator_expression=True)
214
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
215
+ self.visitchildren(node)
216
+ return node
217
+
218
+ def visit_ComprehensionNode(self, node):
219
+ # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...)
220
+ if not node.has_local_scope:
221
+ collector = YieldNodeCollector()
222
+ collector.visitchildren(node.loop)
223
+ if collector.has_await:
224
+ node.has_local_scope = True
225
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
226
+ self.visitchildren(node)
227
+ return node
228
+
229
+ # cdef variables
230
+ def handle_bufferdefaults(self, decl):
231
+ if not isinstance(decl.default, ExprNodes.DictNode):
232
+ raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
233
+ self.scope_node.buffer_defaults_node = decl.default
234
+ self.scope_node.buffer_defaults_pos = decl.pos
235
+
236
+ def visit_CVarDefNode(self, node):
237
+ # This assumes only plain names and pointers are assignable on
238
+ # declaration. Also, it makes use of the fact that a cdef decl
239
+ # must appear before the first use, so we don't have to deal with
240
+ # "i = 3; cdef int i = i" and can simply move the nodes around.
241
+ try:
242
+ self.visitchildren(node)
243
+ stats = [node]
244
+ newdecls = []
245
+ for decl in node.declarators:
246
+ declbase = decl
247
+ while isinstance(declbase, (Nodes.CPtrDeclaratorNode, Nodes.CConstDeclaratorNode)):
248
+ declbase = declbase.base
249
+ if isinstance(declbase, Nodes.CNameDeclaratorNode):
250
+ if declbase.default is not None:
251
+ if self.scope_type in ('cclass', 'pyclass', 'struct'):
252
+ if isinstance(self.scope_node, Nodes.CClassDefNode):
253
+ handler = self.specialattribute_handlers.get(decl.name)
254
+ if handler:
255
+ if decl is not declbase:
256
+ raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
257
+ handler(decl)
258
+ continue # Remove declaration
259
+ raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
260
+ first_assignment = self.scope_type != 'module'
261
+ stats.append(Nodes.SingleAssignmentNode(node.pos,
262
+ lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
263
+ rhs=declbase.default, first=first_assignment))
264
+ declbase.default = None
265
+ newdecls.append(decl)
266
+ node.declarators = newdecls
267
+ return stats
268
+ except PostParseError as e:
269
+ # An error in a cdef clause is ok, simply remove the declaration
270
+ # and try to move on to report more errors
271
+ self.context.nonfatal_error(e)
272
+ return None
273
+
274
+ # Split parallel assignments (a,b = b,a) into separate partial
275
+ # assignments that are executed rhs-first using temps. This
276
+ # restructuring must be applied before type analysis so that known
277
+ # types on rhs and lhs can be matched directly. It is required in
278
+ # the case that the types cannot be coerced to a Python type in
279
+ # order to assign from a tuple.
280
+
281
+ def visit_SingleAssignmentNode(self, node):
282
+ self.visitchildren(node)
283
+ return self._visit_assignment_node(node, [node.lhs, node.rhs])
284
+
285
+ def visit_CascadedAssignmentNode(self, node):
286
+ self.visitchildren(node)
287
+ return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
288
+
289
+ def _visit_assignment_node(self, node, expr_list):
290
+ """Flatten parallel assignments into separate single
291
+ assignments or cascaded assignments.
292
+ """
293
+ if sum([ 1 for expr in expr_list
294
+ if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
295
+ # no parallel assignments => nothing to do
296
+ return node
297
+
298
+ expr_list_list = []
299
+ flatten_parallel_assignments(expr_list, expr_list_list)
300
+ temp_refs = []
301
+ eliminate_rhs_duplicates(expr_list_list, temp_refs)
302
+
303
+ nodes = []
304
+ for expr_list in expr_list_list:
305
+ lhs_list = expr_list[:-1]
306
+ rhs = expr_list[-1]
307
+ if len(lhs_list) == 1:
308
+ node = Nodes.SingleAssignmentNode(rhs.pos,
309
+ lhs = lhs_list[0], rhs = rhs)
310
+ else:
311
+ node = Nodes.CascadedAssignmentNode(rhs.pos,
312
+ lhs_list = lhs_list, rhs = rhs)
313
+ nodes.append(node)
314
+
315
+ if len(nodes) == 1:
316
+ assign_node = nodes[0]
317
+ else:
318
+ assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
319
+
320
+ if temp_refs:
321
+ duplicates_and_temps = [ (temp.expression, temp)
322
+ for temp in temp_refs ]
323
+ sort_common_subsequences(duplicates_and_temps)
324
+ for _, temp_ref in duplicates_and_temps[::-1]:
325
+ assign_node = LetNode(temp_ref, assign_node)
326
+
327
+ return assign_node
328
+
329
+ def _flatten_sequence(self, seq, result):
330
+ for arg in seq.args:
331
+ if arg.is_sequence_constructor:
332
+ self._flatten_sequence(arg, result)
333
+ else:
334
+ result.append(arg)
335
+ return result
336
+
337
+ def visit_DelStatNode(self, node):
338
+ self.visitchildren(node)
339
+ node.args = self._flatten_sequence(node, [])
340
+ return node
341
+
342
+ def visit_ExceptClauseNode(self, node):
343
+ if node.is_except_as:
344
+ # except-as must delete NameNode target at the end
345
+ del_target = Nodes.DelStatNode(
346
+ node.pos,
347
+ args=[ExprNodes.NameNode(
348
+ node.target.pos, name=node.target.name)],
349
+ ignore_nonexisting=True)
350
+ node.body = Nodes.StatListNode(
351
+ node.pos,
352
+ stats=[Nodes.TryFinallyStatNode(
353
+ node.pos,
354
+ body=node.body,
355
+ finally_clause=Nodes.StatListNode(
356
+ node.pos,
357
+ stats=[del_target]))])
358
+ self.visitchildren(node)
359
+ return node
360
+
361
+ def visit_AssertStatNode(self, node):
362
+ """Extract the exception raising into a RaiseStatNode to simplify GIL handling.
363
+ """
364
+ if node.exception is None:
365
+ node.exception = Nodes.RaiseStatNode(
366
+ node.pos,
367
+ exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")),
368
+ exc_value=node.value,
369
+ exc_tb=None,
370
+ cause=None,
371
+ builtin_exc_name="AssertionError",
372
+ wrap_tuple_value=True,
373
+ )
374
+ node.value = None
375
+ self.visitchildren(node)
376
+ return node
377
+
378
+ def visit_ErrorNode(self, node):
379
+ error(node.pos, node.what)
380
+ return None
381
+
382
+ def visit_MatchCaseNode(self, node):
383
+ node.validate_targets()
384
+ self.visitchildren(node)
385
+ return node
386
+
387
+ def visit_MatchNode(self, node):
388
+ node.validate_irrefutable()
389
+ self.visitchildren(node)
390
+ return node
391
+
392
+ def visit_PatternNode(self, node):
393
+ in_pattern_node, self.in_pattern_node = self.in_pattern_node, True
394
+ self.visitchildren(node)
395
+ self.in_pattern_node = in_pattern_node
396
+ return node
397
+
398
+ def visit_JoinedStrNode(self, node):
399
+ if self.in_pattern_node:
400
+ error(node.pos, "f-strings are not accepted for pattern matching")
401
+ self.visitchildren(node)
402
+ return node
403
+
404
+ def visit_DefNode(self, node):
405
+ if (self.scope_type == "cclass" and
406
+ node.name in ["__getreadbuffer__", "__getwritebuffer__", "__getsegcount__", "__getcharbuffer__"]):
407
+ warning(node.pos, f"'{node.name}' relates to the old Python 2 buffer protocol "
408
+ "and is no longer used.", 2)
409
+ return None # drop the node - the arguments are invalid for a def node
410
+ return self.visit_FuncDefNode(node)
411
+
412
+
413
+ class _AssignmentExpressionTargetNameFinder(TreeVisitor):
414
+ def __init__(self):
415
+ super().__init__()
416
+ self.target_names = {}
417
+
418
+ def find_target_names(self, target):
419
+ if target.is_name:
420
+ return [target.name]
421
+ elif target.is_sequence_constructor:
422
+ names = []
423
+ for arg in target.args:
424
+ names.extend(self.find_target_names(arg))
425
+ return names
426
+ # other targets are possible, but it isn't necessary to investigate them here
427
+ return []
428
+
429
+ def visit_ForInStatNode(self, node):
430
+ self.target_names[node] = tuple(self.find_target_names(node.target))
431
+ self.visitchildren(node)
432
+
433
+ def visit_ComprehensionNode(self, node):
434
+ pass # don't recurse into nested comprehensions
435
+
436
+ def visit_LambdaNode(self, node):
437
+ pass # don't recurse into nested lambdas/generator expressions
438
+
439
+ def visit_Node(self, node):
440
+ self.visitchildren(node)
441
+
442
+
443
+ class _AssignmentExpressionChecker(TreeVisitor):
444
+ """
445
+ Enforces rules on AssignmentExpressions within generator expressions and comprehensions
446
+ """
447
+ def __init__(self, loop_node, scope_is_class):
448
+ super().__init__()
449
+
450
+ target_name_finder = _AssignmentExpressionTargetNameFinder()
451
+ target_name_finder.visit(loop_node)
452
+ self.target_names_dict = target_name_finder.target_names
453
+ self.in_iterator = False
454
+ self.in_nested_generator = False
455
+ self.scope_is_class = scope_is_class
456
+ self.current_target_names = ()
457
+ self.all_target_names = set()
458
+ for names in self.target_names_dict.values():
459
+ self.all_target_names.update(names)
460
+
461
+ def _reset_state(self):
462
+ old_state = (self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names)
463
+ # note: not resetting self.in_iterator here, see visit_LambdaNode() below
464
+ self.in_nested_generator = False
465
+ self.scope_is_class = False
466
+ self.current_target_names = ()
467
+ self.all_target_names = set()
468
+ return old_state
469
+
470
+ def _set_state(self, old_state):
471
+ self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names = old_state
472
+
473
+ @classmethod
474
+ def do_checks(cls, loop_node, scope_is_class):
475
+ checker = cls(loop_node, scope_is_class)
476
+ checker.visit(loop_node)
477
+
478
+ def visit_ForInStatNode(self, node):
479
+ if self.in_nested_generator:
480
+ self.visitchildren(node) # once nested, don't do anything special
481
+ return
482
+
483
+ current_target_names = self.current_target_names
484
+ target_name = self.target_names_dict.get(node, None)
485
+ if target_name:
486
+ self.current_target_names += target_name
487
+
488
+ self.in_iterator = True
489
+ self.visit(node.iterator)
490
+ self.in_iterator = False
491
+ self.visitchildren(node, exclude=("iterator",))
492
+
493
+ self.current_target_names = current_target_names
494
+
495
+ def visit_AssignmentExpressionNode(self, node):
496
+ if self.in_iterator:
497
+ error(node.pos, "assignment expression cannot be used in a comprehension iterable expression")
498
+ if self.scope_is_class:
499
+ error(node.pos, "assignment expression within a comprehension cannot be used in a class body")
500
+ if node.target_name in self.current_target_names:
501
+ error(node.pos, "assignment expression cannot rebind comprehension iteration variable '%s'" %
502
+ node.target_name)
503
+ elif node.target_name in self.all_target_names:
504
+ error(node.pos, "comprehension inner loop cannot rebind assignment expression target '%s'" %
505
+ node.target_name)
506
+
507
+ def visit_LambdaNode(self, node):
508
+ # Don't reset "in_iterator" - an assignment expression in a lambda in an
509
+ # iterator is explicitly tested by the Python testcases and banned.
510
+ old_state = self._reset_state()
511
+ # the lambda node's "def_node" is not set up at this point, so we need to recurse into it explicitly.
512
+ self.visit(node.result_expr)
513
+ self._set_state(old_state)
514
+
515
+ def visit_ComprehensionNode(self, node):
516
+ in_nested_generator = self.in_nested_generator
517
+ self.in_nested_generator = True
518
+ self.visitchildren(node)
519
+ self.in_nested_generator = in_nested_generator
520
+
521
+ def visit_GeneratorExpressionNode(self, node):
522
+ in_nested_generator = self.in_nested_generator
523
+ self.in_nested_generator = True
524
+ # def_node isn't set up yet, so we need to visit the loop directly.
525
+ self.visit(node.loop)
526
+ self.in_nested_generator = in_nested_generator
527
+
528
+ def visit_Node(self, node):
529
+ self.visitchildren(node)
530
+
531
+
532
+ def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
533
+ """Replace rhs items by LetRefNodes if they appear more than once.
534
+ Creates a sequence of LetRefNodes that set up the required temps
535
+ and appends them to ref_node_sequence. The input list is modified
536
+ in-place.
537
+ """
538
+ seen_nodes = set()
539
+ ref_nodes = {}
540
+ def find_duplicates(node):
541
+ if node.is_literal or node.is_name:
542
+ # no need to replace those; can't include attributes here
543
+ # as their access is not necessarily side-effect free
544
+ return
545
+ if node in seen_nodes:
546
+ if node not in ref_nodes:
547
+ ref_node = LetRefNode(node)
548
+ ref_nodes[node] = ref_node
549
+ ref_node_sequence.append(ref_node)
550
+ else:
551
+ seen_nodes.add(node)
552
+ if node.is_sequence_constructor:
553
+ for item in node.args:
554
+ find_duplicates(item)
555
+
556
+ for expr_list in expr_list_list:
557
+ rhs = expr_list[-1]
558
+ find_duplicates(rhs)
559
+ if not ref_nodes:
560
+ return
561
+
562
+ def substitute_nodes(node):
563
+ if node in ref_nodes:
564
+ return ref_nodes[node]
565
+ elif node.is_sequence_constructor:
566
+ node.args = list(map(substitute_nodes, node.args))
567
+ return node
568
+
569
+ # replace nodes inside of the common subexpressions
570
+ for node in ref_nodes:
571
+ if node.is_sequence_constructor:
572
+ node.args = list(map(substitute_nodes, node.args))
573
+
574
+ # replace common subexpressions on all rhs items
575
+ for expr_list in expr_list_list:
576
+ expr_list[-1] = substitute_nodes(expr_list[-1])
577
+
578
+ def sort_common_subsequences(items):
579
+ """Sort items/subsequences so that all items and subsequences that
580
+ an item contains appear before the item itself. This is needed
581
+ because each rhs item must only be evaluated once, so its value
582
+ must be evaluated first and then reused when packing sequences
583
+ that contain it.
584
+
585
+ This implies a partial order, and the sort must be stable to
586
+ preserve the original order as much as possible, so we use a
587
+ simple insertion sort (which is very fast for short sequences, the
588
+ normal case in practice).
589
+ """
590
+ def contains(seq, x):
591
+ for item in seq:
592
+ if item is x:
593
+ return True
594
+ elif item.is_sequence_constructor and contains(item.args, x):
595
+ return True
596
+ return False
597
+ def lower_than(a,b):
598
+ return b.is_sequence_constructor and contains(b.args, a)
599
+
600
+ for pos, item in enumerate(items):
601
+ key = item[1] # the ResultRefNode which has already been injected into the sequences
602
+ new_pos = pos
603
+ for i in range(pos-1, -1, -1):
604
+ if lower_than(key, items[i][0]):
605
+ new_pos = i
606
+ if new_pos != pos:
607
+ for i in range(pos, new_pos, -1):
608
+ items[i] = items[i-1]
609
+ items[new_pos] = item
610
+
611
+
612
+ def unpack_string_to_character_literals(literal):
613
+ chars = []
614
+ pos = literal.pos
615
+ stype = literal.__class__
616
+ sval = literal.value
617
+ sval_type = sval.__class__
618
+ for char in sval:
619
+ cval = sval_type(char)
620
+ chars.append(stype(pos, value=cval))
621
+ return chars
622
+
623
+
624
+ @cython.cfunc
625
+ def flatten_parallel_assignments(input: list, output: list):
626
+ # The input is a list of expression nodes, representing the LHSs
627
+ # and RHS of one (possibly cascaded) assignment statement. For
628
+ # sequence constructors, rearranges the matching parts of both
629
+ # sides into a list of equivalent assignments between the
630
+ # individual elements. This transformation is applied
631
+ # recursively, so that nested structures get matched as well.
632
+ rhs = input[-1]
633
+ if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
634
+ or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
635
+ output.append(input)
636
+ return
637
+
638
+ complete_assignments = []
639
+
640
+ if rhs.is_sequence_constructor:
641
+ rhs_args = rhs.args
642
+ elif rhs.is_string_literal:
643
+ rhs_args = unpack_string_to_character_literals(rhs)
644
+
645
+ starred_targets: cython.Py_ssize_t
646
+ lhs_size: cython.Py_ssize_t
647
+ rhs_size: cython.Py_ssize_t = len(rhs_args)
648
+ lhs_targets = [[] for _ in range(rhs_size)]
649
+ starred_assignments = []
650
+
651
+ for lhs in input[:-1]:
652
+ if not lhs.is_sequence_constructor:
653
+ if lhs.is_starred:
654
+ error(lhs.pos, "starred assignment target must be in a list or tuple")
655
+ complete_assignments.append(lhs)
656
+ continue
657
+ lhs_size = len(lhs.args)
658
+ starred_targets = 0
659
+ for expr in lhs.args:
660
+ starred_targets += bool(expr.is_starred)
661
+ if starred_targets > 1:
662
+ error(lhs.pos, "more than 1 starred expression in assignment")
663
+ output.append([lhs,rhs])
664
+ continue
665
+ elif lhs_size - starred_targets > rhs_size:
666
+ error(lhs.pos, "need more than %d value%s to unpack"
667
+ % (rhs_size, (rhs_size != 1) and 's' or ''))
668
+ output.append([lhs,rhs])
669
+ continue
670
+ elif starred_targets:
671
+ map_starred_assignment(lhs_targets, starred_assignments,
672
+ lhs.args, rhs_args)
673
+ elif lhs_size < rhs_size:
674
+ error(lhs.pos, "too many values to unpack (expected %d, got %d)"
675
+ % (lhs_size, rhs_size))
676
+ output.append([lhs,rhs])
677
+ continue
678
+ else:
679
+ for targets, expr in zip(lhs_targets, lhs.args):
680
+ targets.append(expr)
681
+
682
+ if complete_assignments:
683
+ complete_assignments.append(rhs)
684
+ output.append(complete_assignments)
685
+
686
+ # recursively flatten partial assignments
687
+ for cascade, rhs in zip(lhs_targets, rhs_args):
688
+ if cascade:
689
+ cascade.append(rhs)
690
+ flatten_parallel_assignments(cascade, output)
691
+
692
+ # recursively flatten starred assignments
693
+ for cascade in starred_assignments:
694
+ if cascade[0].is_sequence_constructor:
695
+ flatten_parallel_assignments(cascade, output)
696
+ else:
697
+ output.append(cascade)
698
+
699
+
700
+ @cython.cfunc
701
+ def map_starred_assignment(lhs_targets: list, starred_assignments: list, lhs_args: list, rhs_args: list):
702
+ # Appends the fixed-position LHS targets to the target list that
703
+ # appear left and right of the starred argument.
704
+ #
705
+ # The starred_assignments list receives a new tuple
706
+ # (lhs_target, rhs_values_list) that maps the remaining arguments
707
+ # (those that match the starred target) to a list.
708
+
709
+ # left side of the starred target
710
+ i: cython.Py_ssize_t
711
+ starred: cython.Py_ssize_t
712
+ lhs_remaining: cython.Py_ssize_t
713
+ for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
714
+ if expr.is_starred:
715
+ starred = i
716
+ lhs_remaining = len(lhs_args) - i - 1
717
+ break
718
+ targets.append(expr)
719
+ else:
720
+ raise InternalError("no starred arg found when splitting starred assignment")
721
+
722
+ # right side of the starred target
723
+ for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
724
+ lhs_args[starred + 1:])):
725
+ targets.append(expr)
726
+
727
+ # the starred target itself, must be assigned a (potentially empty) list
728
+ target = lhs_args[starred].target # unpack starred node
729
+ starred_rhs = rhs_args[starred:]
730
+ if lhs_remaining:
731
+ starred_rhs = starred_rhs[:-lhs_remaining]
732
+ if starred_rhs:
733
+ pos = starred_rhs[0].pos
734
+ else:
735
+ pos = target.pos
736
+ starred_assignments.append([
737
+ target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
738
+
739
+
740
+ class PxdPostParse(CythonTransform, SkipDeclarations):
741
+ """
742
+ Basic interpretation/validity checking that should only be
743
+ done on pxd trees.
744
+
745
+ A lot of this checking currently happens in the parser; but
746
+ what is listed below happens here.
747
+
748
+ - "def" functions are let through only if they fill the
749
+ getbuffer/releasebuffer slots
750
+
751
+ - cdef functions are let through only if they are on the
752
+ top level and are declared "inline"
753
+ """
754
+ ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
755
+ ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
756
+
757
+ def __call__(self, node):
758
+ self.scope_type = 'pxd'
759
+ return super().__call__(node)
760
+
761
+ def visit_CClassDefNode(self, node):
762
+ old = self.scope_type
763
+ self.scope_type = 'cclass'
764
+ self.visitchildren(node)
765
+ self.scope_type = old
766
+ return node
767
+
768
+ def visit_FuncDefNode(self, node):
769
+ # FuncDefNode always come with an implementation (without
770
+ # an imp they are CVarDefNodes..)
771
+ err = self.ERR_INLINE_ONLY
772
+
773
+ if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
774
+ and node.name in ('__getbuffer__', '__releasebuffer__')):
775
+ err = None # allow these slots
776
+
777
+ if isinstance(node, Nodes.CFuncDefNode):
778
+ if ('inline' in node.modifiers and
779
+ self.scope_type in ('pxd', 'cclass')):
780
+ node.inline_in_pxd = True
781
+ if node.visibility != 'private':
782
+ err = self.ERR_NOGO_WITH_INLINE % node.visibility
783
+ elif node.api:
784
+ err = self.ERR_NOGO_WITH_INLINE % 'api'
785
+ else:
786
+ err = None # allow inline function
787
+ else:
788
+ err = self.ERR_INLINE_ONLY
789
+
790
+ if err:
791
+ self.context.nonfatal_error(PostParseError(node.pos, err))
792
+ return None
793
+ else:
794
+ return node
795
+
796
+
797
+ class TrackNumpyAttributes(VisitorTransform, SkipDeclarations):
798
+ # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two.
799
+ def __init__(self):
800
+ super().__init__()
801
+ self.numpy_module_names = set()
802
+
803
+ def visit_CImportStatNode(self, node):
804
+ if node.module_name == "numpy":
805
+ self.numpy_module_names.add(node.as_name or "numpy")
806
+ return node
807
+
808
+ def visit_AttributeNode(self, node):
809
+ self.visitchildren(node)
810
+ obj = node.obj
811
+ if (obj.is_name and obj.name in self.numpy_module_names) or obj.is_numpy_attribute:
812
+ node.is_numpy_attribute = True
813
+ return node
814
+
815
+ visit_Node = VisitorTransform.recurse_to_children
816
+
817
+
818
+ class InterpretCompilerDirectives(CythonTransform):
819
+ """
820
+ After parsing, directives can be stored in a number of places:
821
+ - #cython-comments at the top of the file (stored in ModuleNode)
822
+ - Command-line arguments overriding these
823
+ - @cython.directivename decorators
824
+ - with cython.directivename: statements
825
+ - replaces "cython.compiled" with BoolNode(value=True)
826
+ allowing unreachable blocks to be removed at a fairly early stage
827
+ before cython typing rules are forced on applied
828
+
829
+ This transform is responsible for interpreting these various sources
830
+ and store the directive in two ways:
831
+ - Set the directives attribute of the ModuleNode for global directives.
832
+ - Use a CompilerDirectivesNode to override directives for a subtree.
833
+
834
+ (The first one is primarily to not have to modify with the tree
835
+ structure, so that ModuleNode stay on top.)
836
+
837
+ The directives are stored in dictionaries from name to value in effect.
838
+ Each such dictionary is always filled in for all possible directives,
839
+ using default values where no value is given by the user.
840
+
841
+ The available directives are controlled in Options.py.
842
+
843
+ Note that we have to run this prior to analysis, and so some minor
844
+ duplication of functionality has to occur: We manually track cimports
845
+ and which names the "cython" module may have been imported to.
846
+ """
847
+ unop_method_nodes = {
848
+ 'typeof': ExprNodes.TypeofNode,
849
+
850
+ 'operator.address': ExprNodes.AmpersandNode,
851
+ 'operator.dereference': ExprNodes.DereferenceNode,
852
+ 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
853
+ 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
854
+ 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
855
+ 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
856
+ 'operator.typeid' : ExprNodes.TypeidNode,
857
+
858
+ # For backwards compatibility.
859
+ 'address': ExprNodes.AmpersandNode,
860
+ }
861
+
862
+ binop_method_nodes = {
863
+ 'operator.comma' : ExprNodes.c_binop_constructor(','),
864
+ }
865
+
866
+ special_methods = {
867
+ 'declare', 'union', 'struct', 'typedef',
868
+ 'sizeof', 'cast', 'pointer', 'compiled',
869
+ 'NULL', 'fused_type', 'parallel',
870
+ }
871
+ special_methods.update(unop_method_nodes)
872
+
873
+ valid_cython_submodules = {
874
+ 'cimports',
875
+ 'dataclasses',
876
+ 'operator',
877
+ 'parallel',
878
+ 'view',
879
+ }
880
+
881
+ valid_parallel_directives = {
882
+ "parallel",
883
+ "prange",
884
+ "threadid",
885
+ #"threadsavailable",
886
+ }
887
+
888
+ def __init__(self, context, compilation_directive_defaults):
889
+ super().__init__(context)
890
+ self.cython_module_names = set()
891
+ self.directive_names = {'staticmethod': 'staticmethod'}
892
+ self.parallel_directives = {}
893
+ directives = copy.deepcopy(Options.get_directive_defaults())
894
+ for key, value in compilation_directive_defaults.items():
895
+ directives[str(key)] = copy.deepcopy(value)
896
+ self.directives = directives
897
+
898
+ def check_directive_scope(self, pos, directive, scope):
899
+ legal_scopes = Options.directive_scopes.get(directive, None)
900
+ if legal_scopes and scope not in legal_scopes:
901
+ self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
902
+ 'is not allowed in %s scope' % (directive, scope)))
903
+ return False
904
+ else:
905
+ if directive not in Options.directive_types:
906
+ error(pos, "Invalid directive: '%s'." % (directive,))
907
+ return True
908
+
909
+ def _check_valid_cython_module(self, pos, module_name):
910
+ if not module_name.startswith("cython."):
911
+ return
912
+ submodule = module_name.split('.', 2)[1]
913
+ if submodule in self.valid_cython_submodules:
914
+ return
915
+
916
+ extra = ""
917
+ # This is very rarely used, so don't waste space on static tuples.
918
+ hints = [
919
+ line.split() for line in """\
920
+ imp cimports
921
+ cimp cimports
922
+ para parallel
923
+ parra parallel
924
+ dataclass dataclasses
925
+ """.splitlines()[:-1]
926
+ ]
927
+ for wrong, correct in hints:
928
+ if module_name.startswith("cython." + wrong):
929
+ extra = "Did you mean 'cython.%s' ?" % correct
930
+ break
931
+ if not extra:
932
+ is_simple_cython_name = submodule in Options.directive_types
933
+ if not is_simple_cython_name and not submodule.startswith("_"):
934
+ # Try to find it in the Shadow module (i.e. the pure Python namespace of cython.*).
935
+ # FIXME: use an internal reference of "cython.*" names instead of Shadow.py
936
+ from .. import Shadow
937
+ is_simple_cython_name = hasattr(Shadow, submodule)
938
+ if is_simple_cython_name:
939
+ extra = "Instead, use 'import cython' and then 'cython.%s'." % submodule
940
+
941
+ error(pos, "'%s' is not a valid cython.* module%s%s" % (
942
+ module_name,
943
+ ". " if extra else "",
944
+ extra,
945
+ ))
946
+
947
+ # Set up processing and handle the cython: comments.
948
+ def visit_ModuleNode(self, node):
949
+ for key in sorted(node.directive_comments):
950
+ if not self.check_directive_scope(node.pos, key, 'module'):
951
+ self.wrong_scope_error(node.pos, key, 'module')
952
+ del node.directive_comments[key]
953
+
954
+ self.module_scope = node.scope
955
+
956
+ self.directives.update(node.directive_comments)
957
+ node.directives = self.directives
958
+ node.parallel_directives = self.parallel_directives
959
+ self.visitchildren(node)
960
+ node.cython_module_names = self.cython_module_names
961
+ return node
962
+
963
+ def visit_CompilerDirectivesNode(self, node):
964
+ old_directives, self.directives = self.directives, node.directives
965
+ self.visitchildren(node)
966
+ self.directives = old_directives
967
+ return node
968
+
969
+ # The following four functions track imports and cimports that
970
+ # begin with "cython"
971
+ def is_cython_directive(self, name):
972
+ return (name in Options.directive_types or
973
+ name in self.special_methods or
974
+ PyrexTypes.parse_basic_type(name))
975
+
976
+ def is_parallel_directive(self, full_name, pos):
977
+ """
978
+ Checks to see if fullname (e.g. cython.parallel.prange) is a valid
979
+ parallel directive. If it is a star import it also updates the
980
+ parallel_directives.
981
+ """
982
+ result = (full_name + ".").startswith("cython.parallel.")
983
+
984
+ if result:
985
+ directive = full_name.split('.')
986
+ if full_name == "cython.parallel":
987
+ self.parallel_directives["parallel"] = "cython.parallel"
988
+ elif full_name == "cython.parallel.*":
989
+ for name in self.valid_parallel_directives:
990
+ self.parallel_directives[name] = "cython.parallel.%s" % name
991
+ elif (len(directive) != 3 or
992
+ directive[-1] not in self.valid_parallel_directives):
993
+ error(pos, "No such directive: %s" % full_name)
994
+
995
+ return result
996
+
997
+ def visit_CImportStatNode(self, node):
998
+ module_name = node.module_name
999
+ if module_name == "cython.cimports":
1000
+ error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.")
1001
+ if module_name.startswith("cython.cimports."):
1002
+ if node.as_name and node.as_name != 'cython':
1003
+ node.module_name = module_name[len("cython.cimports."):]
1004
+ return node
1005
+ error(node.pos,
1006
+ "Python cimports must use 'from cython.cimports... import ...'"
1007
+ " or 'import ... as ...', not just 'import ...'")
1008
+
1009
+ if module_name == "cython":
1010
+ self.cython_module_names.add(node.as_name or "cython")
1011
+ elif module_name.startswith("cython."):
1012
+ if module_name.startswith("cython.parallel."):
1013
+ error(node.pos, node.module_name + " is not a module")
1014
+ else:
1015
+ self._check_valid_cython_module(node.pos, module_name)
1016
+
1017
+ if module_name == "cython.parallel":
1018
+ if node.as_name and node.as_name != "cython":
1019
+ self.parallel_directives[node.as_name] = module_name
1020
+ else:
1021
+ self.cython_module_names.add("cython")
1022
+ self.parallel_directives[
1023
+ "cython.parallel"] = module_name
1024
+ elif node.as_name:
1025
+ self.directive_names[node.as_name] = module_name[7:]
1026
+ else:
1027
+ self.cython_module_names.add("cython")
1028
+ # if this cimport was a compiler directive, we don't
1029
+ # want to leave the cimport node sitting in the tree
1030
+ return None
1031
+ return node
1032
+
1033
+ def visit_FromCImportStatNode(self, node):
1034
+ module_name = node.module_name
1035
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1036
+ # only supported for convenience
1037
+ return self._create_cimport_from_import(
1038
+ node.pos, module_name, node.relative_level, node.imported_names)
1039
+ elif not node.relative_level and (
1040
+ module_name == "cython" or module_name.startswith("cython.")):
1041
+ self._check_valid_cython_module(node.pos, module_name)
1042
+ submodule = (module_name + ".")[7:]
1043
+ newimp = []
1044
+ for pos, name, as_name in node.imported_names:
1045
+ full_name = submodule + name
1046
+ qualified_name = "cython." + full_name
1047
+ if self.is_parallel_directive(qualified_name, node.pos):
1048
+ # from cython cimport parallel, or
1049
+ # from cython.parallel cimport parallel, prange, ...
1050
+ self.parallel_directives[as_name or name] = qualified_name
1051
+ elif self.is_cython_directive(full_name):
1052
+ self.directive_names[as_name or name] = full_name
1053
+ elif full_name in ['dataclasses', 'typing']:
1054
+ self.directive_names[as_name or name] = full_name
1055
+ # unlike many directives, still treat it as a regular module
1056
+ newimp.append((pos, name, as_name))
1057
+ else:
1058
+ newimp.append((pos, name, as_name))
1059
+
1060
+ if not newimp:
1061
+ return None
1062
+
1063
+ node.imported_names = newimp
1064
+ return node
1065
+
1066
+ def visit_FromImportStatNode(self, node):
1067
+ import_node = node.module
1068
+ module_name = import_node.module_name.value
1069
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1070
+ imported_names = []
1071
+ for name, name_node in node.items:
1072
+ imported_names.append(
1073
+ (name_node.pos, name, None if name == name_node.name else name_node.name))
1074
+ return self._create_cimport_from_import(
1075
+ node.pos, module_name, import_node.level, imported_names)
1076
+ elif module_name == "cython" or module_name.startswith("cython."):
1077
+ self._check_valid_cython_module(import_node.module_name.pos, module_name)
1078
+ submodule = (module_name + ".")[7:]
1079
+ newimp = []
1080
+ for name, name_node in node.items:
1081
+ full_name = submodule + name
1082
+ qualified_name = "cython." + full_name
1083
+ if self.is_parallel_directive(qualified_name, node.pos):
1084
+ self.parallel_directives[name_node.name] = qualified_name
1085
+ elif self.is_cython_directive(full_name):
1086
+ self.directive_names[name_node.name] = full_name
1087
+ else:
1088
+ newimp.append((name, name_node))
1089
+ if not newimp:
1090
+ return None
1091
+ node.items = newimp
1092
+ return node
1093
+
1094
+ def _create_cimport_from_import(self, node_pos, module_name, level, imported_names):
1095
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1096
+ module_name = EncodedString(module_name[len("cython.cimports."):]) # may be empty
1097
+
1098
+ if module_name:
1099
+ # from cython.cimports.a.b import x, y, z => from a.b cimport x, y, z
1100
+ return Nodes.FromCImportStatNode(
1101
+ node_pos, module_name=module_name,
1102
+ relative_level=level,
1103
+ imported_names=imported_names)
1104
+ else:
1105
+ # from cython.cimports import x, y, z => cimport x; cimport y; cimport z
1106
+ return [
1107
+ Nodes.CImportStatNode(
1108
+ pos,
1109
+ module_name=dotted_name,
1110
+ as_name=as_name,
1111
+ is_absolute=level == 0)
1112
+ for pos, dotted_name, as_name in imported_names
1113
+ ]
1114
+
1115
+ def visit_SingleAssignmentNode(self, node):
1116
+ if isinstance(node.rhs, ExprNodes.ImportNode):
1117
+ module_name = node.rhs.module_name.value
1118
+ if module_name != "cython" and not module_name.startswith("cython."):
1119
+ return node
1120
+
1121
+ node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name)
1122
+ node = self.visit_CImportStatNode(node)
1123
+ else:
1124
+ self.visitchildren(node)
1125
+
1126
+ return node
1127
+
1128
+ def visit_NameNode(self, node):
1129
+ if node.annotation:
1130
+ self.visitchild(node, 'annotation')
1131
+ if node.name in self.cython_module_names:
1132
+ node.is_cython_module = True
1133
+ else:
1134
+ directive = self.directive_names.get(node.name)
1135
+ if directive is not None:
1136
+ node.cython_attribute = directive
1137
+ if node.as_cython_attribute() == "compiled":
1138
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1139
+ # before they have a chance to cause compile-errors
1140
+ return node
1141
+
1142
+ def visit_AttributeNode(self, node):
1143
+ self.visitchildren(node)
1144
+ if node.as_cython_attribute() == "compiled":
1145
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1146
+ # before they have a chance to cause compile-errors
1147
+ return node
1148
+
1149
+ def visit_AnnotationNode(self, node):
1150
+ # for most transforms annotations are left unvisited (because they're unevaluated)
1151
+ # however, it is important to pick up compiler directives from them
1152
+ if node.expr:
1153
+ self.visit(node.expr)
1154
+ return node
1155
+
1156
+ def visit_NewExprNode(self, node):
1157
+ self.visitchild(node, 'cppclass')
1158
+ self.visitchildren(node)
1159
+ return node
1160
+
1161
+ def try_to_parse_directives(self, node):
1162
+ # If node is the contents of an directive (in a with statement or
1163
+ # decorator), returns a list of (directivename, value) pairs.
1164
+ # Otherwise, returns None
1165
+ if isinstance(node, ExprNodes.CallNode):
1166
+ self.visitchild(node, 'function')
1167
+ optname = node.function.as_cython_attribute()
1168
+ if optname:
1169
+ directivetype = Options.directive_types.get(optname)
1170
+ if directivetype:
1171
+ args, kwds = node.explicit_args_kwds()
1172
+ directives = []
1173
+ key_value_pairs = []
1174
+ if kwds is not None and directivetype is not dict:
1175
+ for keyvalue in kwds.key_value_pairs:
1176
+ key, value = keyvalue
1177
+ sub_optname = "%s.%s" % (optname, key.value)
1178
+ if Options.directive_types.get(sub_optname):
1179
+ directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
1180
+ else:
1181
+ key_value_pairs.append(keyvalue)
1182
+ if not key_value_pairs:
1183
+ kwds = None
1184
+ else:
1185
+ kwds.key_value_pairs = key_value_pairs
1186
+ if directives and not kwds and not args:
1187
+ return directives
1188
+ directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
1189
+ return directives
1190
+ elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
1191
+ self.visit(node)
1192
+ optname = node.as_cython_attribute()
1193
+ if optname:
1194
+ directivetype = Options.directive_types.get(optname)
1195
+ if directivetype is bool:
1196
+ arg = ExprNodes.BoolNode(node.pos, value=True)
1197
+ return [self.try_to_parse_directive(optname, [arg], None, node.pos)]
1198
+ elif directivetype is None or directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1199
+ return [(optname, None)]
1200
+ else:
1201
+ raise PostParseError(
1202
+ node.pos, "The '%s' directive should be used as a function call." % optname)
1203
+ return None
1204
+
1205
+ def try_to_parse_directive(self, optname, args, kwds, pos):
1206
+ if optname == 'np_pythran' and not self.context.cpp:
1207
+ raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname)
1208
+ elif optname == 'exceptval':
1209
+ # default: exceptval(None, check=True)
1210
+ arg_error = len(args) > 1
1211
+ check = True
1212
+ if kwds and kwds.key_value_pairs:
1213
+ kw = kwds.key_value_pairs[0]
1214
+ if (len(kwds.key_value_pairs) == 1 and
1215
+ kw.key.is_string_literal and kw.key.value == 'check' and
1216
+ isinstance(kw.value, ExprNodes.BoolNode)):
1217
+ check = kw.value.value
1218
+ else:
1219
+ arg_error = True
1220
+ if arg_error:
1221
+ raise PostParseError(
1222
+ pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"')
1223
+ return ('exceptval', (args[0] if args else None, check))
1224
+
1225
+ directivetype = Options.directive_types.get(optname)
1226
+ if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
1227
+ return optname, Options.get_directive_defaults()[optname]
1228
+ elif directivetype is bool:
1229
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1230
+ raise PostParseError(pos,
1231
+ 'The %s directive takes one compile-time boolean argument' % optname)
1232
+ return (optname, args[0].value)
1233
+ elif directivetype is int:
1234
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
1235
+ raise PostParseError(pos,
1236
+ 'The %s directive takes one compile-time integer argument' % optname)
1237
+ return (optname, int(args[0].value))
1238
+ elif directivetype is str:
1239
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1240
+ raise PostParseError(pos,
1241
+ 'The %s directive takes one compile-time string argument' % optname)
1242
+ return (optname, str(args[0].value))
1243
+ elif directivetype is type:
1244
+ if kwds is not None or len(args) != 1:
1245
+ raise PostParseError(pos,
1246
+ 'The %s directive takes one type argument' % optname)
1247
+ return (optname, args[0])
1248
+ elif directivetype is dict:
1249
+ if len(args) != 0:
1250
+ raise PostParseError(pos,
1251
+ 'The %s directive takes no prepositional arguments' % optname)
1252
+ return optname, kwds.as_python_dict()
1253
+ elif directivetype is list:
1254
+ if kwds and len(kwds.key_value_pairs) != 0:
1255
+ raise PostParseError(pos,
1256
+ 'The %s directive takes no keyword arguments' % optname)
1257
+ return optname, [ str(arg.value) for arg in args ]
1258
+ elif callable(directivetype):
1259
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1260
+ raise PostParseError(pos,
1261
+ 'The %s directive takes one compile-time string argument' % optname)
1262
+ return (optname, directivetype(optname, str(args[0].value)))
1263
+ elif directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1264
+ # signal to pass things on without processing
1265
+ return (optname, (args, kwds.as_python_dict() if kwds else {}))
1266
+ else:
1267
+ assert False
1268
+
1269
+ def visit_with_directives(self, node, directives, contents_directives):
1270
+ # contents_directives may be None
1271
+ if not directives:
1272
+ assert not contents_directives
1273
+ return self.visit_Node(node)
1274
+
1275
+ old_directives = self.directives
1276
+ new_directives = Options.copy_inherited_directives(old_directives, **directives)
1277
+ if contents_directives is not None:
1278
+ new_contents_directives = Options.copy_inherited_directives(
1279
+ old_directives, **contents_directives)
1280
+ else:
1281
+ new_contents_directives = new_directives
1282
+
1283
+ if new_directives == old_directives:
1284
+ return self.visit_Node(node)
1285
+
1286
+ self.directives = new_directives
1287
+ if (contents_directives is not None and
1288
+ new_contents_directives != new_directives):
1289
+ # we need to wrap the node body in a compiler directives node
1290
+ node.body = Nodes.StatListNode(
1291
+ node.body.pos,
1292
+ stats=[
1293
+ Nodes.CompilerDirectivesNode(
1294
+ node.body.pos,
1295
+ directives=new_contents_directives,
1296
+ body=node.body)
1297
+ ]
1298
+ )
1299
+ retbody = self.visit_Node(node)
1300
+ self.directives = old_directives
1301
+
1302
+ if isinstance(retbody, Nodes.CompilerDirectivesNode):
1303
+ new_directives.update(retbody.directives)
1304
+ retbody = retbody.body
1305
+ if not isinstance(retbody, Nodes.StatListNode):
1306
+ retbody = Nodes.StatListNode(node.pos, stats=[retbody])
1307
+ return Nodes.CompilerDirectivesNode(
1308
+ retbody.pos, body=retbody, directives=new_directives, is_terminator=retbody.is_terminator)
1309
+
1310
+ # Handle decorators
1311
+ def visit_FuncDefNode(self, node):
1312
+ directives, contents_directives = self._extract_directives(node, 'function')
1313
+ return self.visit_with_directives(node, directives, contents_directives)
1314
+
1315
+ def visit_CVarDefNode(self, node):
1316
+ directives, _ = self._extract_directives(node, 'function')
1317
+ for name, value in directives.items():
1318
+ if name == 'locals':
1319
+ node.directive_locals = value
1320
+ elif name not in ('final', 'staticmethod'):
1321
+ self.context.nonfatal_error(PostParseError(
1322
+ node.pos,
1323
+ "Cdef functions can only take cython.locals(), "
1324
+ "staticmethod, or final decorators, got %s." % name))
1325
+ return self.visit_with_directives(node, directives, contents_directives=None)
1326
+
1327
+ def visit_CClassDefNode(self, node):
1328
+ directives, contents_directives = self._extract_directives(node, 'cclass')
1329
+ return self.visit_with_directives(node, directives, contents_directives)
1330
+
1331
+ def visit_CppClassNode(self, node):
1332
+ directives, contents_directives = self._extract_directives(node, 'cppclass')
1333
+ return self.visit_with_directives(node, directives, contents_directives)
1334
+
1335
+ def visit_PyClassDefNode(self, node):
1336
+ directives, contents_directives = self._extract_directives(node, 'class')
1337
+ return self.visit_with_directives(node, directives, contents_directives)
1338
+
1339
+ def _extract_directives(self, node, scope_name):
1340
+ """
1341
+ Returns two dicts - directives applied to this function/class
1342
+ and directives applied to its contents. They aren't always the
1343
+ same (since e.g. cfunc should not be applied to inner functions)
1344
+ """
1345
+ if not node.decorators:
1346
+ return {}, {}
1347
+ # Split the decorators into two lists -- real decorators and directives
1348
+ directives = []
1349
+ realdecs = []
1350
+ both = []
1351
+ current_opt_dict = dict(self.directives)
1352
+ missing = object()
1353
+ # Decorators coming first take precedence.
1354
+ for dec in node.decorators[::-1]:
1355
+ new_directives = self.try_to_parse_directives(dec.decorator)
1356
+ if new_directives is not None:
1357
+ for directive in new_directives:
1358
+ if self.check_directive_scope(node.pos, directive[0], scope_name):
1359
+ name, value = directive
1360
+ if name in ('nogil', 'with_gil'):
1361
+ if value is None:
1362
+ value = True
1363
+ else:
1364
+ args, kwds = value
1365
+ if kwds or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1366
+ raise PostParseError(dec.pos, 'The %s directive takes one compile-time boolean argument' % name)
1367
+ value = args[0].value
1368
+ directive = (name, value)
1369
+ if current_opt_dict.get(name, missing) != value:
1370
+ if name == 'cfunc' and 'ufunc' in current_opt_dict:
1371
+ error(dec.pos, "Cannot apply @cfunc to @ufunc, please reverse the decorators.")
1372
+ directives.append(directive)
1373
+ current_opt_dict[name] = value
1374
+ else:
1375
+ warning(dec.pos, "Directive does not change previous value (%s%s)" % (
1376
+ name, '=%r' % value if value is not None else ''))
1377
+ if directive[0] == 'staticmethod':
1378
+ both.append(dec)
1379
+ # Adapt scope type based on decorators that change it.
1380
+ if directive[0] == 'cclass' and scope_name == 'class':
1381
+ scope_name = 'cclass'
1382
+ else:
1383
+ realdecs.append(dec)
1384
+ node.decorators = realdecs[::-1] + both[::-1]
1385
+ # merge or override repeated directives
1386
+ optdict = {}
1387
+ contents_optdict = {}
1388
+ for name, value in directives:
1389
+ if name in optdict:
1390
+ old_value = optdict[name]
1391
+ # keywords and arg lists can be merged, everything
1392
+ # else overrides completely
1393
+ if isinstance(old_value, dict):
1394
+ old_value.update(value)
1395
+ elif isinstance(old_value, list):
1396
+ old_value.extend(value)
1397
+ else:
1398
+ optdict[name] = value
1399
+ else:
1400
+ optdict[name] = value
1401
+ if name not in Options.immediate_decorator_directives:
1402
+ contents_optdict[name] = value
1403
+ return optdict, contents_optdict
1404
+
1405
+ # Handle with-statements
1406
+ def visit_WithStatNode(self, node):
1407
+ directive_dict = {}
1408
+ for directive in self.try_to_parse_directives(node.manager) or []:
1409
+ if directive is None:
1410
+ continue
1411
+ if node.target is not None:
1412
+ self.context.nonfatal_error(
1413
+ PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
1414
+ continue
1415
+ name, value = directive
1416
+ if name in ('nogil', 'gil'):
1417
+ # special case: in pure mode, "with nogil" spells "with cython.nogil"
1418
+ return self._transform_with_gil(node, name)
1419
+ elif name == "critical_section":
1420
+ args, kwds = value
1421
+ return self._transform_critical_section(node, args, kwds)
1422
+ elif self.check_directive_scope(node.pos, name, 'with statement'):
1423
+ directive_dict[name] = value
1424
+ if directive_dict:
1425
+ return self.visit_with_directives(node.body, directive_dict, contents_directives=None)
1426
+ return self.visit_Node(node)
1427
+
1428
+ def _transform_with_gil(self, node, state):
1429
+ assert state in ('gil', 'nogil')
1430
+ manager = node.manager
1431
+ condition = None
1432
+ if isinstance(manager, ExprNodes.SimpleCallNode) and manager.args:
1433
+ if len(manager.args) > 1:
1434
+ self.context.nonfatal_error(
1435
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % state))
1436
+ condition = manager.args[0]
1437
+ elif isinstance(manager, ExprNodes.GeneralCallNode):
1438
+ self.context.nonfatal_error(
1439
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % state))
1440
+ node = Nodes.GILStatNode(node.pos, state=state, body=node.body, condition=condition)
1441
+ return self.visit_Node(node)
1442
+
1443
+ def _transform_critical_section(self, node, args, kwds):
1444
+ if len(args) < 1 or len(args) > 2 or kwds:
1445
+ self.context.nonfatal_error(
1446
+ PostParseError(node.pos, "critical_section directive accepts one or two positional arguments")
1447
+ )
1448
+ node = Nodes.CriticalSectionStatNode(
1449
+ node.pos, args=args, body=node.body
1450
+ )
1451
+ return self.visit_Node(node)
1452
+
1453
+
1454
+ class ParallelRangeTransform(CythonTransform, SkipDeclarations):
1455
+ """
1456
+ Transform cython.parallel stuff. The parallel_directives come from the
1457
+ module node, set there by InterpretCompilerDirectives.
1458
+
1459
+ x = cython.parallel.threadavailable() -> ParallelThreadAvailableNode
1460
+ with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
1461
+ print cython.parallel.threadid() -> ParallelThreadIdNode
1462
+ for i in cython.parallel.prange(...): -> ParallelRangeNode
1463
+ ...
1464
+ """
1465
+
1466
+ # a list of names, maps 'cython.parallel.prange' in the code to
1467
+ # ['cython', 'parallel', 'prange']
1468
+ parallel_directive = None
1469
+
1470
+ # Indicates whether a namenode in an expression is the cython module
1471
+ namenode_is_cython_module = False
1472
+
1473
+ # Keep track of whether we are the context manager of a 'with' statement
1474
+ in_context_manager_section = False
1475
+
1476
+ # One of 'prange' or 'with parallel'. This is used to disallow closely
1477
+ # nested 'with parallel:' blocks
1478
+ state = None
1479
+
1480
+ directive_to_node = {
1481
+ "cython.parallel.parallel": Nodes.ParallelWithBlockNode,
1482
+ # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
1483
+ "cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
1484
+ "cython.parallel.prange": Nodes.ParallelRangeNode,
1485
+ }
1486
+
1487
+ def node_is_parallel_directive(self, node):
1488
+ return node.name in self.parallel_directives or node.is_cython_module
1489
+
1490
+ def get_directive_class_node(self, node):
1491
+ """
1492
+ Figure out which parallel directive was used and return the associated
1493
+ Node class.
1494
+
1495
+ E.g. for a cython.parallel.prange() call we return ParallelRangeNode
1496
+ """
1497
+ if self.namenode_is_cython_module:
1498
+ directive = '.'.join(self.parallel_directive)
1499
+ else:
1500
+ directive = self.parallel_directives[self.parallel_directive[0]]
1501
+ directive = '%s.%s' % (directive,
1502
+ '.'.join(self.parallel_directive[1:]))
1503
+ directive = directive.rstrip('.')
1504
+
1505
+ cls = self.directive_to_node.get(directive)
1506
+ if cls is None and not (self.namenode_is_cython_module and
1507
+ self.parallel_directive[0] != 'parallel'):
1508
+ error(node.pos, "Invalid directive: %s" % directive)
1509
+
1510
+ self.namenode_is_cython_module = False
1511
+ self.parallel_directive = None
1512
+
1513
+ return cls
1514
+
1515
+ def visit_ModuleNode(self, node):
1516
+ """
1517
+ If any parallel directives were imported, copy them over and visit
1518
+ the AST
1519
+ """
1520
+ if node.parallel_directives:
1521
+ self.parallel_directives = node.parallel_directives
1522
+ return self.visit_Node(node)
1523
+
1524
+ # No parallel directives were imported, so they can't be used :)
1525
+ return node
1526
+
1527
+ def visit_NameNode(self, node):
1528
+ if self.node_is_parallel_directive(node):
1529
+ self.parallel_directive = [node.name]
1530
+ self.namenode_is_cython_module = node.is_cython_module
1531
+ return node
1532
+
1533
+ def visit_AttributeNode(self, node):
1534
+ self.visitchildren(node)
1535
+ if self.parallel_directive:
1536
+ self.parallel_directive.append(node.attribute)
1537
+ return node
1538
+
1539
+ def visit_CallNode(self, node):
1540
+ self.visitchild(node, 'function')
1541
+ if not self.parallel_directive:
1542
+ self.visitchildren(node, exclude=('function',))
1543
+ return node
1544
+
1545
+ # We are a parallel directive, replace this node with the
1546
+ # corresponding ParallelSomethingSomething node
1547
+
1548
+ if isinstance(node, ExprNodes.GeneralCallNode):
1549
+ args = node.positional_args.args
1550
+ kwargs = node.keyword_args
1551
+ else:
1552
+ args = node.args
1553
+ kwargs = {}
1554
+
1555
+ parallel_directive_class = self.get_directive_class_node(node)
1556
+ if parallel_directive_class:
1557
+ # Note: in case of a parallel() the body is set by
1558
+ # visit_WithStatNode
1559
+ node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)
1560
+
1561
+ return node
1562
+
1563
+ def visit_WithStatNode(self, node):
1564
+ "Rewrite with cython.parallel.parallel() blocks"
1565
+ newnode = self.visit(node.manager)
1566
+
1567
+ if isinstance(newnode, Nodes.ParallelWithBlockNode):
1568
+ if self.state == 'parallel with':
1569
+ error(node.manager.pos,
1570
+ "Nested parallel with blocks are disallowed")
1571
+
1572
+ self.state = 'parallel with'
1573
+ body = self.visitchild(node, 'body')
1574
+ self.state = None
1575
+
1576
+ newnode.body = body
1577
+ return newnode
1578
+ elif self.parallel_directive:
1579
+ parallel_directive_class = self.get_directive_class_node(node)
1580
+
1581
+ if not parallel_directive_class:
1582
+ # There was an error, stop here and now
1583
+ return None
1584
+
1585
+ if parallel_directive_class is Nodes.ParallelWithBlockNode:
1586
+ error(node.pos, "The parallel directive must be called")
1587
+ return None
1588
+
1589
+ self.visitchild(node, 'body')
1590
+ return node
1591
+
1592
+ def visit_ForInStatNode(self, node):
1593
+ "Rewrite 'for i in cython.parallel.prange(...):'"
1594
+ self.visitchild(node, 'iterator')
1595
+ self.visitchild(node, 'target')
1596
+
1597
+ in_prange = isinstance(node.iterator.sequence,
1598
+ Nodes.ParallelRangeNode)
1599
+ previous_state = self.state
1600
+
1601
+ if in_prange:
1602
+ # This will replace the entire ForInStatNode, so copy the
1603
+ # attributes
1604
+ parallel_range_node = node.iterator.sequence
1605
+
1606
+ parallel_range_node.target = node.target
1607
+ parallel_range_node.body = node.body
1608
+ parallel_range_node.else_clause = node.else_clause
1609
+
1610
+ node = parallel_range_node
1611
+
1612
+ if not isinstance(node.target, ExprNodes.NameNode):
1613
+ error(node.target.pos,
1614
+ "Can only iterate over an iteration variable")
1615
+
1616
+ self.state = 'prange'
1617
+
1618
+ self.visitchild(node, 'body')
1619
+ self.state = previous_state
1620
+ self.visitchild(node, 'else_clause')
1621
+ return node
1622
+
1623
+ def visit(self, node):
1624
+ "Visit a node that may be None"
1625
+ if node is not None:
1626
+ return super().visit(node)
1627
+
1628
+
1629
+ class WithTransform(VisitorTransform, SkipDeclarations):
1630
+ def visit_WithStatNode(self, node):
1631
+ self.visitchildren(node, ['body'])
1632
+ pos = node.pos
1633
+ is_async = node.is_async
1634
+ body, target, manager = node.body, node.target, node.manager
1635
+ manager = node.manager = ExprNodes.ProxyNode(manager)
1636
+ node.enter_call = ExprNodes.SimpleCallNode(
1637
+ pos, function=ExprNodes.AttributeNode(
1638
+ pos, obj=ExprNodes.CloneNode(manager),
1639
+ attribute=EncodedString('__aenter__' if is_async else '__enter__'),
1640
+ is_special_lookup=True),
1641
+ args=[],
1642
+ is_temp=True)
1643
+
1644
+ if is_async:
1645
+ node.enter_call = ExprNodes.AwaitExprNode(pos, arg=node.enter_call)
1646
+
1647
+ if target is not None:
1648
+ body = Nodes.StatListNode(
1649
+ pos, stats=[
1650
+ Nodes.WithTargetAssignmentStatNode(
1651
+ pos, lhs=target, with_node=node),
1652
+ body])
1653
+
1654
+ excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
1655
+ ExprNodes.ExcValueNode(pos) for _ in range(3)])
1656
+ except_clause = Nodes.ExceptClauseNode(
1657
+ pos, body=Nodes.IfStatNode(
1658
+ pos, if_clauses=[
1659
+ Nodes.IfClauseNode(
1660
+ pos, condition=ExprNodes.NotNode(
1661
+ pos, operand=ExprNodes.WithExitCallNode(
1662
+ pos, with_stat=node,
1663
+ test_if_run=False,
1664
+ args=excinfo_target,
1665
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1666
+ body=Nodes.ReraiseStatNode(pos),
1667
+ ),
1668
+ ],
1669
+ else_clause=None),
1670
+ pattern=None,
1671
+ target=None,
1672
+ excinfo_target=excinfo_target,
1673
+ )
1674
+
1675
+ node.body = Nodes.TryFinallyStatNode(
1676
+ pos, body=Nodes.TryExceptStatNode(
1677
+ pos, body=body,
1678
+ except_clauses=[except_clause],
1679
+ else_clause=None,
1680
+ ),
1681
+ finally_clause=Nodes.ExprStatNode(
1682
+ pos, expr=ExprNodes.WithExitCallNode(
1683
+ pos, with_stat=node,
1684
+ test_if_run=True,
1685
+ args=ExprNodes.TupleNode(
1686
+ pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]),
1687
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1688
+ handle_error_case=False,
1689
+ )
1690
+ return node
1691
+
1692
+ def visit_ExprNode(self, node):
1693
+ # With statements are never inside expressions.
1694
+ return node
1695
+
1696
+ visit_Node = VisitorTransform.recurse_to_children
1697
+
1698
+
1699
+ class _GeneratorExpressionArgumentsMarker(TreeVisitor, SkipDeclarations):
1700
+ # called from "MarkClosureVisitor"
1701
+ def __init__(self, gen_expr):
1702
+ super().__init__()
1703
+ self.gen_expr = gen_expr
1704
+
1705
+ def visit_ExprNode(self, node):
1706
+ if not node.is_literal:
1707
+ # Don't bother tagging literal nodes
1708
+ assert (not node.generator_arg_tag) # nobody has tagged this first
1709
+ node.generator_arg_tag = self.gen_expr
1710
+ self.visitchildren(node)
1711
+
1712
+ def visit_Node(self, node):
1713
+ # We're only interested in the expressions that make up the iterator sequence,
1714
+ # so don't go beyond ExprNodes (e.g. into ForFromStatNode).
1715
+ return
1716
+
1717
+ def visit_GeneratorExpressionNode(self, node):
1718
+ node.generator_arg_tag = self.gen_expr
1719
+ # don't visit children, can't handle overlapping tags
1720
+ # (and assume generator expressions don't end up optimized out in a way
1721
+ # that would require overlapping tags)
1722
+
1723
+
1724
+ class _HandleGeneratorArguments(VisitorTransform, SkipDeclarations):
1725
+ # used from within CreateClosureClasses
1726
+
1727
+ def __call__(self, node):
1728
+ from . import Visitor
1729
+ assert isinstance(node, ExprNodes.GeneratorExpressionNode)
1730
+ self.gen_node = node
1731
+
1732
+ self.args = list(node.def_node.args)
1733
+ self.call_parameters = list(node.call_parameters)
1734
+ self.tag_count = 0
1735
+ self.substitutions = {}
1736
+
1737
+ self.visitchildren(node)
1738
+
1739
+ for k, v in self.substitutions.items():
1740
+ # doing another search for replacements here (at the end) allows us to sweep up
1741
+ # CloneNodes too (which are often generated by the optimizer)
1742
+ # (it could arguably be done more efficiently with a single traversal though)
1743
+ Visitor.recursively_replace_node(node, k, v)
1744
+
1745
+ node.def_node.args = self.args
1746
+ node.call_parameters = self.call_parameters
1747
+ return node
1748
+
1749
+ def visit_GeneratorExpressionNode(self, node):
1750
+ # a generator can also be substituted itself, so handle that case
1751
+ new_node = self._handle_ExprNode(node, do_visit_children=False)
1752
+ # However do not traverse into it. A new _HandleGeneratorArguments visitor will be used
1753
+ # elsewhere to do that.
1754
+ return node
1755
+
1756
+ def _handle_ExprNode(self, node, do_visit_children):
1757
+ if (node.generator_arg_tag is not None and self.gen_node is not None and
1758
+ self.gen_node == node.generator_arg_tag):
1759
+ pos = node.pos
1760
+ # The reason for using ".x" as the name is that this is how CPython
1761
+ # tracks internal variables in loops (e.g.
1762
+ # { locals() for v in range(10) }
1763
+ # will produce "v" and ".0"). We don't replicate this behaviour completely
1764
+ # but use it as a starting point
1765
+ name_source = self.tag_count
1766
+ self.tag_count += 1
1767
+ name = EncodedString(".{}".format(name_source))
1768
+ def_node = self.gen_node.def_node
1769
+ if not def_node.local_scope.lookup_here(name):
1770
+ from . import Symtab
1771
+ cname = EncodedString(Naming.genexpr_arg_prefix + Symtab.punycodify_name(str(name_source)))
1772
+ name_decl = Nodes.CNameDeclaratorNode(pos=pos, name=name)
1773
+ type = node.type
1774
+
1775
+ # strip away cv types - they shouldn't be applied to the
1776
+ # function argument or to the closure struct.
1777
+ # It isn't obvious whether the right thing to do would be to capture by reference or by
1778
+ # value (C++ itself doesn't know either for lambda functions and forces a choice).
1779
+ # However, capture by reference involves converting to FakeReference which would require
1780
+ # re-analysing AttributeNodes. Therefore I've picked capture-by-value out of convenience
1781
+ # TODO - could probably be optimized by making the arg a reference but the closure not
1782
+ # (see https://github.com/cython/cython/issues/2468)
1783
+ type = PyrexTypes.remove_cv_ref(type, remove_fakeref=False)
1784
+
1785
+ name_decl.type = type
1786
+ new_arg = Nodes.CArgDeclNode(pos=pos, declarator=name_decl,
1787
+ base_type=None, default=None, annotation=None)
1788
+ new_arg.name = name_decl.name
1789
+ new_arg.type = type
1790
+
1791
+ self.args.append(new_arg)
1792
+ node.generator_arg_tag = None # avoid the possibility of this being caught again
1793
+ self.call_parameters.append(node)
1794
+ new_arg.entry = def_node.declare_argument(def_node.local_scope, new_arg)
1795
+ new_arg.entry.cname = cname
1796
+ new_arg.entry.in_closure = True
1797
+
1798
+ if do_visit_children:
1799
+ # now visit the Nodes's children (but remove self.gen_node to not to further
1800
+ # argument substitution)
1801
+ gen_node, self.gen_node = self.gen_node, None
1802
+ self.visitchildren(node)
1803
+ self.gen_node = gen_node
1804
+
1805
+ # replace the node inside the generator with a looked-up name
1806
+ # (initialized_check can safely be False because the source variable will be checked
1807
+ # before it is captured if the check is required)
1808
+ name_node = ExprNodes.NameNode(pos, name=name, initialized_check=False)
1809
+ name_node.entry = self.gen_node.def_node.gbody.local_scope.lookup(name_node.name)
1810
+ name_node.type = name_node.entry.type
1811
+ self.substitutions[node] = name_node
1812
+ return name_node
1813
+ if do_visit_children:
1814
+ self.visitchildren(node)
1815
+ return node
1816
+
1817
+ def visit_ExprNode(self, node):
1818
+ return self._handle_ExprNode(node, True)
1819
+
1820
+ visit_Node = VisitorTransform.recurse_to_children
1821
+
1822
+
1823
+ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
1824
+ """
1825
+ Transforms method decorators in cdef classes into nested calls or properties.
1826
+
1827
+ Python-style decorator properties are transformed into a PropertyNode
1828
+ with up to the three getter, setter and deleter DefNodes.
1829
+ The functional style isn't supported yet.
1830
+ """
1831
+ _properties = None
1832
+
1833
+ _map_property_attribute = {
1834
+ 'getter': EncodedString('__get__'),
1835
+ 'setter': EncodedString('__set__'),
1836
+ 'deleter': EncodedString('__del__'),
1837
+ }.get
1838
+
1839
+ def visit_CClassDefNode(self, node):
1840
+ if self._properties is None:
1841
+ self._properties = []
1842
+ self._properties.append({})
1843
+ node = super().visit_CClassDefNode(node)
1844
+ self._properties.pop()
1845
+ return node
1846
+
1847
+ def visit_PropertyNode(self, node):
1848
+ # Low-level warning for other code until we can convert all our uses over.
1849
+ level = 2 if isinstance(node.pos[0], str) else 0
1850
+ warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level)
1851
+ return node
1852
+
1853
+ def visit_CFuncDefNode(self, node):
1854
+ node = self.visit_FuncDefNode(node)
1855
+ if not node.decorators:
1856
+ return node
1857
+ elif self.scope_type != 'cclass' or self.scope_node.visibility != "extern":
1858
+ # at the moment cdef functions are very restricted in what decorators they can take
1859
+ # so it's simple to test for the small number of allowed decorators....
1860
+ if not (len(node.decorators) == 1 and node.decorators[0].decorator.is_name and
1861
+ node.decorators[0].decorator.name == "staticmethod"):
1862
+ error(node.decorators[0].pos, "Cdef functions cannot take arbitrary decorators.")
1863
+ return node
1864
+
1865
+ ret_node = node
1866
+ decorator_node = self._find_property_decorator(node)
1867
+ if decorator_node:
1868
+ if decorator_node.decorator.is_name:
1869
+ name = node.declared_name()
1870
+ if name:
1871
+ ret_node = self._add_property(node, name, decorator_node)
1872
+ else:
1873
+ error(decorator_node.pos, "C property decorator can only be @property")
1874
+
1875
+ if node.decorators:
1876
+ return self._reject_decorated_property(node, node.decorators[0])
1877
+ return ret_node
1878
+
1879
+ def visit_DefNode(self, node):
1880
+ scope_type = self.scope_type
1881
+ node = self.visit_FuncDefNode(node)
1882
+ if scope_type != 'cclass' or not node.decorators:
1883
+ return node
1884
+
1885
+ # transform @property decorators
1886
+ decorator_node = self._find_property_decorator(node)
1887
+ if decorator_node is not None:
1888
+ decorator = decorator_node.decorator
1889
+ if decorator.is_name:
1890
+ return self._add_property(node, node.name, decorator_node)
1891
+ else:
1892
+ handler_name = self._map_property_attribute(decorator.attribute)
1893
+ if handler_name:
1894
+ if decorator.obj.name != node.name:
1895
+ # CPython does not generate an error or warning, but not something useful either.
1896
+ error(decorator_node.pos,
1897
+ "Mismatching property names, expected '%s', got '%s'" % (
1898
+ decorator.obj.name, node.name))
1899
+ elif len(node.decorators) > 1:
1900
+ return self._reject_decorated_property(node, decorator_node)
1901
+ else:
1902
+ return self._add_to_property(node, handler_name, decorator_node)
1903
+
1904
+ # we clear node.decorators, so we need to set the
1905
+ # is_staticmethod/is_classmethod attributes now
1906
+ for decorator in node.decorators:
1907
+ func = decorator.decorator
1908
+ if func.is_name:
1909
+ node.is_classmethod |= func.name == 'classmethod'
1910
+ node.is_staticmethod |= func.name == 'staticmethod'
1911
+
1912
+ # transform normal decorators
1913
+ decs = node.decorators
1914
+ node.decorators = None
1915
+ return self.chain_decorators(node, decs, node.name)
1916
+
1917
+ def _find_property_decorator(self, node):
1918
+ properties = self._properties[-1]
1919
+ for decorator_node in node.decorators[::-1]:
1920
+ decorator = decorator_node.decorator
1921
+ if decorator.is_name and decorator.name == 'property':
1922
+ # @property
1923
+ return decorator_node
1924
+ elif decorator.is_attribute and decorator.obj.name in properties:
1925
+ # @prop.setter etc.
1926
+ return decorator_node
1927
+ return None
1928
+
1929
+ @staticmethod
1930
+ def _reject_decorated_property(node, decorator_node):
1931
+ # restrict transformation to outermost decorator as wrapped properties will probably not work
1932
+ for deco in node.decorators:
1933
+ if deco != decorator_node:
1934
+ error(deco.pos, "Property methods with additional decorators are not supported")
1935
+ return node
1936
+
1937
+ def _add_property(self, node, name, decorator_node):
1938
+ if len(node.decorators) > 1:
1939
+ return self._reject_decorated_property(node, decorator_node)
1940
+ node.decorators.remove(decorator_node)
1941
+ properties = self._properties[-1]
1942
+ is_cproperty = isinstance(node, Nodes.CFuncDefNode)
1943
+ body = Nodes.StatListNode(node.pos, stats=[node])
1944
+ if is_cproperty:
1945
+ if name in properties:
1946
+ error(node.pos, "C property redeclared")
1947
+ if 'inline' not in node.modifiers:
1948
+ error(node.pos, "C property method must be declared 'inline'")
1949
+ prop = Nodes.CPropertyNode(node.pos, doc=node.doc, name=name, body=body)
1950
+ elif name in properties:
1951
+ prop = properties[name]
1952
+ if prop.is_cproperty:
1953
+ error(node.pos, "C property redeclared")
1954
+ else:
1955
+ node.name = EncodedString("__get__")
1956
+ prop.pos = node.pos
1957
+ prop.doc = node.doc
1958
+ prop.body.stats = [node]
1959
+ return None
1960
+ else:
1961
+ node.name = EncodedString("__get__")
1962
+ prop = Nodes.PropertyNode(
1963
+ node.pos, name=name, doc=node.doc, body=body)
1964
+ properties[name] = prop
1965
+ return prop
1966
+
1967
+ def _add_to_property(self, node, name, decorator):
1968
+ properties = self._properties[-1]
1969
+ prop = properties[node.name]
1970
+ if prop.is_cproperty:
1971
+ error(node.pos, "C property redeclared")
1972
+ return None
1973
+ node.name = name
1974
+ node.decorators.remove(decorator)
1975
+ stats = prop.body.stats
1976
+ for i, stat in enumerate(stats):
1977
+ if stat.name == name:
1978
+ stats[i] = node
1979
+ break
1980
+ else:
1981
+ stats.append(node)
1982
+ return None
1983
+
1984
+ @staticmethod
1985
+ def chain_decorators(node, decorators, name):
1986
+ """
1987
+ Decorators are applied directly in DefNode and PyClassDefNode to avoid
1988
+ reassignments to the function/class name - except for cdef class methods.
1989
+ For those, the reassignment is required as methods are originally
1990
+ defined in the PyMethodDef struct.
1991
+
1992
+ The IndirectionNode allows DefNode to override the decorator.
1993
+ """
1994
+ decorator_result = ExprNodes.NameNode(node.pos, name=name)
1995
+ for decorator in decorators[::-1]:
1996
+ decorator_result = ExprNodes.SimpleCallNode(
1997
+ decorator.pos,
1998
+ function=decorator.decorator,
1999
+ args=[decorator_result])
2000
+
2001
+ name_node = ExprNodes.NameNode(node.pos, name=name)
2002
+ reassignment = Nodes.SingleAssignmentNode(
2003
+ node.pos,
2004
+ lhs=name_node,
2005
+ rhs=decorator_result)
2006
+
2007
+ reassignment = Nodes.IndirectionNode([reassignment])
2008
+ node.decorator_indirection = reassignment
2009
+ return [node, reassignment]
2010
+
2011
+
2012
+ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
2013
+ """
2014
+ Only part of the CythonUtilityCode pipeline. Must be run before
2015
+ DecoratorTransform in case this is a decorator for a cdef class.
2016
+ It filters out @cname('my_cname') decorators and rewrites them to
2017
+ CnameDecoratorNodes.
2018
+ """
2019
+
2020
+ def handle_function(self, node):
2021
+ if not getattr(node, 'decorators', None):
2022
+ return self.visit_Node(node)
2023
+
2024
+ for i, decorator in enumerate(node.decorators):
2025
+ decorator = decorator.decorator
2026
+
2027
+ if (isinstance(decorator, ExprNodes.CallNode) and
2028
+ decorator.function.is_name and
2029
+ decorator.function.name == 'cname'):
2030
+ args, kwargs = decorator.explicit_args_kwds()
2031
+
2032
+ if kwargs:
2033
+ raise AssertionError(
2034
+ "cname decorator does not take keyword arguments")
2035
+
2036
+ if len(args) != 1:
2037
+ raise AssertionError(
2038
+ "cname decorator takes exactly one argument")
2039
+
2040
+ if not (args[0].is_literal and args[0].type is Builtin.unicode_type):
2041
+ raise AssertionError(
2042
+ "argument to cname decorator must be a string literal")
2043
+
2044
+ cname = args[0].compile_time_value(None)
2045
+ del node.decorators[i]
2046
+ node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
2047
+ cname=cname)
2048
+ break
2049
+
2050
+ return self.visit_Node(node)
2051
+
2052
+ visit_FuncDefNode = handle_function
2053
+ visit_CClassDefNode = handle_function
2054
+ visit_CEnumDefNode = handle_function
2055
+ visit_CStructOrUnionDefNode = handle_function
2056
+ visit_CVarDefNode = handle_function
2057
+
2058
+
2059
+ class ForwardDeclareTypes(CythonTransform):
2060
+ """
2061
+ Declare all global cdef names that we allow referencing in other places,
2062
+ before declaring everything (else) in source code order.
2063
+ """
2064
+
2065
+ def visit_CompilerDirectivesNode(self, node):
2066
+ env = self.module_scope
2067
+ old = env.directives
2068
+ env.directives = node.directives
2069
+ self.visitchildren(node)
2070
+ env.directives = old
2071
+ return node
2072
+
2073
+ def visit_ModuleNode(self, node):
2074
+ self.module_scope = node.scope
2075
+ self.module_scope.directives = node.directives
2076
+ self.visitchildren(node)
2077
+ return node
2078
+
2079
+ def visit_CDefExternNode(self, node):
2080
+ old_cinclude_flag = self.module_scope.in_cinclude
2081
+ self.module_scope.in_cinclude = 1
2082
+ self.visitchildren(node)
2083
+ self.module_scope.in_cinclude = old_cinclude_flag
2084
+ return node
2085
+
2086
+ def visit_CEnumDefNode(self, node):
2087
+ node.declare(self.module_scope)
2088
+ return node
2089
+
2090
+ def visit_CStructOrUnionDefNode(self, node):
2091
+ if node.name not in self.module_scope.entries:
2092
+ node.declare(self.module_scope)
2093
+ return node
2094
+
2095
+ def visit_CClassDefNode(self, node):
2096
+ if node.class_name not in self.module_scope.entries:
2097
+ node.declare(self.module_scope)
2098
+ # Expand fused methods of .pxd declared types to construct the final vtable order.
2099
+ type = self.module_scope.entries[node.class_name].type
2100
+ if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope:
2101
+ scope = type.scope
2102
+ for entry in scope.cfunc_entries:
2103
+ if entry.type and entry.type.is_fused:
2104
+ entry.type.get_all_specialized_function_types()
2105
+ return node
2106
+
2107
+ def visit_FuncDefNode(self, node):
2108
+ # no traversal needed
2109
+ return node
2110
+
2111
+ def visit_PyClassDefNode(self, node):
2112
+ # no traversal needed
2113
+ return node
2114
+
2115
+
2116
+ class AnalyseDeclarationsTransform(EnvTransform):
2117
+
2118
+ basic_property = TreeFragment("""
2119
+ property NAME:
2120
+ def __get__(self):
2121
+ return ATTR
2122
+ def __set__(self, value):
2123
+ ATTR = value
2124
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2125
+ basic_pyobject_property = TreeFragment("""
2126
+ property NAME:
2127
+ def __get__(self):
2128
+ return ATTR
2129
+ def __set__(self, value):
2130
+ ATTR = value
2131
+ def __del__(self):
2132
+ ATTR = None
2133
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2134
+ basic_property_ro = TreeFragment("""
2135
+ property NAME:
2136
+ def __get__(self):
2137
+ return ATTR
2138
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2139
+
2140
+ struct_or_union_wrapper = TreeFragment("""
2141
+ cdef class NAME:
2142
+ cdef TYPE value
2143
+ def __init__(self, MEMBER=None):
2144
+ cdef int count
2145
+ count = 0
2146
+ INIT_ASSIGNMENTS
2147
+ if IS_UNION and count > 1:
2148
+ raise ValueError, "At most one union member should be specified."
2149
+ def __str__(self):
2150
+ return STR_FORMAT % MEMBER_TUPLE
2151
+ def __repr__(self):
2152
+ return REPR_FORMAT % MEMBER_TUPLE
2153
+ """, pipeline=[NormalizeTree(None)])
2154
+
2155
+ init_assignment = TreeFragment("""
2156
+ if VALUE is not None:
2157
+ ATTR = VALUE
2158
+ count += 1
2159
+ """, pipeline=[NormalizeTree(None)])
2160
+
2161
+ fused_function = None
2162
+ in_lambda = 0
2163
+
2164
+ def __call__(self, root):
2165
+ # needed to determine if a cdef var is declared after it's used.
2166
+ self.seen_vars_stack = []
2167
+ self.fused_error_funcs = set()
2168
+ super_class = super()
2169
+ self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
2170
+ return super_class.__call__(root)
2171
+
2172
+ def visit_NameNode(self, node):
2173
+ self.seen_vars_stack[-1].add(node.name)
2174
+ return node
2175
+
2176
+ def visit_ModuleNode(self, node):
2177
+ # Pickling support requires injecting module-level nodes.
2178
+ self.extra_module_declarations = []
2179
+ self.seen_vars_stack.append(set())
2180
+ node.analyse_declarations(self.current_env())
2181
+ self.visitchildren(node)
2182
+ self.seen_vars_stack.pop()
2183
+ node.body.stats.extend(self.extra_module_declarations)
2184
+ return node
2185
+
2186
+ def visit_LambdaNode(self, node):
2187
+ self.in_lambda += 1
2188
+ node.analyse_declarations(self.current_env())
2189
+ self.visitchildren(node)
2190
+ self.in_lambda -= 1
2191
+ return node
2192
+
2193
+ def visit_CClassDefNode(self, node):
2194
+ node = self.visit_ClassDefNode(node)
2195
+ if node.scope and 'dataclasses.dataclass' in node.scope.directives:
2196
+ from .Dataclass import handle_cclass_dataclass
2197
+ handle_cclass_dataclass(node, node.scope.directives['dataclasses.dataclass'], self)
2198
+ if node.scope and node.scope.implemented and node.body:
2199
+ stats = []
2200
+ for entry in node.scope.var_entries:
2201
+ if entry.needs_property:
2202
+ property = self.create_Property(entry)
2203
+ property.analyse_declarations(node.scope)
2204
+ self.visit(property)
2205
+ stats.append(property)
2206
+ if stats:
2207
+ node.body.stats += stats
2208
+ if (node.visibility != 'extern'
2209
+ and not node.scope.lookup('__reduce__')
2210
+ and not node.scope.lookup('__reduce_ex__')):
2211
+ self._inject_pickle_methods(node)
2212
+ return node
2213
+
2214
+ def _inject_pickle_methods(self, node):
2215
+ env = self.current_env()
2216
+ if node.scope.directives['auto_pickle'] is False: # None means attempt it.
2217
+ # Old behavior of not doing anything.
2218
+ return
2219
+ auto_pickle_forced = node.scope.directives['auto_pickle'] is True
2220
+
2221
+ all_members = []
2222
+ cls = node.entry.type
2223
+ cinit = None
2224
+ inherited_reduce = None
2225
+ while cls is not None:
2226
+ all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__'))
2227
+ cinit = cinit or cls.scope.lookup('__cinit__')
2228
+ inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__')
2229
+ cls = cls.base_type
2230
+ all_members.sort(key=lambda e: e.name)
2231
+
2232
+ if inherited_reduce:
2233
+ # This is not failsafe, as we may not know whether a cimported class defines a __reduce__.
2234
+ # This is why we define __reduce_cython__ and only replace __reduce__
2235
+ # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation.
2236
+ return
2237
+
2238
+ non_py = [
2239
+ e for e in all_members
2240
+ if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env)
2241
+ or not e.type.can_coerce_from_pyobject(env))
2242
+ ]
2243
+
2244
+ structs = [e for e in all_members if e.type.is_struct_or_union]
2245
+
2246
+ if cinit or non_py or (structs and not auto_pickle_forced):
2247
+ if cinit:
2248
+ # TODO(robertwb): We could allow this if __cinit__ has no require arguments.
2249
+ msg = 'no default __reduce__ due to non-trivial __cinit__'
2250
+ elif non_py:
2251
+ msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py)
2252
+ else:
2253
+ # Extern structs may be only partially defined.
2254
+ # TODO(robertwb): Limit the restriction to extern
2255
+ # (and recursively extern-containing) structs.
2256
+ msg = ("Pickling of struct members such as %s must be explicitly requested "
2257
+ "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs))
2258
+
2259
+ if auto_pickle_forced:
2260
+ error(node.pos, msg)
2261
+
2262
+ pickle_func = TreeFragment("""
2263
+ def __reduce_cython__(self):
2264
+ raise TypeError, "%(msg)s"
2265
+ def __setstate_cython__(self, __pyx_state):
2266
+ raise TypeError, "%(msg)s"
2267
+ """ % {'msg': msg},
2268
+ level='c_class', pipeline=[NormalizeTree(None)]).substitute({})
2269
+ pickle_func.analyse_declarations(node.scope)
2270
+ self.visit(pickle_func)
2271
+ node.body.stats.append(pickle_func)
2272
+
2273
+ else:
2274
+ for e in all_members:
2275
+ if not e.type.is_pyobject:
2276
+ e.type.create_to_py_utility_code(env)
2277
+ e.type.create_from_py_utility_code(env)
2278
+
2279
+ all_members_names = [e.name for e in all_members]
2280
+ checksums = _calculate_pickle_checksums(all_members_names)
2281
+
2282
+ unpickle_func_name = '__pyx_unpickle_%s' % node.punycode_class_name
2283
+
2284
+ # TODO(robertwb): Move the state into the third argument
2285
+ # so it can be pickled *after* self is memoized.
2286
+ unpickle_func = TreeFragment("""
2287
+ def %(unpickle_func_name)s(__pyx_type, long __pyx_checksum, __pyx_state):
2288
+ cdef object __pyx_PickleError
2289
+ cdef object __pyx_result
2290
+ if __pyx_checksum not in %(checksums)s:
2291
+ from pickle import PickleError as __pyx_PickleError
2292
+ raise __pyx_PickleError, "Incompatible checksums (0x%%x vs %(checksums)s = (%(members)s))" %% __pyx_checksum
2293
+ __pyx_result = %(class_name)s.__new__(__pyx_type)
2294
+ if __pyx_state is not None:
2295
+ %(unpickle_func_name)s__set_state(<%(class_name)s> __pyx_result, __pyx_state)
2296
+ return __pyx_result
2297
+
2298
+ cdef %(unpickle_func_name)s__set_state(%(class_name)s __pyx_result, tuple __pyx_state):
2299
+ %(assignments)s
2300
+ if len(__pyx_state) > %(num_members)d and hasattr(__pyx_result, '__dict__'):
2301
+ __pyx_result.__dict__.update(__pyx_state[%(num_members)d])
2302
+ """ % {
2303
+ 'unpickle_func_name': unpickle_func_name,
2304
+ 'checksums': "(%s)" % ', '.join(checksums),
2305
+ 'members': ', '.join(all_members_names),
2306
+ 'class_name': node.class_name,
2307
+ 'assignments': '; '.join(
2308
+ '__pyx_result.%s = __pyx_state[%s]' % (v, ix)
2309
+ for ix, v in enumerate(all_members_names)),
2310
+ 'num_members': len(all_members_names),
2311
+ }, level='module', pipeline=[NormalizeTree(None)]).substitute({})
2312
+ unpickle_func.analyse_declarations(node.entry.scope)
2313
+ self.visit(unpickle_func)
2314
+ self.extra_module_declarations.append(unpickle_func)
2315
+
2316
+ pickle_func = TreeFragment("""
2317
+ def __reduce_cython__(self):
2318
+ cdef tuple state
2319
+ cdef object _dict
2320
+ cdef bint use_setstate
2321
+ state = (%(members)s)
2322
+ _dict = getattr(self, '__dict__', None)
2323
+ if _dict is not None:
2324
+ state += (_dict,)
2325
+ use_setstate = True
2326
+ else:
2327
+ use_setstate = %(any_notnone_members)s
2328
+ if use_setstate:
2329
+ return %(unpickle_func_name)s, (type(self), %(checksum)s, None), state
2330
+ else:
2331
+ return %(unpickle_func_name)s, (type(self), %(checksum)s, state)
2332
+
2333
+ def __setstate_cython__(self, __pyx_state):
2334
+ %(unpickle_func_name)s__set_state(self, __pyx_state)
2335
+ """ % {
2336
+ 'unpickle_func_name': unpickle_func_name,
2337
+ 'checksum': checksums[0],
2338
+ 'members': ', '.join('self.%s' % v for v in all_members_names) + (',' if len(all_members_names) == 1 else ''),
2339
+ # Even better, we could check PyType_IS_GC.
2340
+ 'any_notnone_members' : ' or '.join(['self.%s is not None' % e.name for e in all_members if e.type.is_pyobject] or ['False']),
2341
+ },
2342
+ level='c_class', pipeline=[NormalizeTree(None)]).substitute({})
2343
+ pickle_func.analyse_declarations(node.scope)
2344
+ self.enter_scope(node, node.scope) # functions should be visited in the class scope
2345
+ self.visit(pickle_func)
2346
+ self.exit_scope()
2347
+ node.body.stats.append(pickle_func)
2348
+
2349
+ def _handle_fused_def_decorators(self, old_decorators, env, node):
2350
+ """
2351
+ Create function calls to the decorators and reassignments to
2352
+ the function.
2353
+ """
2354
+ # Delete staticmethod and classmethod decorators, this is
2355
+ # handled directly by the fused function object.
2356
+ decorators = []
2357
+ for decorator in old_decorators:
2358
+ func = decorator.decorator
2359
+ if (not func.is_name or
2360
+ func.name not in ('staticmethod', 'classmethod') or
2361
+ env.lookup_here(func.name)):
2362
+ # not a static or classmethod
2363
+ decorators.append(decorator)
2364
+
2365
+ if decorators:
2366
+ transform = DecoratorTransform(self.context)
2367
+ def_node = node.node
2368
+ _, reassignments = transform.chain_decorators(
2369
+ def_node, decorators, def_node.name)
2370
+ reassignments.analyse_declarations(env)
2371
+ node = [node, reassignments]
2372
+
2373
+ return node
2374
+
2375
+ def _handle_def(self, decorators, env, node):
2376
+ "Handle def or cpdef fused functions"
2377
+ # Create PyCFunction nodes for each specialization
2378
+ node.stats.insert(0, node.py_func)
2379
+ self.visitchild(node, 'py_func')
2380
+ node.update_fused_defnode_entry(env)
2381
+ # For the moment, fused functions do not support METH_FASTCALL
2382
+ node.py_func.entry.signature.use_fastcall = False
2383
+ pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True)
2384
+ pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
2385
+ node.resulting_fused_function = pycfunc
2386
+ # Create assignment node for our def function
2387
+ node.fused_func_assignment = self._create_assignment(
2388
+ node.py_func, ExprNodes.CloneNode(pycfunc), env)
2389
+
2390
+ if decorators:
2391
+ node = self._handle_fused_def_decorators(decorators, env, node)
2392
+
2393
+ return node
2394
+
2395
+ def _create_fused_function(self, env, node):
2396
+ "Create a fused function for a DefNode with fused arguments"
2397
+ from . import FusedNode
2398
+
2399
+ if self.fused_function or self.in_lambda:
2400
+ if self.fused_function not in self.fused_error_funcs:
2401
+ if self.in_lambda:
2402
+ error(node.pos, "Fused lambdas not allowed")
2403
+ else:
2404
+ error(node.pos, "Cannot nest fused functions")
2405
+
2406
+ self.fused_error_funcs.add(self.fused_function)
2407
+
2408
+ node.body = Nodes.PassStatNode(node.pos)
2409
+ for arg in node.args:
2410
+ if arg.type.is_fused:
2411
+ arg.type = arg.type.get_fused_types()[0]
2412
+
2413
+ return node
2414
+
2415
+ decorators = getattr(node, 'decorators', None)
2416
+ node = FusedNode.FusedCFuncDefNode(node, env)
2417
+ self.fused_function = node
2418
+ self.visitchildren(node)
2419
+ self.fused_function = None
2420
+ if node.py_func:
2421
+ node = self._handle_def(decorators, env, node)
2422
+
2423
+ return node
2424
+
2425
+ def _handle_fused(self, node):
2426
+ if node.is_generator and node.has_fused_arguments:
2427
+ error(node.pos, "Fused generators not supported")
2428
+ node.has_fused_arguments = False
2429
+ node.gbody.body = Nodes.StatListNode(node.pos, stats=[])
2430
+
2431
+ return node.has_fused_arguments
2432
+
2433
+ def visit_FuncDefNode(self, node):
2434
+ """
2435
+ Analyse a function and its body, as that hasn't happened yet. Also
2436
+ analyse the directive_locals set by @cython.locals().
2437
+
2438
+ Then, if we are a function with fused arguments, replace the function
2439
+ (after it has declared itself in the symbol table!) with a
2440
+ FusedCFuncDefNode, and analyse its children (which are in turn normal
2441
+ functions). If we're a normal function, just analyse the body of the
2442
+ function.
2443
+ """
2444
+ env = self.current_env()
2445
+
2446
+ self.seen_vars_stack.append(set())
2447
+ lenv = node.local_scope
2448
+ node.declare_arguments(lenv)
2449
+
2450
+ # @cython.locals(...)
2451
+ for var, type_node in node.directive_locals.items():
2452
+ if not lenv.lookup_here(var): # don't redeclare args
2453
+ type = type_node.analyse_as_type(lenv)
2454
+ if type and type.is_fused and lenv.fused_to_specific:
2455
+ type = type.specialize(lenv.fused_to_specific)
2456
+ if type:
2457
+ lenv.declare_var(var, type, type_node.pos)
2458
+ else:
2459
+ error(type_node.pos, "Not a type")
2460
+
2461
+ if self._handle_fused(node):
2462
+ node = self._create_fused_function(env, node)
2463
+ else:
2464
+ node.body.analyse_declarations(lenv)
2465
+ node = self._super_visit_FuncDefNode(node)
2466
+
2467
+ self.seen_vars_stack.pop()
2468
+
2469
+ if "ufunc" in lenv.directives:
2470
+ from . import UFuncs
2471
+ return UFuncs.convert_to_ufunc(node)
2472
+ return node
2473
+
2474
+ def visit_DefNode(self, node):
2475
+ node = self.visit_FuncDefNode(node)
2476
+ if not isinstance(node, Nodes.DefNode):
2477
+ return node
2478
+ env = self.current_env()
2479
+ if node.code_object is None:
2480
+ node.code_object = ExprNodes.CodeObjectNode(node)
2481
+ node.code_object.analyse_declarations(env)
2482
+ if node.fused_py_func or node.is_generator_body:
2483
+ return node
2484
+ if not node.needs_assignment_synthesis(env):
2485
+ return node
2486
+ return [node, self._synthesize_assignment(node, env)]
2487
+
2488
+ def visit_CFuncDefNode(self, node):
2489
+ if node.code_object is None and node.py_func is None:
2490
+ node.code_object = ExprNodes.CodeObjectNode.for_cfunc(node)
2491
+ node.code_object.analyse_declarations(self.current_env())
2492
+ return self.visit_FuncDefNode(node)
2493
+
2494
+ def visit_GeneratorBodyDefNode(self, node):
2495
+ return self.visit_FuncDefNode(node)
2496
+
2497
+ def visit_GeneratorDefNode(self, node):
2498
+ # The generator body should use the same code object as the (user facing) generator function that creates it.
2499
+ result = self.visit_DefNode(node)
2500
+ # 'result' will usually be a list of statements, but we still have the original node.
2501
+ node.gbody.code_object = node.code_object
2502
+ return result
2503
+
2504
+ def _synthesize_assignment(self, node, env):
2505
+ # Synthesize assignment node and put it right after defnode
2506
+ genv = env
2507
+ while genv.is_py_class_scope or genv.is_c_class_scope:
2508
+ genv = genv.outer_scope
2509
+
2510
+ binding = env.is_py_class_scope or self.current_directives.get('binding')
2511
+ if genv.is_closure_scope:
2512
+ rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode.from_defnode(node, binding)
2513
+ else:
2514
+ rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
2515
+
2516
+ node.is_cyfunction = rhs.binding
2517
+ return self._create_assignment(node, rhs, env)
2518
+
2519
+ def _create_assignment(self, def_node, rhs, env):
2520
+ if def_node.decorators:
2521
+ for decorator in def_node.decorators[::-1]:
2522
+ rhs = ExprNodes.SimpleCallNode(
2523
+ decorator.pos,
2524
+ function = decorator.decorator,
2525
+ args = [rhs])
2526
+ def_node.decorators = None
2527
+
2528
+ assmt = Nodes.SingleAssignmentNode(
2529
+ def_node.pos,
2530
+ lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
2531
+ rhs=rhs)
2532
+ assmt.analyse_declarations(env)
2533
+ return assmt
2534
+
2535
+ def visit_func_outer_attrs(self, node):
2536
+ # any names in the outer attrs should not be looked up in the function "seen_vars_stack"
2537
+ stack = self.seen_vars_stack.pop()
2538
+ super().visit_func_outer_attrs(node)
2539
+ self.seen_vars_stack.append(stack)
2540
+
2541
+ def visit_ScopedExprNode(self, node):
2542
+ env = self.current_env()
2543
+ node.analyse_declarations(env)
2544
+ # the node may or may not have a local scope
2545
+ if node.expr_scope:
2546
+ self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
2547
+ self.enter_scope(node, node.expr_scope)
2548
+ node.analyse_scoped_declarations(node.expr_scope)
2549
+ self.visitchildren(node)
2550
+ self.exit_scope()
2551
+ self.seen_vars_stack.pop()
2552
+ else:
2553
+
2554
+ node.analyse_scoped_declarations(env)
2555
+ self.visitchildren(node)
2556
+ return node
2557
+
2558
+ def visit_TempResultFromStatNode(self, node):
2559
+ self.visitchildren(node)
2560
+ node.analyse_declarations(self.current_env())
2561
+ return node
2562
+
2563
+ def visit_CppClassNode(self, node):
2564
+ if node.visibility == 'extern':
2565
+ return None
2566
+ else:
2567
+ return self.visit_ClassDefNode(node)
2568
+
2569
+ def visit_CStructOrUnionDefNode(self, node):
2570
+ # Create a wrapper node if needed.
2571
+ # We want to use the struct type information (so it can't happen
2572
+ # before this phase) but also create new objects to be declared
2573
+ # (so it can't happen later).
2574
+ # Note that we don't return the original node, as it is
2575
+ # never used after this phase.
2576
+ if True: # private (default)
2577
+ return None
2578
+
2579
+ self_value = ExprNodes.AttributeNode(
2580
+ pos = node.pos,
2581
+ obj = ExprNodes.NameNode(pos=node.pos, name="self"),
2582
+ attribute = EncodedString("value"))
2583
+ var_entries = node.entry.type.scope.var_entries
2584
+ attributes = []
2585
+ for entry in var_entries:
2586
+ attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
2587
+ obj = self_value,
2588
+ attribute = entry.name))
2589
+ # __init__ assignments
2590
+ init_assignments = []
2591
+ for entry, attr in zip(var_entries, attributes):
2592
+ # TODO: branch on visibility
2593
+ init_assignments.append(self.init_assignment.substitute({
2594
+ "VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
2595
+ "ATTR": attr,
2596
+ }, pos = entry.pos))
2597
+
2598
+ # create the class
2599
+ str_format = "%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
2600
+ wrapper_class = self.struct_or_union_wrapper.substitute({
2601
+ "INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
2602
+ "IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
2603
+ "MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
2604
+ "STR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format)),
2605
+ "REPR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
2606
+ }, pos = node.pos).stats[0]
2607
+ wrapper_class.class_name = node.name
2608
+ wrapper_class.shadow = True
2609
+ class_body = wrapper_class.body.stats
2610
+
2611
+ # fix value type
2612
+ assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
2613
+ class_body[0].base_type.name = node.name
2614
+
2615
+ # fix __init__ arguments
2616
+ init_method = class_body[1]
2617
+ assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
2618
+ arg_template = init_method.args[1]
2619
+ if not node.entry.type.is_struct:
2620
+ arg_template.kw_only = True
2621
+ del init_method.args[1]
2622
+ for entry, attr in zip(var_entries, attributes):
2623
+ arg = copy.deepcopy(arg_template)
2624
+ arg.declarator.name = entry.name
2625
+ init_method.args.append(arg)
2626
+
2627
+ # setters/getters
2628
+ for entry, attr in zip(var_entries, attributes):
2629
+ # TODO: branch on visibility
2630
+ if entry.type.is_pyobject:
2631
+ template = self.basic_pyobject_property
2632
+ else:
2633
+ template = self.basic_property
2634
+ property = template.substitute({
2635
+ "ATTR": attr,
2636
+ }, pos = entry.pos).stats[0]
2637
+ property.name = entry.name
2638
+ wrapper_class.body.stats.append(property)
2639
+
2640
+ wrapper_class.analyse_declarations(self.current_env())
2641
+ return self.visit_CClassDefNode(wrapper_class)
2642
+
2643
+ # Some nodes are no longer needed after declaration
2644
+ # analysis and can be dropped. The analysis was performed
2645
+ # on these nodes in a separate recursive process from the
2646
+ # enclosing function or module, so we can simply drop them.
2647
+ def visit_CDeclaratorNode(self, node):
2648
+ # necessary to ensure that all CNameDeclaratorNodes are visited.
2649
+ self.visitchildren(node)
2650
+ return node
2651
+
2652
+ def visit_CTypeDefNode(self, node):
2653
+ return node
2654
+
2655
+ def visit_CBaseTypeNode(self, node):
2656
+ return None
2657
+
2658
+ def visit_CEnumDefNode(self, node):
2659
+ if node.visibility == 'public':
2660
+ return node
2661
+ else:
2662
+ return None
2663
+
2664
+ def visit_CNameDeclaratorNode(self, node):
2665
+ if node.name in self.seen_vars_stack[-1]:
2666
+ entry = self.current_env().lookup(node.name)
2667
+ if (entry is None or entry.visibility != 'extern'
2668
+ and not entry.scope.is_c_class_scope):
2669
+ error(node.pos, "cdef variable '%s' declared after it is used" % node.name)
2670
+ self.visitchildren(node)
2671
+ return node
2672
+
2673
+ def visit_CVarDefNode(self, node):
2674
+ # to ensure all CNameDeclaratorNodes are visited.
2675
+ self.visitchildren(node)
2676
+ return None
2677
+
2678
+ def visit_CnameDecoratorNode(self, node):
2679
+ child_node = self.visitchild(node, 'node')
2680
+ if not child_node:
2681
+ return None
2682
+ if type(child_node) is list: # Assignment synthesized
2683
+ node.node = child_node[0]
2684
+ return [node] + child_node[1:]
2685
+ return node
2686
+
2687
+ def create_Property(self, entry):
2688
+ if entry.visibility == 'public':
2689
+ if entry.type.is_pyobject:
2690
+ template = self.basic_pyobject_property
2691
+ else:
2692
+ template = self.basic_property
2693
+ elif entry.visibility == 'readonly':
2694
+ template = self.basic_property_ro
2695
+ property = template.substitute({
2696
+ "ATTR": ExprNodes.AttributeNode(pos=entry.pos,
2697
+ obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
2698
+ attribute=entry.name),
2699
+ }, pos=entry.pos).stats[0]
2700
+ property.name = entry.name
2701
+ property.doc = entry.doc
2702
+ return property
2703
+
2704
+ def visit_AssignmentExpressionNode(self, node):
2705
+ self.visitchildren(node)
2706
+ node.analyse_declarations(self.current_env())
2707
+ return node
2708
+
2709
+
2710
+ def _calculate_pickle_checksums(member_names):
2711
+ # Cython 0.x used MD5 for the checksum, which a few Python installations remove for security reasons.
2712
+ # SHA-256 should be ok for years to come, but early Cython 3.0 alpha releases used SHA-1,
2713
+ # which may not be.
2714
+ member_names_string = ' '.join(member_names).encode('utf-8')
2715
+ hash_kwargs = {'usedforsecurity': False} if sys.version_info >= (3, 9) else {}
2716
+ checksums = []
2717
+ for algo_name in ['sha256', 'sha1', 'md5']:
2718
+ try:
2719
+ mkchecksum = getattr(hashlib, algo_name)
2720
+ checksum = mkchecksum(member_names_string, **hash_kwargs).hexdigest()
2721
+ except (AttributeError, ValueError):
2722
+ # The algorithm (i.e. MD5) might not be there at all, or might be blocked at runtime.
2723
+ continue
2724
+ checksums.append('0x' + checksum[:7])
2725
+ return checksums
2726
+
2727
+
2728
+ class CalculateQualifiedNamesTransform(EnvTransform):
2729
+ """
2730
+ Calculate and store the '__qualname__' and the global
2731
+ module name on some nodes.
2732
+ """
2733
+ needs_qualname_assignment = False
2734
+ needs_module_assignment = False
2735
+
2736
+ def visit_ModuleNode(self, node):
2737
+ self.module_name = self.global_scope().qualified_name
2738
+ self.qualified_name = []
2739
+ _super = super()
2740
+ self._super_visit_FuncDefNode = _super.visit_FuncDefNode
2741
+ self._super_visit_ClassDefNode = _super.visit_ClassDefNode
2742
+ self.visitchildren(node)
2743
+ return node
2744
+
2745
+ def _set_qualname(self, node, name=None):
2746
+ if name:
2747
+ qualname = self.qualified_name[:]
2748
+ qualname.append(name)
2749
+ else:
2750
+ qualname = self.qualified_name
2751
+ node.qualname = EncodedString('.'.join(qualname))
2752
+ node.module_name = self.module_name
2753
+
2754
+ def _append_entry(self, entry):
2755
+ if entry.is_pyglobal and not entry.is_pyclass_attr:
2756
+ self.qualified_name = [entry.name]
2757
+ else:
2758
+ self.qualified_name.append(entry.name)
2759
+
2760
+ def visit_ClassNode(self, node):
2761
+ self._set_qualname(node, node.name)
2762
+ self.visitchildren(node)
2763
+ return node
2764
+
2765
+ def visit_PyClassNamespaceNode(self, node):
2766
+ # class name was already added by parent node
2767
+ self._set_qualname(node)
2768
+ self.visitchildren(node)
2769
+ return node
2770
+
2771
+ def visit_PyCFunctionNode(self, node):
2772
+ orig_qualified_name = self.qualified_name[:]
2773
+ if node.def_node.is_wrapper and self.qualified_name and self.qualified_name[-1] == '<locals>':
2774
+ self.qualified_name.pop()
2775
+ self._set_qualname(node)
2776
+ else:
2777
+ self._set_qualname(node, node.def_node.name)
2778
+ self.visitchildren(node)
2779
+ self.qualified_name = orig_qualified_name
2780
+ return node
2781
+
2782
+ def visit_DefNode(self, node):
2783
+ if node.is_wrapper and self.qualified_name:
2784
+ assert self.qualified_name[-1] == '<locals>', self.qualified_name
2785
+ orig_qualified_name = self.qualified_name[:]
2786
+ self.qualified_name.pop()
2787
+ self._set_qualname(node)
2788
+ self._super_visit_FuncDefNode(node)
2789
+ self.qualified_name = orig_qualified_name
2790
+ else:
2791
+ self._set_qualname(node, node.name)
2792
+ self.visit_FuncDefNode(node)
2793
+ return node
2794
+
2795
+ def visit_FuncDefNode(self, node):
2796
+ orig_qualified_name = self.qualified_name[:]
2797
+ if getattr(node, 'name', None) == '<lambda>':
2798
+ self.qualified_name.append('<lambda>')
2799
+ else:
2800
+ self._append_entry(node.entry)
2801
+ self.qualified_name.append('<locals>')
2802
+ self._super_visit_FuncDefNode(node)
2803
+ self.qualified_name = orig_qualified_name
2804
+ return node
2805
+
2806
+ def generate_assignment(self, node, name, value):
2807
+ entry = node.scope.lookup_here(name)
2808
+ lhs = ExprNodes.NameNode(
2809
+ node.pos,
2810
+ name=EncodedString(name),
2811
+ entry=entry,
2812
+ is_target=True)
2813
+ rhs = ExprNodes.UnicodeNode(node.pos, value=value)
2814
+ node.body.stats.insert(0, Nodes.SingleAssignmentNode(
2815
+ node.pos,
2816
+ lhs=lhs,
2817
+ rhs=rhs,
2818
+ ).analyse_expressions(self.current_env()))
2819
+
2820
+ def visit_ClassDefNode(self, node):
2821
+ orig_needs_qualname_assignment = self.needs_qualname_assignment
2822
+ self.needs_qualname_assignment = False
2823
+ orig_needs_module_assignment = self.needs_module_assignment
2824
+ self.needs_module_assignment = False
2825
+ orig_qualified_name = self.qualified_name[:]
2826
+ entry = (getattr(node, 'entry', None) or # PyClass
2827
+ self.current_env().lookup_here(node.target.name)) # CClass
2828
+ self._append_entry(entry)
2829
+ self._super_visit_ClassDefNode(node)
2830
+ if self.needs_qualname_assignment:
2831
+ self.generate_assignment(node, "__qualname__",
2832
+ EncodedString(".".join(self.qualified_name)))
2833
+ if self.needs_module_assignment:
2834
+ self.generate_assignment(node, "__module__",
2835
+ EncodedString(self.module_name))
2836
+ self.qualified_name = orig_qualified_name
2837
+ self.needs_qualname_assignment = orig_needs_qualname_assignment
2838
+ self.needs_module_assignment = orig_needs_module_assignment
2839
+ return node
2840
+
2841
+ def visit_NameNode(self, node):
2842
+ scope = self.current_env()
2843
+ if scope.is_c_class_scope:
2844
+ # unlike for a PyClass scope, these attributes aren't defined in the
2845
+ # dictionary when the class definition is executed, therefore we ask
2846
+ # the compiler to generate an assignment to them at the start of the
2847
+ # body.
2848
+ # NOTE: this doesn't put them in locals()
2849
+ if node.name == "__qualname__":
2850
+ self.needs_qualname_assignment = True
2851
+ elif node.name == "__module__":
2852
+ self.needs_module_assignment = True
2853
+ return node
2854
+
2855
+
2856
+ class AnalyseExpressionsTransform(CythonTransform):
2857
+
2858
+ def visit_ModuleNode(self, node):
2859
+ node.scope.infer_types()
2860
+ node.body = node.body.analyse_expressions(node.scope)
2861
+ self.positions = [{node.pos}]
2862
+ self.visitchildren(node)
2863
+ self._build_positions(node)
2864
+ return node
2865
+
2866
+ def visit_FuncDefNode(self, node):
2867
+ node.local_scope.infer_types()
2868
+ node.body = node.body.analyse_expressions(node.local_scope)
2869
+ self.positions[-1].add(node.pos)
2870
+
2871
+ if node.is_wrapper:
2872
+ # Share positions between function and Python wrapper.
2873
+ local_positions = self.positions[-1]
2874
+ else:
2875
+ local_positions = {node.pos}
2876
+ self.positions.append(local_positions)
2877
+
2878
+ self.visitchildren(node)
2879
+ self._build_positions(node)
2880
+ return node
2881
+
2882
+ def visit_ScopedExprNode(self, node):
2883
+ if node.has_local_scope:
2884
+ node.expr_scope.infer_types()
2885
+ node = node.analyse_scoped_expressions(node.expr_scope)
2886
+ self.visit_ExprNode(node)
2887
+ return node
2888
+
2889
+ def visit_IndexNode(self, node):
2890
+ """
2891
+ Replace index nodes used to specialize cdef functions with fused
2892
+ argument types with the Attribute- or NameNode referring to the
2893
+ function. We then need to copy over the specialization properties to
2894
+ the attribute or name node.
2895
+
2896
+ Because the indexing might be a Python indexing operation on a fused
2897
+ function, or (usually) a Cython indexing operation, we need to
2898
+ re-analyse the types.
2899
+ """
2900
+ self.visit_ExprNode(node)
2901
+ if node.is_fused_index and not node.type.is_error:
2902
+ node = node.base
2903
+ return node
2904
+
2905
+ # Build the line table according to PEP-626.
2906
+ # We mostly just do this here to avoid yet another transform traversal.
2907
+
2908
+ def visit_ExprNode(self, node):
2909
+ self.positions[-1].add(node.pos)
2910
+ self.visitchildren(node)
2911
+ return node
2912
+
2913
+ def visit_StatNode(self, node):
2914
+ self.positions[-1].add(node.pos)
2915
+ self.visitchildren(node)
2916
+ return node
2917
+
2918
+ def _build_positions(self, func_node):
2919
+ """
2920
+ Build the PEP-626 line table and "bytecode-to-position" mapping used for CodeObjects.
2921
+ """
2922
+ # Code can originate from different source files and string code fragments, even within a single function.
2923
+ # Thus, it's not completely correct to just ignore the source files when sorting the line numbers,
2924
+ # but it also doesn't hurt much for the moment. Eventually, we might need different CodeObjects
2925
+ # even within a single function if it uses code from different sources / line number ranges.
2926
+ positions: list = sorted(
2927
+ self.positions.pop(),
2928
+ key=itemgetter(1, 2), # (line, column)
2929
+ # Build ranges backwards to know the end column before we see the start column in the same line.
2930
+ reverse=True,
2931
+ )
2932
+
2933
+ next_line = -1
2934
+ next_column_in_line = 0
2935
+
2936
+ ranges = []
2937
+ for _, line, start_column in positions:
2938
+ ranges.append((line, line, start_column, next_column_in_line if line == next_line else start_column + 1))
2939
+ next_line, next_column_in_line = line, start_column
2940
+
2941
+ ranges.reverse()
2942
+ func_node.node_positions = ranges
2943
+
2944
+ positions.reverse()
2945
+ i: cython.Py_ssize_t
2946
+ func_node.local_scope.node_positions_to_offset = {
2947
+ position: i
2948
+ for i, position in enumerate(positions)
2949
+ }
2950
+
2951
+
2952
+ class FindInvalidUseOfFusedTypes(TreeVisitor):
2953
+
2954
+ def __call__(self, tree):
2955
+ self._in_fused_function = False
2956
+ self.visit(tree)
2957
+ return tree
2958
+
2959
+ def visit_Node(self, node):
2960
+ self.visitchildren(node)
2961
+
2962
+ def visit_FuncDefNode(self, node):
2963
+ outer_status = self._in_fused_function
2964
+ self._in_fused_function = node.has_fused_arguments
2965
+
2966
+ if not self._in_fused_function:
2967
+ # Errors related to use in functions with fused args will already
2968
+ # have been detected.
2969
+ if not node.is_generator_body and node.return_type.is_fused:
2970
+ error(node.pos, "Return type is not specified as argument type")
2971
+
2972
+ self.visitchildren(node)
2973
+ self._in_fused_function = outer_status
2974
+
2975
+ def visit_ExprNode(self, node):
2976
+ if not self._in_fused_function and node.type and node.type.is_fused:
2977
+ error(node.pos, "Invalid use of fused types, type cannot be specialized")
2978
+ # Errors in subtrees are likely related, so do not recurse.
2979
+ else:
2980
+ self.visitchildren(node)
2981
+
2982
+
2983
+ class ExpandInplaceOperators(EnvTransform):
2984
+
2985
+ def visit_InPlaceAssignmentNode(self, node):
2986
+ lhs = node.lhs
2987
+ rhs = node.rhs
2988
+ if lhs.type.is_cpp_class:
2989
+ # No getting around this exact operator here.
2990
+ return node
2991
+ if isinstance(lhs, ExprNodes.BufferIndexNode):
2992
+ # There is code to handle this case in InPlaceAssignmentNode
2993
+ return node
2994
+
2995
+ env = self.current_env()
2996
+ def side_effect_free_reference(node, setting=False):
2997
+ if node.is_name:
2998
+ return node, []
2999
+ elif node.type.is_pyobject and not setting:
3000
+ node = LetRefNode(node)
3001
+ return node, [node]
3002
+ elif node.is_subscript:
3003
+ base, temps = side_effect_free_reference(node.base)
3004
+ index = LetRefNode(node.index)
3005
+ return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
3006
+ elif node.is_attribute:
3007
+ obj, temps = side_effect_free_reference(node.obj, setting=setting)
3008
+ return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
3009
+ elif isinstance(node, ExprNodes.BufferIndexNode):
3010
+ raise ValueError("Don't allow things like attributes of buffer indexing operations")
3011
+ else:
3012
+ node = LetRefNode(node)
3013
+ return node, [node]
3014
+ try:
3015
+ lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
3016
+ except ValueError:
3017
+ return node
3018
+ dup = lhs.__class__(**lhs.__dict__)
3019
+ binop = ExprNodes.binop_node(node.pos,
3020
+ operator = node.operator,
3021
+ operand1 = dup,
3022
+ operand2 = rhs,
3023
+ inplace=True)
3024
+ # Manually analyse types for new node.
3025
+ lhs.is_target = True
3026
+ lhs = lhs.analyse_target_types(env)
3027
+ dup.analyse_types(env) # FIXME: no need to reanalyse the copy, right?
3028
+ binop.analyse_operation(env)
3029
+ node = Nodes.SingleAssignmentNode(
3030
+ node.pos,
3031
+ lhs = lhs,
3032
+ rhs=binop.coerce_to(lhs.type, env))
3033
+ # Use LetRefNode to avoid side effects.
3034
+ let_ref_nodes.reverse()
3035
+ for t in let_ref_nodes:
3036
+ node = LetNode(t, node)
3037
+ return node
3038
+
3039
+ def visit_ExprNode(self, node):
3040
+ # In-place assignments can't happen within an expression.
3041
+ return node
3042
+
3043
+
3044
+ class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
3045
+ """
3046
+ Adjust function and class definitions by the decorator directives:
3047
+
3048
+ @cython.cfunc
3049
+ @cython.cclass
3050
+ @cython.ccall
3051
+ @cython.inline
3052
+ @cython.nogil
3053
+ @cython.critical_section
3054
+ """
3055
+ # list of directives that cause conversion to cclass
3056
+ converts_to_cclass = ('cclass', 'total_ordering', 'dataclasses.dataclass')
3057
+
3058
+ def visit_ModuleNode(self, node):
3059
+ self.directives = node.directives
3060
+ self.in_py_class = False
3061
+ self.visitchildren(node)
3062
+ return node
3063
+
3064
+ def visit_CompilerDirectivesNode(self, node):
3065
+ old_directives = self.directives
3066
+ self.directives = node.directives
3067
+ self.visitchildren(node)
3068
+ self.directives = old_directives
3069
+ return node
3070
+
3071
+ def visit_DefNode(self, node):
3072
+ modifiers = []
3073
+ if 'inline' in self.directives:
3074
+ modifiers.append('inline')
3075
+ nogil = self.directives.get('nogil')
3076
+ with_gil = self.directives.get('with_gil')
3077
+ except_val = self.directives.get('exceptval')
3078
+ has_explicit_exc_clause = False if except_val is None else True
3079
+ return_type_node = self.directives.get('returns')
3080
+ if return_type_node is None and self.directives['annotation_typing']:
3081
+ return_type_node = node.return_type_annotation
3082
+ # for Python annotations, prefer safe exception handling by default
3083
+ if return_type_node is not None and except_val is None:
3084
+ except_val = (None, True) # except *
3085
+ elif except_val is None:
3086
+ # backward compatible default: no exception check, unless there's also a "@returns" declaration
3087
+ except_val = (None, True if return_type_node else False)
3088
+ if self.directives.get('c_compile_guard') and 'cfunc' not in self.directives:
3089
+ error(node.pos, "c_compile_guard only allowed on C functions")
3090
+ if 'ccall' in self.directives:
3091
+ if 'cfunc' in self.directives:
3092
+ error(node.pos, "cfunc and ccall directives cannot be combined")
3093
+ if with_gil:
3094
+ error(node.pos, "ccall functions cannot be declared 'with_gil'")
3095
+ node = node.as_cfunction(
3096
+ overridable=True, modifiers=modifiers, nogil=nogil,
3097
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3098
+ return self.visit(node)
3099
+ if 'cfunc' in self.directives:
3100
+ if self.in_py_class:
3101
+ error(node.pos, "cfunc directive is not allowed here")
3102
+ else:
3103
+ node = node.as_cfunction(
3104
+ overridable=False, modifiers=modifiers, nogil=nogil, with_gil=with_gil,
3105
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3106
+ return self.visit(node)
3107
+ if 'inline' in modifiers:
3108
+ error(node.pos, "Python functions cannot be declared 'inline'")
3109
+ if nogil:
3110
+ # TODO: turn this into a "with gil" declaration.
3111
+ error(node.pos, "Python functions cannot be declared 'nogil'")
3112
+ if with_gil:
3113
+ error(node.pos, "Python functions cannot be declared 'with_gil'")
3114
+ self.visit_FuncDefNode(node)
3115
+ return node
3116
+
3117
+ def visit_FuncDefNode(self, node):
3118
+ if "critical_section" in self.directives:
3119
+ value = self.directives["critical_section"]
3120
+ if value is not None:
3121
+ error(node.pos, "critical_section decorator does not take arguments")
3122
+ new_body = Nodes.CriticalSectionStatNode(
3123
+ node.pos,
3124
+ args=[ExprNodes.FirstArgumentForCriticalSectionNode(node.pos, func_node=node)],
3125
+ body=node.body
3126
+ )
3127
+ node.body = new_body
3128
+ self.visitchildren(node)
3129
+ return node
3130
+
3131
+ def visit_LambdaNode(self, node):
3132
+ # No directives should modify lambdas or generator expressions (and also nothing in them).
3133
+ return node
3134
+
3135
+ def visit_PyClassDefNode(self, node):
3136
+ if any(directive in self.directives for directive in self.converts_to_cclass):
3137
+ node = node.as_cclass()
3138
+ return self.visit(node)
3139
+ else:
3140
+ old_in_pyclass = self.in_py_class
3141
+ self.in_py_class = True
3142
+ self.visitchildren(node)
3143
+ self.in_py_class = old_in_pyclass
3144
+ return node
3145
+
3146
+ def visit_CClassDefNode(self, node):
3147
+ old_in_pyclass = self.in_py_class
3148
+ self.in_py_class = False
3149
+ self.visitchildren(node)
3150
+ self.in_py_class = old_in_pyclass
3151
+ return node
3152
+
3153
+
3154
+ class AlignFunctionDefinitions(CythonTransform):
3155
+ """
3156
+ This class takes the signatures from a .pxd file and applies them to
3157
+ the def methods in a .py file.
3158
+ """
3159
+
3160
+ def visit_ModuleNode(self, node):
3161
+ self.scope = node.scope
3162
+ self.visitchildren(node)
3163
+ return node
3164
+
3165
+ def visit_PyClassDefNode(self, node):
3166
+ pxd_def = self.scope.lookup(node.name)
3167
+ if pxd_def:
3168
+ if pxd_def.is_cclass:
3169
+ return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
3170
+ elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
3171
+ error(node.pos, "'%s' redeclared" % node.name)
3172
+ if pxd_def.pos:
3173
+ error(pxd_def.pos, "previous declaration here")
3174
+ return None
3175
+ return node
3176
+
3177
+ def visit_CClassDefNode(self, node, pxd_def=None):
3178
+ if pxd_def is None:
3179
+ pxd_def = self.scope.lookup(node.class_name)
3180
+ if pxd_def:
3181
+ if not pxd_def.defined_in_pxd:
3182
+ return node
3183
+ outer_scope = self.scope
3184
+ self.scope = pxd_def.type.scope
3185
+ self.visitchildren(node)
3186
+ if pxd_def:
3187
+ self.scope = outer_scope
3188
+ return node
3189
+
3190
+ def visit_DefNode(self, node):
3191
+ pxd_def = self.scope.lookup(node.name)
3192
+ if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
3193
+ if not pxd_def.is_cfunction:
3194
+ error(node.pos, "'%s' redeclared" % node.name)
3195
+ if pxd_def.pos:
3196
+ error(pxd_def.pos, "previous declaration here")
3197
+ return None
3198
+ node = node.as_cfunction(pxd_def)
3199
+ # Enable this when nested cdef functions are allowed.
3200
+ # self.visitchildren(node)
3201
+ return node
3202
+
3203
+ def visit_ExprNode(self, node):
3204
+ # ignore lambdas and everything else that appears in expressions
3205
+ return node
3206
+
3207
+
3208
+ class AutoCpdefFunctionDefinitions(CythonTransform):
3209
+
3210
+ def visit_ModuleNode(self, node):
3211
+ self.directives = node.directives
3212
+ self.imported_names = set() # hack, see visit_FromImportStatNode()
3213
+ self.scope = node.scope
3214
+ self.visitchildren(node)
3215
+ return node
3216
+
3217
+ def visit_DefNode(self, node):
3218
+ if (self.scope.is_module_scope and self.directives['auto_cpdef']
3219
+ and node.name not in self.imported_names
3220
+ and node.is_cdef_func_compatible()):
3221
+ # FIXME: cpdef-ing should be done in analyse_declarations()
3222
+ node = node.as_cfunction(scope=self.scope)
3223
+ return node
3224
+
3225
+ def visit_CClassDefNode(self, node, pxd_def=None):
3226
+ if pxd_def is None:
3227
+ pxd_def = self.scope.lookup(node.class_name)
3228
+ if pxd_def:
3229
+ if not pxd_def.defined_in_pxd:
3230
+ return node
3231
+ outer_scope = self.scope
3232
+ self.scope = pxd_def.type.scope
3233
+ self.visitchildren(node)
3234
+ if pxd_def:
3235
+ self.scope = outer_scope
3236
+ return node
3237
+
3238
+ def visit_FromImportStatNode(self, node):
3239
+ # hack to prevent conditional import fallback functions from
3240
+ # being cdpef-ed (global Python variables currently conflict
3241
+ # with imports)
3242
+ if self.scope.is_module_scope:
3243
+ for name, _ in node.items:
3244
+ self.imported_names.add(name)
3245
+ return node
3246
+
3247
+ def visit_ExprNode(self, node):
3248
+ # ignore lambdas and everything else that appears in expressions
3249
+ return node
3250
+
3251
+
3252
+ class RemoveUnreachableCode(CythonTransform):
3253
+
3254
+ def visit_StatListNode(self, node):
3255
+ if not self.current_directives['remove_unreachable']:
3256
+ return node
3257
+ self.visitchildren(node)
3258
+ if len(node.stats) == 1 and isinstance(node.stats[0], Nodes.StatListNode) and not node.stats[0].stats:
3259
+ del node.stats[:]
3260
+ for idx, stat in enumerate(node.stats, 1):
3261
+ if stat.is_terminator:
3262
+ if idx < len(node.stats):
3263
+ if self.current_directives['warn.unreachable']:
3264
+ warning(node.stats[idx].pos, "Unreachable code", 2)
3265
+ node.stats = node.stats[:idx]
3266
+ node.is_terminator = True
3267
+ break
3268
+ return node
3269
+
3270
+ def visit_IfClauseNode(self, node):
3271
+ self.visitchildren(node)
3272
+ if node.body.is_terminator:
3273
+ node.is_terminator = True
3274
+ return node
3275
+
3276
+ def visit_IfStatNode(self, node):
3277
+ self.visitchildren(node)
3278
+ if node.else_clause and node.else_clause.is_terminator:
3279
+ for clause in node.if_clauses:
3280
+ if not clause.is_terminator:
3281
+ break
3282
+ else:
3283
+ node.is_terminator = True
3284
+ return node
3285
+
3286
+ def visit_TryExceptStatNode(self, node):
3287
+ self.visitchildren(node)
3288
+ if node.body.is_terminator and node.else_clause:
3289
+ if self.current_directives['warn.unreachable']:
3290
+ warning(node.else_clause.pos, "Unreachable code", 2)
3291
+ node.else_clause = None
3292
+ return node
3293
+
3294
+ def visit_TryFinallyStatNode(self, node):
3295
+ self.visitchildren(node)
3296
+ if node.finally_clause.is_terminator:
3297
+ node.is_terminator = True
3298
+ return node
3299
+
3300
+ def visit_PassStatNode(self, node):
3301
+ """Eliminate useless PassStatNode"""
3302
+ # 'pass' statements often appear in a separate line and must be traced.
3303
+ if not self.current_directives['linetrace']:
3304
+ node = Nodes.StatListNode(pos=node.pos, stats=[])
3305
+ return node
3306
+
3307
+
3308
+ class YieldNodeCollector(TreeVisitor):
3309
+
3310
+ def __init__(self, excludes=[]):
3311
+ super().__init__()
3312
+ self.yields = []
3313
+ self.returns = []
3314
+ self.finallys = []
3315
+ self.excepts = []
3316
+ self.has_return_value = False
3317
+ self.has_yield = False
3318
+ self.has_await = False
3319
+ self.excludes = excludes
3320
+
3321
+ def visit_Node(self, node):
3322
+ if node not in self.excludes:
3323
+ self.visitchildren(node)
3324
+
3325
+ def visit_YieldExprNode(self, node):
3326
+ self.yields.append(node)
3327
+ self.has_yield = True
3328
+ self.visitchildren(node)
3329
+
3330
+ def visit_AwaitExprNode(self, node):
3331
+ self.yields.append(node)
3332
+ self.has_await = True
3333
+ self.visitchildren(node)
3334
+
3335
+ def visit_ReturnStatNode(self, node):
3336
+ self.visitchildren(node)
3337
+ if node.value:
3338
+ self.has_return_value = True
3339
+ self.returns.append(node)
3340
+
3341
+ def visit_TryFinallyStatNode(self, node):
3342
+ self.visitchildren(node)
3343
+ self.finallys.append(node)
3344
+
3345
+ def visit_TryExceptStatNode(self, node):
3346
+ self.visitchildren(node)
3347
+ self.excepts.append(node)
3348
+
3349
+ def visit_ClassDefNode(self, node):
3350
+ pass
3351
+
3352
+ def visit_FuncDefNode(self, node):
3353
+ pass
3354
+
3355
+ def visit_LambdaNode(self, node):
3356
+ pass
3357
+
3358
+ def visit_GeneratorExpressionNode(self, node):
3359
+ # node.loop iterator is evaluated outside the generator expression
3360
+ if isinstance(node.loop, Nodes._ForInStatNode):
3361
+ # Possibly should handle ForFromStatNode
3362
+ # but for now do nothing
3363
+ self.visit(node.loop.iterator)
3364
+
3365
+ def visit_CArgDeclNode(self, node):
3366
+ # do not look into annotations
3367
+ # FIXME: support (yield) in default arguments (currently crashes)
3368
+ pass
3369
+
3370
+
3371
+ class MarkClosureVisitor(CythonTransform):
3372
+ # In addition to marking closures this is also responsible to finding parts of the
3373
+ # generator iterable and marking them
3374
+
3375
+ def visit_ModuleNode(self, node):
3376
+ self.needs_closure = False
3377
+ self.excludes = []
3378
+ self.visitchildren(node)
3379
+ return node
3380
+
3381
+ def visit_FuncDefNode(self, node):
3382
+ self.needs_closure = False
3383
+ self.visitchildren(node)
3384
+ node.needs_closure = self.needs_closure
3385
+ self.needs_closure = True
3386
+
3387
+ collector = YieldNodeCollector(self.excludes)
3388
+ collector.visitchildren(node)
3389
+
3390
+ if node.is_async_def:
3391
+ coroutine_type = Nodes.AsyncDefNode
3392
+ if collector.has_yield:
3393
+ coroutine_type = Nodes.AsyncGenNode
3394
+ for yield_expr in collector.yields + collector.returns:
3395
+ yield_expr.in_async_gen = True
3396
+ elif self.current_directives['iterable_coroutine']:
3397
+ coroutine_type = Nodes.IterableAsyncDefNode
3398
+ elif collector.has_await:
3399
+ found = next(y for y in collector.yields if y.is_await)
3400
+ error(found.pos, "'await' not allowed in generators (use 'yield')")
3401
+ return node
3402
+ elif collector.has_yield:
3403
+ coroutine_type = Nodes.GeneratorDefNode
3404
+ else:
3405
+ return node
3406
+
3407
+ for i, yield_expr in enumerate(collector.yields, 1):
3408
+ yield_expr.label_num = i
3409
+ for retnode in collector.returns + collector.finallys + collector.excepts:
3410
+ retnode.in_generator = True
3411
+
3412
+ gbody = Nodes.GeneratorBodyDefNode(
3413
+ pos=node.pos, name=node.name, body=node.body,
3414
+ is_coroutine_body=node.is_async_def,
3415
+ is_async_gen_body=node.is_async_def and collector.has_yield)
3416
+ coroutine = coroutine_type(
3417
+ pos=node.pos, name=node.name, args=node.args,
3418
+ star_arg=node.star_arg, starstar_arg=node.starstar_arg,
3419
+ doc=node.doc, decorators=node.decorators,
3420
+ gbody=gbody, lambda_name=node.lambda_name,
3421
+ return_type_annotation=node.return_type_annotation,
3422
+ is_generator_expression=node.is_generator_expression)
3423
+ return coroutine
3424
+
3425
+ def visit_CFuncDefNode(self, node):
3426
+ self.needs_closure = False
3427
+ self.visitchildren(node)
3428
+ node.needs_closure = self.needs_closure
3429
+ self.needs_closure = True
3430
+ if node.needs_closure and node.overridable:
3431
+ error(node.pos, "closures inside cpdef functions not yet supported")
3432
+ return node
3433
+
3434
+ def visit_LambdaNode(self, node):
3435
+ self.needs_closure = False
3436
+ self.visitchildren(node)
3437
+ node.needs_closure = self.needs_closure
3438
+ self.needs_closure = True
3439
+ return node
3440
+
3441
+ def visit_ClassDefNode(self, node):
3442
+ self.visitchildren(node)
3443
+ self.needs_closure = True
3444
+ return node
3445
+
3446
+ def visit_GeneratorExpressionNode(self, node):
3447
+ excludes = self.excludes
3448
+ if isinstance(node.loop, Nodes._ForInStatNode):
3449
+ self.excludes = [node.loop.iterator]
3450
+ node = self.visit_LambdaNode(node)
3451
+ self.excludes = excludes
3452
+ if not isinstance(node.loop, Nodes._ForInStatNode):
3453
+ # Possibly should handle ForFromStatNode
3454
+ # but for now do nothing
3455
+ return node
3456
+ itseq = node.loop.iterator.sequence
3457
+ # literals do not need replacing with an argument
3458
+ if itseq.is_literal:
3459
+ return node
3460
+ _GeneratorExpressionArgumentsMarker(node).visit(itseq)
3461
+ return node
3462
+
3463
+
3464
+ class CreateClosureClasses(CythonTransform):
3465
+ # Output closure classes in module scope for all functions
3466
+ # that really need it.
3467
+
3468
+ def __init__(self, context):
3469
+ super().__init__(context)
3470
+ self.path = []
3471
+ self.in_lambda = False
3472
+
3473
+ def visit_ModuleNode(self, node):
3474
+ self.module_scope = node.scope
3475
+ self.visitchildren(node)
3476
+ return node
3477
+
3478
+ def find_entries_used_in_closures(self, node):
3479
+ from_closure = []
3480
+ in_closure = []
3481
+ for scope in node.local_scope.iter_local_scopes():
3482
+ for name, entry in scope.entries.items():
3483
+ if not name:
3484
+ continue
3485
+ if entry.from_closure:
3486
+ from_closure.append((name, entry))
3487
+ elif entry.in_closure:
3488
+ in_closure.append((name, entry))
3489
+ return from_closure, in_closure
3490
+
3491
+ def create_class_from_scope(self, node, target_module_scope, inner_node=None):
3492
+ # move local variables into closure
3493
+ if node.is_generator:
3494
+ for scope in node.local_scope.iter_local_scopes():
3495
+ for entry in scope.entries.values():
3496
+ if not (entry.from_closure or entry.is_pyglobal or entry.is_cglobal):
3497
+ entry.in_closure = True
3498
+
3499
+ from_closure, in_closure = self.find_entries_used_in_closures(node)
3500
+ in_closure.sort()
3501
+
3502
+ # Now from the beginning
3503
+ node.needs_closure = False
3504
+ node.needs_outer_scope = False
3505
+
3506
+ func_scope = node.local_scope
3507
+ cscope = node.entry.scope
3508
+ while cscope.is_py_class_scope or cscope.is_c_class_scope:
3509
+ cscope = cscope.outer_scope
3510
+
3511
+ if not from_closure and (self.path or inner_node):
3512
+ if not inner_node:
3513
+ if not node.py_cfunc_node:
3514
+ raise InternalError("DefNode does not have assignment node")
3515
+ inner_node = node.py_cfunc_node
3516
+ inner_node.needs_closure_code = False
3517
+ node.needs_outer_scope = False
3518
+
3519
+ if node.is_generator:
3520
+ pass
3521
+ elif not in_closure and not from_closure:
3522
+ return
3523
+ elif not in_closure:
3524
+ func_scope.is_passthrough = True
3525
+ func_scope.scope_class = cscope.scope_class
3526
+ node.needs_outer_scope = True
3527
+ return
3528
+
3529
+ # entry.cname can contain periods (eg. a derived C method of a class).
3530
+ # We want to use the cname as part of a C struct name, so we replace
3531
+ # periods with double underscores.
3532
+ as_name = '%s_%s' % (
3533
+ target_module_scope.next_id(Naming.closure_class_prefix),
3534
+ node.entry.cname.replace('.','__'))
3535
+ as_name = EncodedString(as_name)
3536
+
3537
+ entry = target_module_scope.declare_c_class(
3538
+ name=as_name, pos=node.pos, defining=True,
3539
+ implementing=True)
3540
+ entry.type.is_final_type = True
3541
+
3542
+ func_scope.scope_class = entry
3543
+ class_scope = entry.type.scope
3544
+ class_scope.is_internal = True
3545
+ class_scope.is_closure_class_scope = True
3546
+ if node.is_async_def or node.is_generator:
3547
+ # Generators need their closure intact during cleanup as they resume to handle GeneratorExit
3548
+ class_scope.directives['no_gc_clear'] = True
3549
+ if Options.closure_freelist_size:
3550
+ class_scope.directives['freelist'] = Options.closure_freelist_size
3551
+
3552
+ if from_closure:
3553
+ assert cscope.is_closure_scope
3554
+ class_scope.declare_var(pos=node.pos,
3555
+ name=Naming.outer_scope_cname,
3556
+ cname=Naming.outer_scope_cname,
3557
+ type=cscope.scope_class.type,
3558
+ is_cdef=True)
3559
+ node.needs_outer_scope = True
3560
+ for name, entry in in_closure:
3561
+ closure_entry = class_scope.declare_var(
3562
+ pos=entry.pos,
3563
+ name=entry.name if not entry.in_subscope else None,
3564
+ cname=entry.cname,
3565
+ type=entry.type,
3566
+ is_cdef=True)
3567
+ if entry.is_declared_generic:
3568
+ closure_entry.is_declared_generic = 1
3569
+ node.needs_closure = True
3570
+ # Do it here because other classes are already checked
3571
+ target_module_scope.check_c_class(func_scope.scope_class)
3572
+
3573
+ def visit_LambdaNode(self, node):
3574
+ if not isinstance(node.def_node, Nodes.DefNode):
3575
+ # fused function, an error has been previously issued
3576
+ return node
3577
+
3578
+ was_in_lambda = self.in_lambda
3579
+ self.in_lambda = True
3580
+ self.create_class_from_scope(node.def_node, self.module_scope, node)
3581
+ self.visitchildren(node)
3582
+ self.in_lambda = was_in_lambda
3583
+ return node
3584
+
3585
+ def visit_FuncDefNode(self, node):
3586
+ if self.in_lambda:
3587
+ self.visitchildren(node)
3588
+ return node
3589
+ if node.needs_closure or self.path:
3590
+ self.create_class_from_scope(node, self.module_scope)
3591
+ self.path.append(node)
3592
+ self.visitchildren(node)
3593
+ self.path.pop()
3594
+ return node
3595
+
3596
+ def visit_GeneratorBodyDefNode(self, node):
3597
+ self.visitchildren(node)
3598
+ return node
3599
+
3600
+ def visit_CFuncDefNode(self, node):
3601
+ if not node.overridable:
3602
+ return self.visit_FuncDefNode(node)
3603
+ else:
3604
+ self.visitchildren(node)
3605
+ return node
3606
+
3607
+ def visit_GeneratorExpressionNode(self, node):
3608
+ node = _HandleGeneratorArguments()(node)
3609
+ return self.visit_LambdaNode(node)
3610
+
3611
+
3612
+ class InjectGilHandling(VisitorTransform, SkipDeclarations):
3613
+ """
3614
+ Allow certain Python operations inside of nogil blocks by implicitly acquiring the GIL.
3615
+
3616
+ Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get
3617
+ set up, parallel sections know that the GIL is acquired inside of them, etc.
3618
+ """
3619
+ nogil = False
3620
+
3621
+ # special node handling
3622
+
3623
+ def _inject_gil_in_nogil(self, node):
3624
+ """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block."""
3625
+ if self.nogil:
3626
+ node = Nodes.GILStatNode(node.pos, state='gil', body=node)
3627
+ return node
3628
+
3629
+ visit_RaiseStatNode = _inject_gil_in_nogil
3630
+ visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function
3631
+
3632
+ # further candidates:
3633
+ # def visit_ReraiseStatNode(self, node):
3634
+
3635
+ # nogil tracking
3636
+
3637
+ def visit_GILStatNode(self, node):
3638
+ was_nogil = self.nogil
3639
+ self.nogil = (node.state == 'nogil')
3640
+ self.visitchildren(node)
3641
+ self.nogil = was_nogil
3642
+ return node
3643
+
3644
+ def visit_CFuncDefNode(self, node):
3645
+ was_nogil = self.nogil
3646
+ if isinstance(node.declarator, Nodes.CFuncDeclaratorNode):
3647
+ self.nogil = node.declarator.nogil and not node.declarator.with_gil
3648
+ self.visitchildren(node)
3649
+ self.nogil = was_nogil
3650
+ return node
3651
+
3652
+ def visit_ParallelRangeNode(self, node):
3653
+ was_nogil = self.nogil
3654
+ self.nogil = node.nogil
3655
+ self.visitchildren(node)
3656
+ self.nogil = was_nogil
3657
+ return node
3658
+
3659
+ def visit_ExprNode(self, node):
3660
+ # No special GIL handling inside of expressions for now.
3661
+ return node
3662
+
3663
+ visit_Node = VisitorTransform.recurse_to_children
3664
+
3665
+
3666
+ class GilCheck(VisitorTransform):
3667
+ """
3668
+ Call `node.gil_check(env)` on each node to make sure we hold the
3669
+ GIL when we need it. Raise an error when on Python operations
3670
+ inside a `nogil` environment.
3671
+
3672
+ Additionally, raise exceptions for closely nested with gil or with nogil
3673
+ statements. The latter would abort Python.
3674
+ """
3675
+
3676
+ def __call__(self, root):
3677
+ self.env_stack = [root.scope]
3678
+ self.nogil_state = Nodes.NoGilState.HasGil
3679
+
3680
+ self.nogil_state_at_current_gilstatnode = Nodes.NoGilState.HasGil
3681
+ return super().__call__(root)
3682
+
3683
+ def _visit_scoped_children(self, node, nogil_state):
3684
+ was_nogil = self.nogil_state
3685
+ outer_attrs = node.outer_attrs
3686
+ if outer_attrs and len(self.env_stack) > 1:
3687
+ self.nogil_state = (
3688
+ Nodes.NoGilState.NoGil if self.env_stack[-2].nogil else Nodes.NoGilState.HasGil)
3689
+ self.visitchildren(node, outer_attrs)
3690
+
3691
+ self.nogil_state = nogil_state
3692
+ self.visitchildren(node, attrs=None, exclude=outer_attrs)
3693
+ self.nogil_state = was_nogil
3694
+
3695
+ def visit_FuncDefNode(self, node):
3696
+ self.env_stack.append(node.local_scope)
3697
+ inner_nogil = node.local_scope.nogil
3698
+
3699
+ nogil_state = self.nogil_state
3700
+ if inner_nogil:
3701
+ self.nogil_state = Nodes.NoGilState.NoGilScope
3702
+
3703
+ if inner_nogil and node.nogil_check:
3704
+ node.nogil_check(node.local_scope)
3705
+
3706
+ self._visit_scoped_children(node, self.nogil_state)
3707
+
3708
+ # FuncDefNodes can be nested, because a cpdef function contains a def function
3709
+ # inside it. Therefore restore to previous state
3710
+ self.nogil_state = nogil_state
3711
+
3712
+ self.env_stack.pop()
3713
+ return node
3714
+
3715
+ def visit_GILStatNode(self, node):
3716
+ if node.condition is not None:
3717
+ error(node.condition.pos,
3718
+ "Non-constant condition in a "
3719
+ "`with %s(<condition>)` statement" % node.state)
3720
+ return node
3721
+
3722
+ if self.nogil_state and node.nogil_check:
3723
+ node.nogil_check()
3724
+
3725
+ was_nogil = self.nogil_state
3726
+ is_nogil = (node.state == 'nogil')
3727
+
3728
+ if was_nogil == is_nogil and not self.nogil_state == Nodes.NoGilState.NoGilScope:
3729
+ if not was_nogil:
3730
+ error(node.pos, "Trying to acquire the GIL while it is "
3731
+ "already held.")
3732
+ else:
3733
+ error(node.pos, "Trying to release the GIL while it was "
3734
+ "previously released.")
3735
+ if self.nogil_state == Nodes.NoGilState.NoGilScope:
3736
+ node.scope_gil_state_known = False
3737
+
3738
+ if isinstance(node.finally_clause, Nodes.StatListNode):
3739
+ # The finally clause of the GILStatNode is a GILExitNode,
3740
+ # which is wrapped in a StatListNode. Just unpack that.
3741
+ node.finally_clause, = node.finally_clause.stats
3742
+
3743
+ nogil_state_at_current_gilstatnode = self.nogil_state_at_current_gilstatnode
3744
+ self.nogil_state_at_current_gilstatnode = self.nogil_state
3745
+ nogil_state = Nodes.NoGilState.NoGil if is_nogil else Nodes.NoGilState.HasGil
3746
+ self._visit_scoped_children(node, nogil_state)
3747
+ self.nogil_state_at_current_gilstatnode = nogil_state_at_current_gilstatnode
3748
+ return node
3749
+
3750
+ def visit_ParallelRangeNode(self, node):
3751
+ if node.nogil or self.nogil_state == Nodes.NoGilState.NoGilScope:
3752
+ node_was_nogil, node.nogil = node.nogil, False
3753
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
3754
+ if not node_was_nogil and self.nogil_state == Nodes.NoGilState.NoGilScope:
3755
+ # We're in a "nogil" function, but that doesn't prove we
3756
+ # didn't have the gil
3757
+ node.scope_gil_state_known = False
3758
+ return self.visit_GILStatNode(node)
3759
+
3760
+ if not self.nogil_state:
3761
+ error(node.pos, "prange() can only be used without the GIL")
3762
+ # Forget about any GIL-related errors that may occur in the body
3763
+ return None
3764
+
3765
+ node.nogil_check(self.env_stack[-1])
3766
+ self.visitchildren(node)
3767
+ return node
3768
+
3769
+ def visit_ParallelWithBlockNode(self, node):
3770
+ if not self.nogil_state:
3771
+ error(node.pos, "The parallel section may only be used without "
3772
+ "the GIL")
3773
+ return None
3774
+ if self.nogil_state == Nodes.NoGilState.NoGilScope:
3775
+ # We're in a "nogil" function but that doesn't prove we didn't
3776
+ # have the gil, so release it
3777
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
3778
+ node.scope_gil_state_known = False
3779
+ return self.visit_GILStatNode(node)
3780
+
3781
+ if node.nogil_check:
3782
+ # It does not currently implement this, but test for it anyway to
3783
+ # avoid potential future surprises
3784
+ node.nogil_check(self.env_stack[-1])
3785
+
3786
+ self.visitchildren(node)
3787
+ return node
3788
+
3789
+ def visit_TryFinallyStatNode(self, node):
3790
+ """
3791
+ Take care of try/finally statements in nogil code sections.
3792
+ """
3793
+ if not self.nogil_state:
3794
+ return self.visit_Node(node)
3795
+
3796
+ node.nogil_check = None
3797
+ node.is_try_finally_in_nogil = True
3798
+ self.visitchildren(node)
3799
+ return node
3800
+
3801
+ def visit_CriticalSectionStatNode(self, node):
3802
+ # skip normal "try/finally node" handling
3803
+ return self.visit_Node(node)
3804
+
3805
+ def visit_CythonLockStatNode(self, node):
3806
+ # skip normal "try/finally node" handling
3807
+ return self.visit_Node(node)
3808
+
3809
+ def visit_GILExitNode(self, node):
3810
+ if self.nogil_state_at_current_gilstatnode == Nodes.NoGilState.NoGilScope:
3811
+ node.scope_gil_state_known = False
3812
+ self.visitchildren(node)
3813
+ return node
3814
+
3815
+ def visit_Node(self, node):
3816
+ if self.env_stack and self.nogil_state and node.nogil_check:
3817
+ node.nogil_check(self.env_stack[-1])
3818
+ if node.outer_attrs:
3819
+ self._visit_scoped_children(node, self.nogil_state)
3820
+ else:
3821
+ self.visitchildren(node)
3822
+ if self.nogil_state:
3823
+ node.in_nogil_context = self.nogil_state
3824
+ return node
3825
+
3826
+ def visit_SimpleCallNode(self, node):
3827
+ if (node.self and node.self.type.is_cython_lock_type and
3828
+ node.function.is_attribute and node.function.attribute == "acquire" and
3829
+ len(node.args) == 1):
3830
+ # For the cython lock types we can optimize if we know the GIL state.
3831
+ # (Remove this in the distant future when it's all PyMutexes because for these
3832
+ # it doesn't matter)
3833
+ suffix = None
3834
+ if self.nogil_state == Nodes.NoGilState.NoGil:
3835
+ suffix = "Nogil"
3836
+ elif self.nogil_state == Nodes.NoGilState.HasGil:
3837
+ suffix = "Gil"
3838
+ if suffix:
3839
+ node = ExprNodes.PythonCapiCallNode(
3840
+ node.pos,
3841
+ node.function.entry.cname + suffix,
3842
+ node.function.type,
3843
+ args=[node.self],
3844
+ )
3845
+ return self.visit_Node(node)
3846
+
3847
+
3848
+ class CoerceCppTemps(EnvTransform, SkipDeclarations):
3849
+ """
3850
+ For temporary expression that are implemented using std::optional it's necessary the temps are
3851
+ assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform
3852
+ inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be
3853
+ inserted into the tree)
3854
+
3855
+ TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_result() and ExprNode.lhs_result()???
3856
+ """
3857
+ def visit_ModuleNode(self, node):
3858
+ if self.current_env().cpp:
3859
+ # skipping this makes it essentially free for C files
3860
+ self.visitchildren(node)
3861
+ return node
3862
+
3863
+ def visit_ExprNode(self, node):
3864
+ self.visitchildren(node)
3865
+ if (self.current_env().directives['cpp_locals'] and
3866
+ node.result_in_temp() and node.type.is_cpp_class and
3867
+ # Fake references are not replaced with "std::optional()".
3868
+ not node.type.is_fake_reference):
3869
+ node = ExprNodes.CppOptionalTempCoercion(node)
3870
+
3871
+ return node
3872
+
3873
+ def visit_CppOptionalTempCoercion(self, node):
3874
+ return node
3875
+
3876
+ def visit_CppIteratorNode(self, node):
3877
+ return node
3878
+
3879
+ def visit_ExprStatNode(self, node):
3880
+ # Deliberately skip `expr` in ExprStatNode - we don't need to access it.
3881
+ self.visitchildren(node.expr)
3882
+ return node
3883
+
3884
+
3885
+ class TransformBuiltinMethods(EnvTransform):
3886
+ """
3887
+ Replace Cython's own cython.* builtins by the corresponding tree nodes.
3888
+ Also handle some Python special builtin functions (e.g. super()/locals())
3889
+ that require introspection by the compiler.
3890
+ """
3891
+ def __init__(self, *args, **kwds):
3892
+ super().__init__(*args, **kwds)
3893
+ self.def_node_body_insertions = {}
3894
+
3895
+ def visit_SingleAssignmentNode(self, node):
3896
+ if node.declaration_only:
3897
+ return None
3898
+ else:
3899
+ self.visitchildren(node)
3900
+ return node
3901
+
3902
+ def visit_AttributeNode(self, node):
3903
+ self.visitchildren(node)
3904
+ return self.visit_cython_attribute(node)
3905
+
3906
+ def visit_NameNode(self, node):
3907
+ if node.name == u'__class__':
3908
+ lenv = self.current_env()
3909
+ entry = lenv.lookup_here(u'__class__')
3910
+ if not entry:
3911
+ node = self._inject_class(node)
3912
+ return self.visit_cython_attribute(node)
3913
+
3914
+ def visit_cython_attribute(self, node):
3915
+ attribute = node.as_cython_attribute()
3916
+ if attribute:
3917
+ if attribute == '__version__':
3918
+ from .. import __version__ as version
3919
+ node = ExprNodes.UnicodeNode(node.pos, value=EncodedString(version))
3920
+ elif attribute == 'NULL':
3921
+ node = ExprNodes.NullNode(node.pos)
3922
+ elif attribute in ('set', 'frozenset', 'staticmethod'):
3923
+ node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
3924
+ entry=self.current_env().builtin_scope().lookup_here(attribute))
3925
+ elif PyrexTypes.parse_basic_type(attribute):
3926
+ pass
3927
+ elif self.context.cython_scope.lookup_qualified_name(attribute):
3928
+ pass
3929
+ else:
3930
+ error(node.pos, "'%s' not a valid cython attribute or is being used incorrectly" % attribute)
3931
+ return node
3932
+
3933
+ def visit_ExecStatNode(self, node):
3934
+ lenv = self.current_env()
3935
+ self.visitchildren(node)
3936
+ if len(node.args) == 1:
3937
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
3938
+ if not lenv.is_module_scope:
3939
+ node.args.append(
3940
+ ExprNodes.LocalsExprNode(
3941
+ node.pos, self.current_scope_node(), lenv))
3942
+ return node
3943
+
3944
+ def _inject_locals(self, node, func_name):
3945
+ # locals()/dir()/vars() builtins
3946
+ lenv = self.current_env()
3947
+ entry = lenv.lookup_here(func_name)
3948
+ if entry:
3949
+ # not the builtin
3950
+ return node
3951
+ pos = node.pos
3952
+ if func_name in ('locals', 'vars'):
3953
+ if func_name == 'locals' and len(node.args) > 0:
3954
+ error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
3955
+ % len(node.args))
3956
+ return node
3957
+ elif func_name == 'vars':
3958
+ if len(node.args) > 1:
3959
+ error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
3960
+ % len(node.args))
3961
+ if len(node.args) > 0:
3962
+ return node # nothing to do
3963
+ return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
3964
+ else: # dir()
3965
+ if len(node.args) > 1:
3966
+ error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
3967
+ % len(node.args))
3968
+ if len(node.args) > 0:
3969
+ # optimised in Builtin.py
3970
+ return node
3971
+ if lenv.is_py_class_scope or lenv.is_module_scope:
3972
+ if lenv.is_py_class_scope:
3973
+ pyclass = self.current_scope_node()
3974
+ locals_dict = ExprNodes.CloneNode(pyclass.dict)
3975
+ else:
3976
+ locals_dict = ExprNodes.GlobalsExprNode(pos)
3977
+ return ExprNodes.SortedDictKeysNode(locals_dict)
3978
+ local_names = sorted(var.name for var in lenv.entries.values() if var.name)
3979
+ items = [ExprNodes.IdentifierStringNode(pos, value=var)
3980
+ for var in local_names]
3981
+ return ExprNodes.ListNode(pos, args=items)
3982
+
3983
+ def visit_PrimaryCmpNode(self, node):
3984
+ # special case: for in/not-in test, we do not need to sort locals()
3985
+ self.visitchildren(node)
3986
+ if node.operator in 'not_in': # in/not_in
3987
+ if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
3988
+ arg = node.operand2.arg
3989
+ if isinstance(arg, ExprNodes.NoneCheckNode):
3990
+ arg = arg.arg
3991
+ node.operand2 = arg
3992
+ return node
3993
+
3994
+ def visit_CascadedCmpNode(self, node):
3995
+ return self.visit_PrimaryCmpNode(node)
3996
+
3997
+ def _inject_eval(self, node, func_name):
3998
+ lenv = self.current_env()
3999
+ entry = lenv.lookup(func_name)
4000
+ if len(node.args) != 1 or (entry and not entry.is_builtin):
4001
+ return node
4002
+ # Inject globals and locals
4003
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
4004
+ if not lenv.is_module_scope:
4005
+ node.args.append(
4006
+ ExprNodes.LocalsExprNode(
4007
+ node.pos, self.current_scope_node(), lenv))
4008
+ return node
4009
+
4010
+ def _inject_class(self, node):
4011
+ # bare __class__ reference inside function
4012
+ current_def_node = self.current_scope_node()
4013
+
4014
+ if not isinstance(current_def_node, Nodes.FuncDefNode):
4015
+ return node
4016
+
4017
+ # Go up the stack, find the first class node and its direct method (i.e. function).
4018
+ fdef_node = class_node = generator_node = None
4019
+ for stack_node, stack_scope in reversed(self.env_stack):
4020
+ if isinstance(stack_node, Nodes.ClassDefNode):
4021
+ class_node = stack_node
4022
+ class_scope = stack_scope
4023
+ break
4024
+ elif isinstance(stack_node, Nodes.GeneratorDefNode):
4025
+ generator_node = stack_node
4026
+ fdef_node = stack_node.gbody
4027
+ fdef_scope = stack_scope
4028
+ elif isinstance(stack_node, Nodes.FuncDefNode):
4029
+ fdef_node = stack_node
4030
+ fdef_scope = stack_scope
4031
+
4032
+ if not fdef_node or not class_node:
4033
+ # failed to find a class or function
4034
+ return node
4035
+
4036
+ # now we arrange to inject:
4037
+ # __class__ = ... at the start of the def_node body
4038
+ # The advantage of doing it like this is that it automatically appears in locals()
4039
+ # and it can be captured by inner functions
4040
+ if fdef_node not in self.def_node_body_insertions:
4041
+ pos = fdef_node.body.pos
4042
+ if class_scope.is_c_class_scope:
4043
+ # c-classes can be resolved at compile-time, so they have a simpler
4044
+ # implementation
4045
+ rhs = ExprNodes.NameNode(
4046
+ pos, name=class_node.scope.name,
4047
+ entry=class_node.entry)
4048
+ elif class_scope.is_py_class_scope:
4049
+ rhs = ExprNodes.ClassCellNode(pos, is_generator=generator_node is not None)
4050
+ if generator_node:
4051
+ generator_node.requires_classobj = True
4052
+ else:
4053
+ fdef_node.requires_classobj = True
4054
+ class_node.class_cell.is_active = True
4055
+ else:
4056
+ return node # should never happen
4057
+
4058
+ assign_node = Nodes.SingleAssignmentNode(pos,
4059
+ lhs=ExprNodes.NameNode(pos, name=EncodedString("__class__")),
4060
+ rhs=rhs)
4061
+
4062
+ assign_node.analyse_declarations(fdef_scope)
4063
+
4064
+ assert fdef_node not in self.def_node_body_insertions
4065
+ self.def_node_body_insertions[fdef_node] = assign_node
4066
+
4067
+ return node
4068
+
4069
+ def _inject_super(self, node, func_name):
4070
+ lenv = self.current_env()
4071
+ entry = lenv.lookup_here(func_name)
4072
+ if entry or node.args:
4073
+ return node
4074
+ # Inject no-args super
4075
+ def_node = self.current_scope_node()
4076
+ if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2:
4077
+ return node
4078
+ class_node, class_scope = self.env_stack[-2]
4079
+ if class_scope.is_py_class_scope:
4080
+ def_node.requires_classobj = True
4081
+ class_node.class_cell.is_active = True
4082
+ node.args = [
4083
+ ExprNodes.ClassCellNode(
4084
+ node.pos, is_generator=def_node.is_generator),
4085
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4086
+ ]
4087
+ elif class_scope.is_c_class_scope:
4088
+ node.args = [
4089
+ ExprNodes.NameNode(
4090
+ node.pos, name=class_node.scope.name,
4091
+ entry=class_node.entry),
4092
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4093
+ ]
4094
+ return node
4095
+
4096
+ def _do_body_insertion(self, node):
4097
+ body_insertion = self.def_node_body_insertions.pop(node, None)
4098
+ if body_insertion:
4099
+ if isinstance(node.body, Nodes.StatListNode):
4100
+ node.body.stats.insert(0, body_insertion)
4101
+ else:
4102
+ node.body = Nodes.StatListNode(node.body.pos,
4103
+ stats=[body_insertion, node.body])
4104
+
4105
+ def visit_FuncDefNode(self, node):
4106
+ node = super().visit_FuncDefNode(node)
4107
+ self._do_body_insertion(node)
4108
+ return node
4109
+
4110
+ def visit_GeneratorBodyDefNode(self, node):
4111
+ node = super().visit_GeneratorBodyDefNode(node)
4112
+ self._do_body_insertion(node)
4113
+ return node
4114
+
4115
+ def visit_SimpleCallNode(self, node):
4116
+ # cython.foo
4117
+ function = node.function.as_cython_attribute()
4118
+ if function:
4119
+ if function in InterpretCompilerDirectives.unop_method_nodes:
4120
+ if len(node.args) != 1:
4121
+ error(node.function.pos, "%s() takes exactly one argument" % function)
4122
+ else:
4123
+ node = InterpretCompilerDirectives.unop_method_nodes[function](
4124
+ node.function.pos, operand=node.args[0])
4125
+ elif function in InterpretCompilerDirectives.binop_method_nodes:
4126
+ if len(node.args) != 2:
4127
+ error(node.function.pos, "%s() takes exactly two arguments" % function)
4128
+ else:
4129
+ node = InterpretCompilerDirectives.binop_method_nodes[function](
4130
+ node.function.pos, operand1=node.args[0], operand2=node.args[1])
4131
+ elif function == 'cast':
4132
+ if len(node.args) != 2:
4133
+ error(node.function.pos,
4134
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4135
+ else:
4136
+ type = node.args[0].analyse_as_type(self.current_env())
4137
+ if type:
4138
+ node = ExprNodes.TypecastNode(
4139
+ node.function.pos, type=type, operand=node.args[1], typecheck=False)
4140
+ else:
4141
+ error(node.args[0].pos, "Not a type")
4142
+ elif function == 'sizeof':
4143
+ if len(node.args) != 1:
4144
+ error(node.function.pos, "sizeof() takes exactly one argument")
4145
+ else:
4146
+ type = node.args[0].analyse_as_type(self.current_env())
4147
+ if type:
4148
+ node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
4149
+ else:
4150
+ node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
4151
+ elif function == 'cmod':
4152
+ if len(node.args) != 2:
4153
+ error(node.function.pos, "cmod() takes exactly two arguments")
4154
+ else:
4155
+ node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
4156
+ node.cdivision = True
4157
+ elif function == 'cdiv':
4158
+ if len(node.args) != 2:
4159
+ error(node.function.pos, "cdiv() takes exactly two arguments")
4160
+ else:
4161
+ node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
4162
+ node.cdivision = True
4163
+ elif function == 'set':
4164
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
4165
+ elif function == 'staticmethod':
4166
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod'))
4167
+ elif self.context.cython_scope.lookup_qualified_name(function):
4168
+ pass
4169
+ else:
4170
+ error(node.function.pos,
4171
+ "'%s' not a valid cython language construct" % function)
4172
+
4173
+ self.visitchildren(node)
4174
+
4175
+ if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
4176
+ func_name = node.function.name
4177
+ if func_name in ('dir', 'locals', 'vars'):
4178
+ return self._inject_locals(node, func_name)
4179
+ if func_name == 'eval':
4180
+ return self._inject_eval(node, func_name)
4181
+ if func_name == 'super':
4182
+ return self._inject_super(node, func_name)
4183
+ return node
4184
+
4185
+ def visit_GeneralCallNode(self, node):
4186
+ function = node.function.as_cython_attribute()
4187
+ if function == 'cast':
4188
+ # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args
4189
+ args = node.positional_args.args
4190
+ kwargs = node.keyword_args.compile_time_value(None)
4191
+ if (len(args) != 2 or len(kwargs) > 1 or
4192
+ (len(kwargs) == 1 and 'typecheck' not in kwargs)):
4193
+ error(node.function.pos,
4194
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4195
+ else:
4196
+ type = args[0].analyse_as_type(self.current_env())
4197
+ if type:
4198
+ typecheck = kwargs.get('typecheck', False)
4199
+ node = ExprNodes.TypecastNode(
4200
+ node.function.pos, type=type, operand=args[1], typecheck=typecheck)
4201
+ else:
4202
+ error(args[0].pos, "Not a type")
4203
+
4204
+ self.visitchildren(node)
4205
+ return node
4206
+
4207
+
4208
+ class ReplaceFusedTypeChecks(VisitorTransform):
4209
+ """
4210
+ This is not a transform in the pipeline. It is invoked on the specific
4211
+ versions of a cdef function with fused argument types. It filters out any
4212
+ type branches that don't match. e.g.
4213
+
4214
+ if fused_t is mytype:
4215
+ ...
4216
+ elif fused_t in other_fused_type:
4217
+ ...
4218
+ """
4219
+ def __init__(self, local_scope):
4220
+ super().__init__()
4221
+ self.local_scope = local_scope
4222
+ # defer the import until now to avoid circular import time dependencies
4223
+ from .Optimize import ConstantFolding
4224
+ self.transform = ConstantFolding(reevaluate=True)
4225
+
4226
+ def visit_IfStatNode(self, node):
4227
+ """
4228
+ Filters out any if clauses with false compile time type check
4229
+ expression.
4230
+ """
4231
+ self.visitchildren(node)
4232
+ return self.transform(node)
4233
+
4234
+ def visit_GILStatNode(self, node):
4235
+ """
4236
+ Fold constant condition of GILStatNode.
4237
+ """
4238
+ self.visitchildren(node)
4239
+ return self.transform(node)
4240
+
4241
+ def visit_PrimaryCmpNode(self, node):
4242
+ with Errors.local_errors(ignore=True):
4243
+ type1 = node.operand1.analyse_as_type(self.local_scope)
4244
+ type2 = node.operand2.analyse_as_type(self.local_scope)
4245
+
4246
+ if type1 and type2:
4247
+ false_node = ExprNodes.BoolNode(node.pos, value=False)
4248
+ true_node = ExprNodes.BoolNode(node.pos, value=True)
4249
+
4250
+ type1 = self.specialize_type(type1, node.operand1.pos)
4251
+ op = node.operator
4252
+
4253
+ if op in ('is', 'is_not', '==', '!='):
4254
+ type2 = self.specialize_type(type2, node.operand2.pos)
4255
+
4256
+ is_same = type1.same_as(type2)
4257
+ eq = op in ('is', '==')
4258
+
4259
+ if (is_same and eq) or (not is_same and not eq):
4260
+ return true_node
4261
+
4262
+ elif op in ('in', 'not_in'):
4263
+ # We have to do an instance check directly, as operand2
4264
+ # needs to be a fused type and not a type with a subtype
4265
+ # that is fused. First unpack the typedef
4266
+ if isinstance(type2, PyrexTypes.CTypedefType):
4267
+ type2 = type2.typedef_base_type
4268
+
4269
+ if type1.is_fused:
4270
+ error(node.operand1.pos, "Type is fused")
4271
+ elif not type2.is_fused:
4272
+ error(node.operand2.pos,
4273
+ "Can only use 'in' or 'not in' on a fused type")
4274
+ else:
4275
+ types = PyrexTypes.get_specialized_types(type2)
4276
+
4277
+ for specialized_type in types:
4278
+ if type1.same_as(specialized_type):
4279
+ if op == 'in':
4280
+ return true_node
4281
+ else:
4282
+ return false_node
4283
+
4284
+ if op == 'not_in':
4285
+ return true_node
4286
+
4287
+ return false_node
4288
+
4289
+ return node
4290
+
4291
+ def specialize_type(self, type, pos):
4292
+ try:
4293
+ return type.specialize(self.local_scope.fused_to_specific)
4294
+ except KeyError:
4295
+ error(pos, "Type is not specific")
4296
+ return type
4297
+
4298
+ def visit_Node(self, node):
4299
+ self.visitchildren(node)
4300
+ return node
4301
+
4302
+
4303
+ class DebugTransform(CythonTransform):
4304
+ """
4305
+ Write debug information for this Cython module.
4306
+ """
4307
+
4308
+ def __init__(self, context, options, result):
4309
+ super().__init__(context)
4310
+ self.visited = set()
4311
+ # our treebuilder and debug output writer
4312
+ # (see Cython.Debugger.debug_output.CythonDebugWriter)
4313
+ self.tb = self.context.gdb_debug_outputwriter
4314
+ #self.c_output_file = options.output_file
4315
+ self.c_output_file = result.c_file
4316
+
4317
+ # Closure support, basically treat nested functions as if the AST were
4318
+ # never nested
4319
+ self.nested_funcdefs = []
4320
+
4321
+ # tells visit_NameNode whether it should register step-into functions
4322
+ self.register_stepinto = False
4323
+
4324
+ def visit_ModuleNode(self, node):
4325
+ self.tb.module_name = node.full_module_name
4326
+ attrs = dict(
4327
+ module_name=node.full_module_name,
4328
+ filename=node.pos[0].filename,
4329
+ c_filename=self.c_output_file)
4330
+
4331
+ self.tb.start('Module', attrs)
4332
+
4333
+ # serialize functions
4334
+ self.tb.start('Functions')
4335
+ # First, serialize functions normally...
4336
+ self.visitchildren(node)
4337
+
4338
+ # ... then, serialize nested functions
4339
+ for nested_funcdef in self.nested_funcdefs:
4340
+ self.visit_FuncDefNode(nested_funcdef)
4341
+
4342
+ self.register_stepinto = True
4343
+ self.serialize_modulenode_as_function(node)
4344
+ self.register_stepinto = False
4345
+ self.tb.end('Functions')
4346
+
4347
+ # 2.3 compatibility. Serialize global variables
4348
+ self.tb.start('Globals')
4349
+ entries = {}
4350
+
4351
+ for k, v in node.scope.entries.items():
4352
+ if (v.qualified_name not in self.visited and not
4353
+ v.name.startswith('__pyx_') and not
4354
+ v.type.is_cfunction and not
4355
+ v.type.is_extension_type):
4356
+ entries[k]= v
4357
+
4358
+ self.serialize_local_variables(entries)
4359
+ self.tb.end('Globals')
4360
+ # self.tb.end('Module') # end Module after the line number mapping in
4361
+ # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
4362
+ return node
4363
+
4364
+ def visit_FuncDefNode(self, node):
4365
+ self.visited.add(node.local_scope.qualified_name)
4366
+
4367
+ if getattr(node, 'is_wrapper', False):
4368
+ return node
4369
+
4370
+ if self.register_stepinto:
4371
+ self.nested_funcdefs.append(node)
4372
+ return node
4373
+
4374
+ # node.entry.visibility = 'extern'
4375
+ if node.py_func is None:
4376
+ pf_cname = ''
4377
+ else:
4378
+ pf_cname = node.py_func.entry.func_cname
4379
+
4380
+ # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_*
4381
+ # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_*
4382
+ # In all cases, cname will be the name of the function containing the actual code
4383
+ cname = node.entry.pyfunc_cname or node.entry.func_cname
4384
+
4385
+ attrs = dict(
4386
+ name=node.entry.name or getattr(node, 'name', '<unknown>'),
4387
+ cname=cname,
4388
+ pf_cname=pf_cname,
4389
+ qualified_name=node.local_scope.qualified_name,
4390
+ lineno=str(node.pos[1]))
4391
+
4392
+ self.tb.start('Function', attrs=attrs)
4393
+
4394
+ self.tb.start('Locals')
4395
+ self.serialize_local_variables(node.local_scope.entries)
4396
+ self.tb.end('Locals')
4397
+
4398
+ self.tb.start('Arguments')
4399
+ for arg in node.local_scope.arg_entries:
4400
+ self.tb.start(arg.name)
4401
+ self.tb.end(arg.name)
4402
+ self.tb.end('Arguments')
4403
+
4404
+ self.tb.start('StepIntoFunctions')
4405
+ self.register_stepinto = True
4406
+ self.visitchildren(node)
4407
+ self.register_stepinto = False
4408
+ self.tb.end('StepIntoFunctions')
4409
+ self.tb.end('Function')
4410
+
4411
+ return node
4412
+
4413
+ def visit_NameNode(self, node):
4414
+ if (self.register_stepinto and
4415
+ node.type is not None and
4416
+ node.type.is_cfunction and
4417
+ getattr(node, 'is_called', False) and
4418
+ node.entry.func_cname is not None):
4419
+ # don't check node.entry.in_cinclude, as 'cdef extern: ...'
4420
+ # declared functions are not 'in_cinclude'.
4421
+ # This means we will list called 'cdef' functions as
4422
+ # "step into functions", but this is not an issue as they will be
4423
+ # recognized as Cython functions anyway.
4424
+ attrs = dict(name=node.entry.func_cname)
4425
+ self.tb.start('StepIntoFunction', attrs=attrs)
4426
+ self.tb.end('StepIntoFunction')
4427
+
4428
+ self.visitchildren(node)
4429
+ return node
4430
+
4431
+ def serialize_modulenode_as_function(self, node):
4432
+ """
4433
+ Serialize the module-level code as a function so the debugger will know
4434
+ it's a "relevant frame" and it will know where to set the breakpoint
4435
+ for 'break modulename'.
4436
+ """
4437
+ self._serialize_modulenode_as_function(node, dict(
4438
+ name=node.full_module_name.rpartition('.')[-1],
4439
+ cname=node.module_init_func_cname(),
4440
+ pf_cname='',
4441
+ # Ignore the qualified_name, breakpoints should be set using
4442
+ # `cy break modulename:lineno` for module-level breakpoints.
4443
+ qualified_name='',
4444
+ lineno='1',
4445
+ is_initmodule_function="True",
4446
+ ))
4447
+
4448
+ def _serialize_modulenode_as_function(self, node, attrs):
4449
+ self.tb.start('Function', attrs=attrs)
4450
+
4451
+ self.tb.start('Locals')
4452
+ self.serialize_local_variables(node.scope.entries)
4453
+ self.tb.end('Locals')
4454
+
4455
+ self.tb.start('Arguments')
4456
+ self.tb.end('Arguments')
4457
+
4458
+ self.tb.start('StepIntoFunctions')
4459
+ self.register_stepinto = True
4460
+ self.visitchildren(node)
4461
+ self.register_stepinto = False
4462
+ self.tb.end('StepIntoFunctions')
4463
+
4464
+ self.tb.end('Function')
4465
+
4466
+ def serialize_local_variables(self, entries):
4467
+ for entry in entries.values():
4468
+ if not entry.cname:
4469
+ # not a local variable
4470
+ continue
4471
+ if entry.type.is_pyobject:
4472
+ vartype = 'PythonObject'
4473
+ else:
4474
+ vartype = 'CObject'
4475
+
4476
+ if entry.from_closure:
4477
+ # We're dealing with a closure where a variable from an outer
4478
+ # scope is accessed, get it from the scope object.
4479
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4480
+ entry.outer_entry.cname)
4481
+
4482
+ qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
4483
+ entry.scope.name,
4484
+ entry.name)
4485
+ elif entry.in_closure:
4486
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4487
+ entry.cname)
4488
+ qname = entry.qualified_name
4489
+ else:
4490
+ cname = entry.cname
4491
+ qname = entry.qualified_name
4492
+
4493
+ if not entry.pos:
4494
+ # this happens for variables that are not in the user's code,
4495
+ # e.g. for the global __builtins__, __doc__, etc. We can just
4496
+ # set the lineno to 0 for those.
4497
+ lineno = '0'
4498
+ else:
4499
+ lineno = str(entry.pos[1])
4500
+
4501
+ attrs = dict(
4502
+ name=entry.name,
4503
+ cname=cname,
4504
+ qualified_name=qname,
4505
+ type=vartype,
4506
+ lineno=lineno)
4507
+
4508
+ self.tb.start('LocalVar', attrs)
4509
+ self.tb.end('LocalVar')