Cython 3.3.0__cp315-cp315-win_amd64.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (339) hide show
  1. Cython/Build/BuildExecutable.py +156 -0
  2. Cython/Build/Cache.py +199 -0
  3. Cython/Build/Cythonize.py +349 -0
  4. Cython/Build/Dependencies.py +1281 -0
  5. Cython/Build/Distutils.py +1 -0
  6. Cython/Build/Inline.py +467 -0
  7. Cython/Build/IpythonMagic.py +559 -0
  8. Cython/Build/SharedModule.py +165 -0
  9. Cython/Build/Tests/TestCyCache.py +195 -0
  10. Cython/Build/Tests/TestCythonizeArgsParser.py +480 -0
  11. Cython/Build/Tests/TestDependencies.py +133 -0
  12. Cython/Build/Tests/TestInline.py +177 -0
  13. Cython/Build/Tests/TestIpythonMagic.py +303 -0
  14. Cython/Build/Tests/TestRecythonize.py +212 -0
  15. Cython/Build/Tests/TestStripLiterals.py +155 -0
  16. Cython/Build/Tests/__init__.py +1 -0
  17. Cython/Build/__init__.py +11 -0
  18. Cython/CodeWriter.py +825 -0
  19. Cython/Compiler/AnalysedTreeTransforms.py +97 -0
  20. Cython/Compiler/Annotate.py +328 -0
  21. Cython/Compiler/AutoDocTransforms.py +320 -0
  22. Cython/Compiler/Buffer.py +679 -0
  23. Cython/Compiler/Builtin.py +1102 -0
  24. Cython/Compiler/CmdLine.py +373 -0
  25. Cython/Compiler/Code.cp315-win_amd64.pyd +0 -0
  26. Cython/Compiler/Code.pxd +154 -0
  27. Cython/Compiler/Code.py +3760 -0
  28. Cython/Compiler/CodeGeneration.py +33 -0
  29. Cython/Compiler/CythonScope.py +208 -0
  30. Cython/Compiler/Dataclass.py +890 -0
  31. Cython/Compiler/DebugFlags.py +24 -0
  32. Cython/Compiler/Errors.py +310 -0
  33. Cython/Compiler/ExprNodes.py +16273 -0
  34. Cython/Compiler/FlowControl.cp315-win_amd64.pyd +0 -0
  35. Cython/Compiler/FlowControl.pxd +112 -0
  36. Cython/Compiler/FlowControl.py +1573 -0
  37. Cython/Compiler/FusedNode.cp315-win_amd64.pyd +0 -0
  38. Cython/Compiler/FusedNode.py +978 -0
  39. Cython/Compiler/Future.py +16 -0
  40. Cython/Compiler/Interpreter.py +57 -0
  41. Cython/Compiler/Lexicon.py +422 -0
  42. Cython/Compiler/LineTable.cp315-win_amd64.pyd +0 -0
  43. Cython/Compiler/LineTable.py +114 -0
  44. Cython/Compiler/Main.py +856 -0
  45. Cython/Compiler/MatchCaseNodes.py +2200 -0
  46. Cython/Compiler/MemoryView.py +930 -0
  47. Cython/Compiler/ModuleNode.py +4548 -0
  48. Cython/Compiler/Naming.py +370 -0
  49. Cython/Compiler/Nodes.py +11304 -0
  50. Cython/Compiler/Optimize.py +5564 -0
  51. Cython/Compiler/Options.py +840 -0
  52. Cython/Compiler/ParseTreeTransforms.pxd +80 -0
  53. Cython/Compiler/ParseTreeTransforms.py +4808 -0
  54. Cython/Compiler/Parsing.cp315-win_amd64.pyd +0 -0
  55. Cython/Compiler/Parsing.pxd +9 -0
  56. Cython/Compiler/Parsing.py +4809 -0
  57. Cython/Compiler/Pipeline.py +439 -0
  58. Cython/Compiler/PyrexTypes.py +6588 -0
  59. Cython/Compiler/Pythran.py +232 -0
  60. Cython/Compiler/Scanning.cp315-win_amd64.pyd +0 -0
  61. Cython/Compiler/Scanning.pxd +70 -0
  62. Cython/Compiler/Scanning.py +720 -0
  63. Cython/Compiler/StringEncoding.cp315-win_amd64.pyd +0 -0
  64. Cython/Compiler/StringEncoding.py +354 -0
  65. Cython/Compiler/Symtab.py +3121 -0
  66. Cython/Compiler/Tests/TestBuffer.py +105 -0
  67. Cython/Compiler/Tests/TestBuiltin.py +196 -0
  68. Cython/Compiler/Tests/TestCmdLine.py +652 -0
  69. Cython/Compiler/Tests/TestCode.py +145 -0
  70. Cython/Compiler/Tests/TestFlowControl.py +65 -0
  71. Cython/Compiler/Tests/TestGrammar.py +202 -0
  72. Cython/Compiler/Tests/TestMemView.py +71 -0
  73. Cython/Compiler/Tests/TestParseTreeTransforms.py +285 -0
  74. Cython/Compiler/Tests/TestScanning.py +132 -0
  75. Cython/Compiler/Tests/TestSignatureMatching.py +73 -0
  76. Cython/Compiler/Tests/TestStringEncoding.py +20 -0
  77. Cython/Compiler/Tests/TestTreeFragment.py +63 -0
  78. Cython/Compiler/Tests/TestTreePath.py +103 -0
  79. Cython/Compiler/Tests/TestTypes.py +119 -0
  80. Cython/Compiler/Tests/TestUtilityLoad.py +112 -0
  81. Cython/Compiler/Tests/TestVisitor.py +119 -0
  82. Cython/Compiler/Tests/Utils.py +36 -0
  83. Cython/Compiler/Tests/__init__.py +1 -0
  84. Cython/Compiler/TreeFragment.py +279 -0
  85. Cython/Compiler/TreePath.py +303 -0
  86. Cython/Compiler/TypeInference.py +611 -0
  87. Cython/Compiler/TypeSlots.py +1329 -0
  88. Cython/Compiler/UFuncs.py +317 -0
  89. Cython/Compiler/UtilNodes.py +389 -0
  90. Cython/Compiler/UtilityCode.py +354 -0
  91. Cython/Compiler/Version.py +8 -0
  92. Cython/Compiler/Visitor.cp315-win_amd64.pyd +0 -0
  93. Cython/Compiler/Visitor.pxd +52 -0
  94. Cython/Compiler/Visitor.py +914 -0
  95. Cython/Compiler/__init__.py +1 -0
  96. Cython/Coverage.py +448 -0
  97. Cython/Debugger/Cygdb.py +214 -0
  98. Cython/Debugger/DebugWriter.py +82 -0
  99. Cython/Debugger/Tests/TestLibCython.py +280 -0
  100. Cython/Debugger/Tests/__init__.py +1 -0
  101. Cython/Debugger/Tests/cfuncs.c +8 -0
  102. Cython/Debugger/Tests/codefile +49 -0
  103. Cython/Debugger/Tests/test_libcython_in_gdb.py +580 -0
  104. Cython/Debugger/Tests/test_libpython_in_gdb.py +90 -0
  105. Cython/Debugger/__init__.py +1 -0
  106. Cython/Debugger/libcython.py +1548 -0
  107. Cython/Debugger/libpython.py +2821 -0
  108. Cython/Debugging.py +20 -0
  109. Cython/Distutils/__init__.py +2 -0
  110. Cython/Distutils/build_ext.py +143 -0
  111. Cython/Distutils/extension.py +96 -0
  112. Cython/Distutils/old_build_ext.py +351 -0
  113. Cython/Includes/cpython/__init__.pxd +173 -0
  114. Cython/Includes/cpython/array.pxd +152 -0
  115. Cython/Includes/cpython/bool.pxd +37 -0
  116. Cython/Includes/cpython/buffer.pxd +112 -0
  117. Cython/Includes/cpython/bytearray.pxd +33 -0
  118. Cython/Includes/cpython/bytes.pxd +200 -0
  119. Cython/Includes/cpython/cellobject.pxd +35 -0
  120. Cython/Includes/cpython/ceval.pxd +8 -0
  121. Cython/Includes/cpython/codecs.pxd +121 -0
  122. Cython/Includes/cpython/complex.pxd +60 -0
  123. Cython/Includes/cpython/contextvars.pxd +145 -0
  124. Cython/Includes/cpython/conversion.pxd +36 -0
  125. Cython/Includes/cpython/datetime.pxd +395 -0
  126. Cython/Includes/cpython/descr.pxd +26 -0
  127. Cython/Includes/cpython/dict.pxd +268 -0
  128. Cython/Includes/cpython/exc.pxd +263 -0
  129. Cython/Includes/cpython/fileobject.pxd +57 -0
  130. Cython/Includes/cpython/float.pxd +56 -0
  131. Cython/Includes/cpython/frozendict.pxd +37 -0
  132. Cython/Includes/cpython/function.pxd +65 -0
  133. Cython/Includes/cpython/genobject.pxd +25 -0
  134. Cython/Includes/cpython/getargs.pxd +12 -0
  135. Cython/Includes/cpython/instance.pxd +25 -0
  136. Cython/Includes/cpython/iterator.pxd +36 -0
  137. Cython/Includes/cpython/iterobject.pxd +24 -0
  138. Cython/Includes/cpython/list.pxd +144 -0
  139. Cython/Includes/cpython/long.pxd +180 -0
  140. Cython/Includes/cpython/longintrepr.pxd +14 -0
  141. Cython/Includes/cpython/mapping.pxd +63 -0
  142. Cython/Includes/cpython/marshal.pxd +66 -0
  143. Cython/Includes/cpython/mem.pxd +120 -0
  144. Cython/Includes/cpython/memoryview.pxd +50 -0
  145. Cython/Includes/cpython/method.pxd +49 -0
  146. Cython/Includes/cpython/module.pxd +208 -0
  147. Cython/Includes/cpython/number.pxd +258 -0
  148. Cython/Includes/cpython/object.pxd +430 -0
  149. Cython/Includes/cpython/pycapsule.pxd +143 -0
  150. Cython/Includes/cpython/pylifecycle.pxd +68 -0
  151. Cython/Includes/cpython/pyport.pxd +8 -0
  152. Cython/Includes/cpython/pystate.pxd +95 -0
  153. Cython/Includes/cpython/pythread.pxd +53 -0
  154. Cython/Includes/cpython/ref.pxd +141 -0
  155. Cython/Includes/cpython/sentinel.pxd +17 -0
  156. Cython/Includes/cpython/sequence.pxd +134 -0
  157. Cython/Includes/cpython/set.pxd +119 -0
  158. Cython/Includes/cpython/slice.pxd +70 -0
  159. Cython/Includes/cpython/time.pxd +129 -0
  160. Cython/Includes/cpython/tuple.pxd +72 -0
  161. Cython/Includes/cpython/type.pxd +146 -0
  162. Cython/Includes/cpython/unicode.pxd +639 -0
  163. Cython/Includes/cpython/version.pxd +32 -0
  164. Cython/Includes/cpython/weakref.pxd +78 -0
  165. Cython/Includes/libc/__init__.pxd +1 -0
  166. Cython/Includes/libc/complex.pxd +35 -0
  167. Cython/Includes/libc/errno.pxd +127 -0
  168. Cython/Includes/libc/float.pxd +43 -0
  169. Cython/Includes/libc/limits.pxd +28 -0
  170. Cython/Includes/libc/locale.pxd +46 -0
  171. Cython/Includes/libc/math.pxd +209 -0
  172. Cython/Includes/libc/setjmp.pxd +10 -0
  173. Cython/Includes/libc/signal.pxd +64 -0
  174. Cython/Includes/libc/stddef.pxd +9 -0
  175. Cython/Includes/libc/stdint.pxd +105 -0
  176. Cython/Includes/libc/stdio.pxd +80 -0
  177. Cython/Includes/libc/stdlib.pxd +72 -0
  178. Cython/Includes/libc/string.pxd +50 -0
  179. Cython/Includes/libc/threads.pxd +234 -0
  180. Cython/Includes/libc/time.pxd +52 -0
  181. Cython/Includes/libcpp/__init__.pxd +4 -0
  182. Cython/Includes/libcpp/algorithm.pxd +320 -0
  183. Cython/Includes/libcpp/any.pxd +16 -0
  184. Cython/Includes/libcpp/atomic.pxd +59 -0
  185. Cython/Includes/libcpp/barrier.pxd +22 -0
  186. Cython/Includes/libcpp/bit.pxd +29 -0
  187. Cython/Includes/libcpp/cast.pxd +12 -0
  188. Cython/Includes/libcpp/cmath.pxd +518 -0
  189. Cython/Includes/libcpp/complex.pxd +106 -0
  190. Cython/Includes/libcpp/condition_variable.pxd +322 -0
  191. Cython/Includes/libcpp/deque.pxd +165 -0
  192. Cython/Includes/libcpp/exception.pxd +216 -0
  193. Cython/Includes/libcpp/execution.pxd +15 -0
  194. Cython/Includes/libcpp/forward_list.pxd +63 -0
  195. Cython/Includes/libcpp/functional.pxd +26 -0
  196. Cython/Includes/libcpp/future.pxd +103 -0
  197. Cython/Includes/libcpp/iterator.pxd +34 -0
  198. Cython/Includes/libcpp/latch.pxd +17 -0
  199. Cython/Includes/libcpp/limits.pxd +61 -0
  200. Cython/Includes/libcpp/list.pxd +117 -0
  201. Cython/Includes/libcpp/map.pxd +252 -0
  202. Cython/Includes/libcpp/memory.pxd +115 -0
  203. Cython/Includes/libcpp/mutex.pxd +387 -0
  204. Cython/Includes/libcpp/numbers.pxd +15 -0
  205. Cython/Includes/libcpp/numeric.pxd +131 -0
  206. Cython/Includes/libcpp/optional.pxd +34 -0
  207. Cython/Includes/libcpp/pair.pxd +1 -0
  208. Cython/Includes/libcpp/queue.pxd +25 -0
  209. Cython/Includes/libcpp/random.pxd +166 -0
  210. Cython/Includes/libcpp/semaphore.pxd +43 -0
  211. Cython/Includes/libcpp/set.pxd +228 -0
  212. Cython/Includes/libcpp/shared_mutex.pxd +96 -0
  213. Cython/Includes/libcpp/span.pxd +87 -0
  214. Cython/Includes/libcpp/stack.pxd +11 -0
  215. Cython/Includes/libcpp/stop_token.pxd +117 -0
  216. Cython/Includes/libcpp/string.pxd +355 -0
  217. Cython/Includes/libcpp/string_view.pxd +183 -0
  218. Cython/Includes/libcpp/typeindex.pxd +15 -0
  219. Cython/Includes/libcpp/typeinfo.pxd +10 -0
  220. Cython/Includes/libcpp/unordered_map.pxd +193 -0
  221. Cython/Includes/libcpp/unordered_set.pxd +152 -0
  222. Cython/Includes/libcpp/utility.pxd +30 -0
  223. Cython/Includes/libcpp/vector.pxd +186 -0
  224. Cython/Includes/numpy/math.pxd +150 -0
  225. Cython/Includes/openmp.pxd +50 -0
  226. Cython/Includes/posix/__init__.pxd +1 -0
  227. Cython/Includes/posix/dlfcn.pxd +14 -0
  228. Cython/Includes/posix/fcntl.pxd +86 -0
  229. Cython/Includes/posix/ioctl.pxd +4 -0
  230. Cython/Includes/posix/mman.pxd +101 -0
  231. Cython/Includes/posix/resource.pxd +57 -0
  232. Cython/Includes/posix/select.pxd +21 -0
  233. Cython/Includes/posix/signal.pxd +73 -0
  234. Cython/Includes/posix/stat.pxd +98 -0
  235. Cython/Includes/posix/stdio.pxd +37 -0
  236. Cython/Includes/posix/stdlib.pxd +29 -0
  237. Cython/Includes/posix/strings.pxd +9 -0
  238. Cython/Includes/posix/time.pxd +71 -0
  239. Cython/Includes/posix/types.pxd +30 -0
  240. Cython/Includes/posix/uio.pxd +26 -0
  241. Cython/Includes/posix/unistd.pxd +271 -0
  242. Cython/Includes/posix/wait.pxd +38 -0
  243. Cython/LZSS.cp315-win_amd64.pyd +0 -0
  244. Cython/LZSS.py +184 -0
  245. Cython/Plex/Actions.cp315-win_amd64.pyd +0 -0
  246. Cython/Plex/Actions.pxd +24 -0
  247. Cython/Plex/Actions.py +119 -0
  248. Cython/Plex/DFA.cp315-win_amd64.pyd +0 -0
  249. Cython/Plex/DFA.pxd +14 -0
  250. Cython/Plex/DFA.py +164 -0
  251. Cython/Plex/Errors.py +48 -0
  252. Cython/Plex/Lexicons.py +178 -0
  253. Cython/Plex/Machines.cp315-win_amd64.pyd +0 -0
  254. Cython/Plex/Machines.pxd +36 -0
  255. Cython/Plex/Machines.py +238 -0
  256. Cython/Plex/Regexps.py +535 -0
  257. Cython/Plex/Scanners.cp315-win_amd64.pyd +0 -0
  258. Cython/Plex/Scanners.pxd +45 -0
  259. Cython/Plex/Scanners.py +328 -0
  260. Cython/Plex/Transitions.cp315-win_amd64.pyd +0 -0
  261. Cython/Plex/Transitions.pxd +14 -0
  262. Cython/Plex/Transitions.py +239 -0
  263. Cython/Plex/__init__.py +34 -0
  264. Cython/Runtime/__init__.py +1 -0
  265. Cython/Runtime/refnanny.cp315-win_amd64.pyd +0 -0
  266. Cython/Runtime/refnanny.pyx +237 -0
  267. Cython/Shadow.py +1174 -0
  268. Cython/StringIOTree.cp315-win_amd64.pyd +0 -0
  269. Cython/StringIOTree.py +169 -0
  270. Cython/Tempita/__init__.py +4 -0
  271. Cython/Tempita/_looper.py +154 -0
  272. Cython/Tempita/_tempita.cp315-win_amd64.pyd +0 -0
  273. Cython/Tempita/_tempita.py +1087 -0
  274. Cython/TestUtils.py +472 -0
  275. Cython/Tests/TestCodeWriter.py +128 -0
  276. Cython/Tests/TestCythonUtils.py +202 -0
  277. Cython/Tests/TestJediTyper.py +223 -0
  278. Cython/Tests/TestShadow.py +125 -0
  279. Cython/Tests/TestStringIOTree.py +68 -0
  280. Cython/Tests/TestTestUtils.py +89 -0
  281. Cython/Tests/__init__.py +1 -0
  282. Cython/Tests/xmlrunner.py +390 -0
  283. Cython/Utility/AsyncGen.c +1152 -0
  284. Cython/Utility/Buffer.c +866 -0
  285. Cython/Utility/BufferFormatFromTypeInfo.pxd +2 -0
  286. Cython/Utility/Builtins.c +1068 -0
  287. Cython/Utility/CConvert.pyx +153 -0
  288. Cython/Utility/CMath.c +104 -0
  289. Cython/Utility/CommonStructures.c +244 -0
  290. Cython/Utility/Complex.c +378 -0
  291. Cython/Utility/Coroutine.c +2344 -0
  292. Cython/Utility/CpdefEnums.pyx +119 -0
  293. Cython/Utility/CppConvert.pyx +282 -0
  294. Cython/Utility/CppSupport.cpp +151 -0
  295. Cython/Utility/CythonFunction.c +2185 -0
  296. Cython/Utility/Dataclasses.c +101 -0
  297. Cython/Utility/Embed.c +129 -0
  298. Cython/Utility/Exceptions.c +1331 -0
  299. Cython/Utility/Exceptions_Cy.pyx +109 -0
  300. Cython/Utility/ExtensionTypes.c +1199 -0
  301. Cython/Utility/FunctionArguments.c +1052 -0
  302. Cython/Utility/FusedFunction.pyx +44 -0
  303. Cython/Utility/ImportExport.c +972 -0
  304. Cython/Utility/MatchCase.c +981 -0
  305. Cython/Utility/MatchCase_Cy.pyx +12 -0
  306. Cython/Utility/MemoryView.pxd +108 -0
  307. Cython/Utility/MemoryView.pyx +1499 -0
  308. Cython/Utility/MemoryView_C.c +1056 -0
  309. Cython/Utility/ModuleSetupCode.c +3319 -0
  310. Cython/Utility/NumpyImportArray.c +46 -0
  311. Cython/Utility/ObjectHandling.c +3404 -0
  312. Cython/Utility/Optimize.c +2564 -0
  313. Cython/Utility/Overflow.c +378 -0
  314. Cython/Utility/Profile.c +736 -0
  315. Cython/Utility/StringTools.c +1534 -0
  316. Cython/Utility/Synchronization.c +438 -0
  317. Cython/Utility/TString.c +369 -0
  318. Cython/Utility/TestCyUtilityLoader.pyx +8 -0
  319. Cython/Utility/TestCythonScope.pyx +75 -0
  320. Cython/Utility/TestUtilityLoader.c +12 -0
  321. Cython/Utility/TypeConversion.c +1588 -0
  322. Cython/Utility/UFuncs.pyx +50 -0
  323. Cython/Utility/UFuncs_C.c +89 -0
  324. Cython/Utility/__init__.py +28 -0
  325. Cython/Utility/arrayarray.h +172 -0
  326. Cython/Utils.cp315-win_amd64.pyd +0 -0
  327. Cython/Utils.py +680 -0
  328. Cython/__init__.py +12 -0
  329. Cython/_shared.cp315-win_amd64.pyd +0 -0
  330. Cython/py.typed +0 -0
  331. cython-3.3.0.dist-info/METADATA +555 -0
  332. cython-3.3.0.dist-info/RECORD +339 -0
  333. cython-3.3.0.dist-info/WHEEL +5 -0
  334. cython-3.3.0.dist-info/entry_points.txt +4 -0
  335. cython-3.3.0.dist-info/top_level.txt +3 -0
  336. cython.py +29 -0
  337. pyximport/__init__.py +4 -0
  338. pyximport/pyxbuild.py +160 -0
  339. pyximport/pyximport.py +482 -0
@@ -0,0 +1,4808 @@
1
+ import cython
2
+ cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
3
+ Options=object, UtilNodes=object, LetNode=object,
4
+ LetRefNode=object, TreeFragment=object, EncodedString=object,
5
+ error=object, warning=object, copy=object, hashlib=object,
6
+ itemgetter=object)
7
+
8
+ import copy
9
+ import hashlib
10
+ from operator import itemgetter
11
+
12
+ from . import Code
13
+ from . import PyrexTypes
14
+ from . import Naming
15
+ from . import ExprNodes
16
+ from . import Nodes
17
+ from . import Options
18
+ from . import Builtin
19
+ from . import Errors
20
+
21
+ from .Visitor import VisitorTransform, TreeVisitor
22
+ from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
23
+ from .UtilNodes import LetNode, LetRefNode
24
+ from .TreeFragment import TreeFragment
25
+ from .StringEncoding import EncodedString
26
+ from .Errors import error, warning, CompileError, InternalError
27
+
28
+
29
+ class SkipDeclarations:
30
+ """
31
+ Variable and function declarations can often have a deep tree structure,
32
+ and yet most transformations don't need to descend to this depth.
33
+
34
+ Declaration nodes are removed after AnalyseDeclarationsTransform, so there
35
+ is no need to use this for transformations after that point.
36
+ """
37
+ def visit_CTypeDefNode(self, node):
38
+ return node
39
+
40
+ def visit_CVarDefNode(self, node):
41
+ return node
42
+
43
+ def visit_CDeclaratorNode(self, node):
44
+ return node
45
+
46
+ def visit_CBaseTypeNode(self, node):
47
+ return node
48
+
49
+ def visit_CEnumDefNode(self, node):
50
+ return node
51
+
52
+ def visit_CStructOrUnionDefNode(self, node):
53
+ return node
54
+
55
+ def visit_CppClassNode(self, node):
56
+ if node.visibility != "extern":
57
+ # Need to traverse methods.
58
+ self.visitchildren(node)
59
+ return node
60
+
61
+
62
+ class NormalizeTree(CythonTransform):
63
+ """
64
+ This transform fixes up a few things after parsing
65
+ in order to make the parse tree more suitable for
66
+ transforms.
67
+
68
+ a) After parsing, blocks with only one statement will
69
+ be represented by that statement, not by a StatListNode.
70
+ When doing transforms this is annoying and inconsistent,
71
+ as one cannot in general remove a statement in a consistent
72
+ way and so on. This transform wraps any single statements
73
+ in a StatListNode containing a single statement.
74
+
75
+ b) The PassStatNode is a noop and serves no purpose beyond
76
+ plugging such one-statement blocks; i.e., once parsed a
77
+ ` "pass" can just as well be represented using an empty
78
+ StatListNode. This means less special cases to worry about
79
+ in subsequent transforms (one always checks to see if a
80
+ StatListNode has no children to see if the block is empty).
81
+ """
82
+
83
+ def __init__(self, context):
84
+ super().__init__(context)
85
+ self.is_in_statlist = False
86
+ self.is_in_expr = False
87
+
88
+ def visit_ModuleNode(self, node):
89
+ self.visitchildren(node)
90
+ if not isinstance(node.body, Nodes.StatListNode):
91
+ # This can happen when the body only consists of a single (unused) declaration and no statements.
92
+ node.body = Nodes.StatListNode(pos=node.pos, stats=[node.body])
93
+ return node
94
+
95
+ def visit_ExprNode(self, node):
96
+ stacktmp = self.is_in_expr
97
+ self.is_in_expr = True
98
+ self.visitchildren(node)
99
+ self.is_in_expr = stacktmp
100
+ return node
101
+
102
+ def visit_StatNode(self, node, is_listcontainer=False):
103
+ stacktmp = self.is_in_statlist
104
+ self.is_in_statlist = is_listcontainer
105
+ self.visitchildren(node)
106
+ self.is_in_statlist = stacktmp
107
+ if not self.is_in_statlist and not self.is_in_expr:
108
+ return Nodes.StatListNode(pos=node.pos, stats=[node])
109
+ else:
110
+ return node
111
+
112
+ def visit_StatListNode(self, node):
113
+ self.is_in_statlist = True
114
+ self.visitchildren(node)
115
+ self.is_in_statlist = False
116
+ return node
117
+
118
+ def visit_ParallelAssignmentNode(self, node):
119
+ return self.visit_StatNode(node, True)
120
+
121
+ def visit_CEnumDefNode(self, node):
122
+ return self.visit_StatNode(node, True)
123
+
124
+ def visit_CStructOrUnionDefNode(self, node):
125
+ return self.visit_StatNode(node, True)
126
+
127
+ def visit_ExprStatNode(self, node):
128
+ """Eliminate useless string literals"""
129
+ if node.expr.is_string_literal:
130
+ return Nodes.PassStatNode(node.expr.pos)
131
+ else:
132
+ return self.visit_StatNode(node)
133
+
134
+ def visit_CDeclaratorNode(self, node):
135
+ return node
136
+
137
+
138
+ class PostParseError(CompileError): pass
139
+
140
+ # error strings checked by unit tests, so define them
141
+ ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
142
+ ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
143
+ ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
144
+ class PostParse(ScopeTrackingTransform):
145
+ """
146
+ Basic interpretation of the parse tree, as well as validity
147
+ checking that can be done on a very basic level on the parse
148
+ tree (while still not being a problem with the basic syntax,
149
+ as such).
150
+
151
+ Specifically:
152
+ - Default values to cdef assignments are turned into single
153
+ assignments following the declaration (everywhere but in class
154
+ bodies, where they raise a compile error)
155
+
156
+ - Interpret some node structures into Python runtime values.
157
+ Some nodes take compile-time arguments (currently:
158
+ TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
159
+ which should be interpreted. This happens in a general way
160
+ and other steps should be taken to ensure validity.
161
+
162
+ Type arguments cannot be interpreted in this way.
163
+
164
+ - For __cythonbufferdefaults__ the arguments are checked for
165
+ validity.
166
+
167
+ TemplatedTypeNode has its directives interpreted:
168
+ Any first positional argument goes into the "dtype" attribute,
169
+ any "ndim" keyword argument goes into the "ndim" attribute and
170
+ so on. Also it is checked that the directive combination is valid.
171
+ - __cythonbufferdefaults__ attributes are parsed and put into the
172
+ type information.
173
+
174
+ Note: Currently Parsing.py does a lot of interpretation and
175
+ reorganization that can be refactored into this transform
176
+ if a more pure Abstract Syntax Tree is wanted.
177
+
178
+ - Some invalid uses of := assignment expressions are detected
179
+
180
+ - Validate that return, continue and break aren't in except*
181
+ """
182
+ def __init__(self, context):
183
+ super().__init__(context)
184
+ self.specialattribute_handlers = {
185
+ '__cythonbufferdefaults__' : self.handle_bufferdefaults
186
+ }
187
+ self.except_star_validation_tracker = None
188
+ self.in_pattern_node = False
189
+
190
+ def visit_LambdaNode(self, node):
191
+ # unpack a lambda expression into the corresponding DefNode
192
+ collector = YieldNodeCollector()
193
+ collector.visitchildren(node.result_expr)
194
+ if collector.has_yield or collector.has_await or isinstance(node.result_expr, ExprNodes.YieldExprNode):
195
+ body = Nodes.ExprStatNode(
196
+ node.result_expr.pos, expr=node.result_expr)
197
+ else:
198
+ body = Nodes.ReturnStatNode(
199
+ node.result_expr.pos, value=node.result_expr)
200
+ node.def_node = Nodes.DefNode(
201
+ node.pos, name=node.name,
202
+ args=node.args, star_arg=node.star_arg,
203
+ starstar_arg=node.starstar_arg,
204
+ body=body, doc=None)
205
+ self.visitchildren(node)
206
+ return node
207
+
208
+ def visit_GeneratorExpressionNode(self, node):
209
+ # unpack a generator expression into the corresponding DefNode
210
+ collector = YieldNodeCollector()
211
+ collector.visitchildren(node.loop, attrs=None, exclude=["iterator"])
212
+ node.def_node = Nodes.DefNode(
213
+ node.pos, name=node.name, doc=None,
214
+ args=[], star_arg=None, starstar_arg=None,
215
+ body=node.loop, is_async_def=collector.has_await,
216
+ is_generator_expression=True)
217
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
218
+ self.visitchildren(node)
219
+ return node
220
+
221
+ def visit_ComprehensionNode(self, node):
222
+ # enforce local scope also in Py2 for async generators (seriously, that's a Py3.6 feature...)
223
+ if not node.has_local_scope:
224
+ collector = YieldNodeCollector()
225
+ collector.visitchildren(node.loop)
226
+ if collector.has_await:
227
+ node.has_local_scope = True
228
+ _AssignmentExpressionChecker.do_checks(node.loop, scope_is_class=self.scope_type in ("pyclass", "cclass"))
229
+ self.visitchildren(node)
230
+ return node
231
+
232
+ # cdef variables
233
+ def handle_bufferdefaults(self, decl):
234
+ if not isinstance(decl.default, ExprNodes.DictNode):
235
+ raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
236
+ self.scope_node.buffer_defaults_node = decl.default
237
+ self.scope_node.buffer_defaults_pos = decl.pos
238
+
239
+ def visit_CVarDefNode(self, node):
240
+ # This assumes only plain names and pointers are assignable on
241
+ # declaration. Also, it makes use of the fact that a cdef decl
242
+ # must appear before the first use, so we don't have to deal with
243
+ # "i = 3; cdef int i = i" and can simply move the nodes around.
244
+ try:
245
+ self.visitchildren(node)
246
+ stats = [node]
247
+ newdecls = []
248
+ for decl in node.declarators:
249
+ declbase = decl
250
+ while isinstance(declbase, (Nodes.CPtrDeclaratorNode, Nodes.CQualifierDeclaratorNode)):
251
+ declbase = declbase.base
252
+ if isinstance(declbase, Nodes.CNameDeclaratorNode):
253
+ if declbase.default is not None:
254
+ if self.scope_type in ('cclass', 'pyclass', 'struct'):
255
+ if isinstance(self.scope_node, Nodes.CClassDefNode):
256
+ handler = self.specialattribute_handlers.get(decl.name)
257
+ if handler:
258
+ if decl is not declbase:
259
+ raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
260
+ handler(decl)
261
+ continue # Remove declaration
262
+ raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
263
+ first_assignment = self.scope_type != 'module'
264
+ stats.append(Nodes.SingleAssignmentNode(node.pos,
265
+ lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
266
+ rhs=declbase.default, first=first_assignment,
267
+ from_pxd_cvardef=node.in_pxd))
268
+ declbase.default = None
269
+ newdecls.append(decl)
270
+ node.declarators = newdecls
271
+ return stats
272
+ except PostParseError as e:
273
+ # An error in a cdef clause is ok, simply remove the declaration
274
+ # and try to move on to report more errors
275
+ self.context.nonfatal_error(e)
276
+ return None
277
+
278
+ # Split parallel assignments (a,b = b,a) into separate partial
279
+ # assignments that are executed rhs-first using temps. This
280
+ # restructuring must be applied before type analysis so that known
281
+ # types on rhs and lhs can be matched directly. It is required in
282
+ # the case that the types cannot be coerced to a Python type in
283
+ # order to assign from a tuple.
284
+
285
+ def visit_SingleAssignmentNode(self, node):
286
+ self.visitchildren(node)
287
+ return self._visit_assignment_node(node, [node.lhs, node.rhs])
288
+
289
+ def visit_CascadedAssignmentNode(self, node):
290
+ self.visitchildren(node)
291
+ return self._visit_assignment_node(node, node.lhs_list + [node.rhs])
292
+
293
+ def _visit_assignment_node(self, node, expr_list):
294
+ """Flatten parallel assignments into separate single
295
+ assignments or cascaded assignments.
296
+ """
297
+ if sum([ 1 for expr in expr_list
298
+ if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
299
+ # no parallel assignments => nothing to do
300
+ return node
301
+
302
+ expr_list_list = []
303
+ flatten_parallel_assignments(expr_list, expr_list_list)
304
+ temp_refs = []
305
+ eliminate_rhs_duplicates(expr_list_list, temp_refs)
306
+
307
+ nodes = []
308
+ for expr_list in expr_list_list:
309
+ lhs_list = expr_list[:-1]
310
+ rhs = expr_list[-1]
311
+ if len(lhs_list) == 1:
312
+ node = Nodes.SingleAssignmentNode(rhs.pos,
313
+ lhs = lhs_list[0], rhs = rhs)
314
+ else:
315
+ node = Nodes.CascadedAssignmentNode(rhs.pos,
316
+ lhs_list = lhs_list, rhs = rhs)
317
+ nodes.append(node)
318
+
319
+ if len(nodes) == 1:
320
+ assign_node = nodes[0]
321
+ else:
322
+ assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
323
+
324
+ if temp_refs:
325
+ duplicates_and_temps = [ (temp.expression, temp)
326
+ for temp in temp_refs ]
327
+ sort_common_subsequences(duplicates_and_temps)
328
+ for _, temp_ref in duplicates_and_temps[::-1]:
329
+ assign_node = LetNode(temp_ref, assign_node)
330
+
331
+ return assign_node
332
+
333
+ def _flatten_sequence(self, seq, result):
334
+ for arg in seq.args:
335
+ if arg.is_sequence_constructor:
336
+ self._flatten_sequence(arg, result)
337
+ else:
338
+ result.append(arg)
339
+ return result
340
+
341
+ def visit_DelStatNode(self, node):
342
+ self.visitchildren(node)
343
+ node.args = self._flatten_sequence(node, [])
344
+ return node
345
+
346
+ def visit_ExceptClauseNode(self, node):
347
+ if node.is_except_as:
348
+ # except-as must delete NameNode target at the end
349
+ del_target = Nodes.DelStatNode(
350
+ node.pos,
351
+ args=[ExprNodes.NameNode(
352
+ node.target.pos, name=node.target.name)],
353
+ ignore_nonexisting=True)
354
+ node.body = Nodes.StatListNode(
355
+ node.pos,
356
+ stats=[Nodes.TryFinallyStatNode(
357
+ node.pos,
358
+ body=node.body,
359
+ finally_clause=Nodes.StatListNode(
360
+ node.pos,
361
+ stats=[del_target]))])
362
+ self.visitchildren(node)
363
+ return node
364
+
365
+ def visit_AssertStatNode(self, node):
366
+ """Extract the exception raising into a RaiseStatNode to simplify GIL handling.
367
+ """
368
+ if node.exception is None:
369
+ node.exception = Nodes.RaiseStatNode(
370
+ node.pos,
371
+ exc_type=ExprNodes.NameNode(node.pos, name=EncodedString("AssertionError")),
372
+ exc_value=node.value,
373
+ exc_tb=None,
374
+ cause=None,
375
+ builtin_exc_name="AssertionError",
376
+ wrap_tuple_value=True,
377
+ )
378
+ node.value = None
379
+ self.visitchildren(node)
380
+ return node
381
+
382
+ def _track_node_for_except_star_validation(self, node):
383
+ old_validation_tracker = self.except_star_validation_tracker
384
+ self.except_star_validation_tracker = node
385
+ self.visitchildren(node)
386
+ self.except_star_validation_tracker = old_validation_tracker
387
+ return node
388
+
389
+ def visit_LoopNode(self, node):
390
+ return self._track_node_for_except_star_validation(node)
391
+
392
+ def visit_FuncDefNode(self, node):
393
+ old_validation_tracker = self.except_star_validation_tracker
394
+ self.except_star_validation_tracker = node
395
+ node = super(PostParse, self).visit_FuncDefNode(node)
396
+ self.except_star_validation_tracker = old_validation_tracker
397
+ return node
398
+
399
+ def visit_ExceptStarChainNode(self, node):
400
+ return self._track_node_for_except_star_validation(node)
401
+
402
+ def _validate_break_return_continue_in_except_star(self, node):
403
+ if isinstance(self.except_star_validation_tracker, Nodes.ExceptStarChainNode):
404
+ # error message copied from Python 3.11
405
+ raise PostParseError(node.pos, "'break', 'continue' and 'return' cannot appear in an except* block")
406
+ self.visitchildren(node)
407
+ return node
408
+
409
+ def visit_ReturnStatNode(self, node):
410
+ return self._validate_break_return_continue_in_except_star(node)
411
+
412
+ def visit_BreakStatNode(self, node):
413
+ return self._validate_break_return_continue_in_except_star(node)
414
+
415
+ def visit_ContinueStatNode(self, node):
416
+ return self._validate_break_return_continue_in_except_star(node)
417
+
418
+ def visit_ErrorNode(self, node):
419
+ error(node.pos, node.what)
420
+ return None
421
+
422
+ def visit_MatchCaseNode(self, node):
423
+ node.validate_targets()
424
+ self.visitchildren(node)
425
+ return node
426
+
427
+ def visit_MatchNode(self, node):
428
+ node.validate_irrefutable()
429
+ self.visitchildren(node)
430
+ return node
431
+
432
+ def visit_PatternNode(self, node):
433
+ in_pattern_node, self.in_pattern_node = self.in_pattern_node, True
434
+ self.visitchildren(node)
435
+ self.in_pattern_node = in_pattern_node
436
+ return node
437
+
438
+ def visit_JoinedStrNode(self, node):
439
+ if self.in_pattern_node:
440
+ error(node.pos, "f-strings are not accepted for pattern matching")
441
+ self.visitchildren(node)
442
+ return node
443
+
444
+ def visit_DefNode(self, node):
445
+ if (self.scope_type == "cclass" and
446
+ node.name in ["__getreadbuffer__", "__getwritebuffer__", "__getsegcount__", "__getcharbuffer__"]):
447
+ warning(node.pos, f"'{node.name}' relates to the old Python 2 buffer protocol "
448
+ "and is no longer used.", 2)
449
+ return None # drop the node - the arguments are invalid for a def node
450
+ return self.visit_FuncDefNode(node)
451
+
452
+
453
+ class _AssignmentExpressionTargetNameFinder(TreeVisitor):
454
+ def __init__(self):
455
+ super().__init__()
456
+ self.target_names = {}
457
+
458
+ def find_target_names(self, target):
459
+ if target.is_name:
460
+ return [target.name]
461
+ elif target.is_sequence_constructor:
462
+ names = []
463
+ for arg in target.args:
464
+ names.extend(self.find_target_names(arg))
465
+ return names
466
+ # other targets are possible, but it isn't necessary to investigate them here
467
+ return []
468
+
469
+ def visit_ForInStatNode(self, node):
470
+ self.target_names[node] = tuple(self.find_target_names(node.target))
471
+ self.visitchildren(node)
472
+
473
+ def visit_ComprehensionNode(self, node):
474
+ pass # don't recurse into nested comprehensions
475
+
476
+ def visit_LambdaNode(self, node):
477
+ pass # don't recurse into nested lambdas/generator expressions
478
+
479
+ def visit_Node(self, node):
480
+ self.visitchildren(node)
481
+
482
+
483
+ class _AssignmentExpressionChecker(TreeVisitor):
484
+ """
485
+ Enforces rules on AssignmentExpressions within generator expressions and comprehensions
486
+ """
487
+ def __init__(self, loop_node, scope_is_class):
488
+ super().__init__()
489
+
490
+ target_name_finder = _AssignmentExpressionTargetNameFinder()
491
+ target_name_finder.visit(loop_node)
492
+ self.target_names_dict = target_name_finder.target_names
493
+ self.in_iterator = False
494
+ self.in_nested_generator = False
495
+ self.scope_is_class = scope_is_class
496
+ self.current_target_names = ()
497
+ self.all_target_names = set()
498
+ for names in self.target_names_dict.values():
499
+ self.all_target_names.update(names)
500
+
501
+ def _reset_state(self):
502
+ old_state = (self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names)
503
+ # note: not resetting self.in_iterator here, see visit_LambdaNode() below
504
+ self.in_nested_generator = False
505
+ self.scope_is_class = False
506
+ self.current_target_names = ()
507
+ self.all_target_names = set()
508
+ return old_state
509
+
510
+ def _set_state(self, old_state):
511
+ self.in_iterator, self.in_nested_generator, self.scope_is_class, self.all_target_names, self.current_target_names = old_state
512
+
513
+ @classmethod
514
+ def do_checks(cls, loop_node, scope_is_class):
515
+ checker = cls(loop_node, scope_is_class)
516
+ checker.visit(loop_node)
517
+
518
+ def visit_ForInStatNode(self, node):
519
+ if self.in_nested_generator:
520
+ self.visitchildren(node) # once nested, don't do anything special
521
+ return
522
+
523
+ current_target_names = self.current_target_names
524
+ target_name = self.target_names_dict.get(node, None)
525
+ if target_name:
526
+ self.current_target_names += target_name
527
+
528
+ self.in_iterator = True
529
+ self.visit(node.iterator)
530
+ self.in_iterator = False
531
+ self.visitchildren(node, exclude=("iterator",))
532
+
533
+ self.current_target_names = current_target_names
534
+
535
+ def visit_AssignmentExpressionNode(self, node):
536
+ if self.in_iterator:
537
+ error(node.pos, "assignment expression cannot be used in a comprehension iterable expression")
538
+ if self.scope_is_class:
539
+ error(node.pos, "assignment expression within a comprehension cannot be used in a class body")
540
+ if node.target_name in self.current_target_names:
541
+ error(node.pos, "assignment expression cannot rebind comprehension iteration variable '%s'" %
542
+ node.target_name)
543
+ elif node.target_name in self.all_target_names:
544
+ error(node.pos, "comprehension inner loop cannot rebind assignment expression target '%s'" %
545
+ node.target_name)
546
+
547
+ def visit_LambdaNode(self, node):
548
+ # Don't reset "in_iterator" - an assignment expression in a lambda in an
549
+ # iterator is explicitly tested by the Python testcases and banned.
550
+ old_state = self._reset_state()
551
+ # the lambda node's "def_node" is not set up at this point, so we need to recurse into it explicitly.
552
+ self.visit(node.result_expr)
553
+ self._set_state(old_state)
554
+
555
+ def visit_ComprehensionNode(self, node):
556
+ in_nested_generator = self.in_nested_generator
557
+ self.in_nested_generator = True
558
+ self.visitchildren(node)
559
+ self.in_nested_generator = in_nested_generator
560
+
561
+ def visit_GeneratorExpressionNode(self, node):
562
+ in_nested_generator = self.in_nested_generator
563
+ self.in_nested_generator = True
564
+ # def_node isn't set up yet, so we need to visit the loop directly.
565
+ self.visit(node.loop)
566
+ self.in_nested_generator = in_nested_generator
567
+
568
+ def visit_Node(self, node):
569
+ self.visitchildren(node)
570
+
571
+
572
+ def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
573
+ """Replace rhs items by LetRefNodes if they appear more than once.
574
+ Creates a sequence of LetRefNodes that set up the required temps
575
+ and appends them to ref_node_sequence. The input list is modified
576
+ in-place.
577
+ """
578
+ seen_nodes = set()
579
+ ref_nodes = {}
580
+ def find_duplicates(node):
581
+ if node.is_literal or node.is_name:
582
+ # no need to replace those; can't include attributes here
583
+ # as their access is not necessarily side-effect free
584
+ return
585
+ if node in seen_nodes:
586
+ if node not in ref_nodes:
587
+ ref_node = LetRefNode(node)
588
+ ref_nodes[node] = ref_node
589
+ ref_node_sequence.append(ref_node)
590
+ else:
591
+ seen_nodes.add(node)
592
+ if node.is_sequence_constructor:
593
+ for item in node.args:
594
+ find_duplicates(item)
595
+
596
+ for expr_list in expr_list_list:
597
+ rhs = expr_list[-1]
598
+ find_duplicates(rhs)
599
+ if not ref_nodes:
600
+ return
601
+
602
+ def substitute_nodes(node):
603
+ if node in ref_nodes:
604
+ return ref_nodes[node]
605
+ elif node.is_sequence_constructor:
606
+ node.args = list(map(substitute_nodes, node.args))
607
+ return node
608
+
609
+ # replace nodes inside of the common subexpressions
610
+ for node in ref_nodes:
611
+ if node.is_sequence_constructor:
612
+ node.args = list(map(substitute_nodes, node.args))
613
+
614
+ # replace common subexpressions on all rhs items
615
+ for expr_list in expr_list_list:
616
+ expr_list[-1] = substitute_nodes(expr_list[-1])
617
+
618
+ def sort_common_subsequences(items):
619
+ """Sort items/subsequences so that all items and subsequences that
620
+ an item contains appear before the item itself. This is needed
621
+ because each rhs item must only be evaluated once, so its value
622
+ must be evaluated first and then reused when packing sequences
623
+ that contain it.
624
+
625
+ This implies a partial order, and the sort must be stable to
626
+ preserve the original order as much as possible, so we use a
627
+ simple insertion sort (which is very fast for short sequences, the
628
+ normal case in practice).
629
+ """
630
+ def contains(seq, x):
631
+ for item in seq:
632
+ if item is x:
633
+ return True
634
+ elif item.is_sequence_constructor and contains(item.args, x):
635
+ return True
636
+ return False
637
+ def lower_than(a,b):
638
+ return b.is_sequence_constructor and contains(b.args, a)
639
+
640
+ for pos, item in enumerate(items):
641
+ key = item[1] # the ResultRefNode which has already been injected into the sequences
642
+ new_pos = pos
643
+ for i in range(pos-1, -1, -1):
644
+ if lower_than(key, items[i][0]):
645
+ new_pos = i
646
+ if new_pos != pos:
647
+ for i in range(pos, new_pos, -1):
648
+ items[i] = items[i-1]
649
+ items[new_pos] = item
650
+
651
+
652
+ def unpack_string_to_character_literals(literal):
653
+ chars = []
654
+ pos = literal.pos
655
+ stype = literal.__class__
656
+ sval = literal.value
657
+ sval_type = sval.__class__
658
+ for char in sval:
659
+ cval = sval_type(char)
660
+ chars.append(stype(pos, value=cval))
661
+ return chars
662
+
663
+
664
+ @cython.cfunc
665
+ def flatten_parallel_assignments(input: list, output: list):
666
+ # The input is a list of expression nodes, representing the LHSs
667
+ # and RHS of one (possibly cascaded) assignment statement. For
668
+ # sequence constructors, rearranges the matching parts of both
669
+ # sides into a list of equivalent assignments between the
670
+ # individual elements. This transformation is applied
671
+ # recursively, so that nested structures get matched as well.
672
+ rhs = input[-1]
673
+ if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
674
+ or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
675
+ output.append(input)
676
+ return
677
+
678
+ complete_assignments = []
679
+
680
+ if rhs.is_sequence_constructor:
681
+ rhs_args = rhs.args
682
+ elif rhs.is_string_literal:
683
+ rhs_args = unpack_string_to_character_literals(rhs)
684
+
685
+ starred_targets: cython.Py_ssize_t
686
+ lhs_size: cython.Py_ssize_t
687
+ rhs_size: cython.Py_ssize_t = len(rhs_args)
688
+ lhs_targets = [[] for _ in range(rhs_size)]
689
+ starred_assignments = []
690
+
691
+ for lhs in input[:-1]:
692
+ if not lhs.is_sequence_constructor:
693
+ if lhs.is_starred:
694
+ error(lhs.pos, "starred assignment target must be in a list or tuple")
695
+ complete_assignments.append(lhs)
696
+ continue
697
+ lhs_size = len(lhs.args)
698
+ starred_targets = 0
699
+ for expr in lhs.args:
700
+ starred_targets += bool(expr.is_starred)
701
+ if starred_targets > 1:
702
+ error(lhs.pos, "more than 1 starred expression in assignment")
703
+ output.append([lhs,rhs])
704
+ continue
705
+ elif lhs_size - starred_targets > rhs_size:
706
+ error(lhs.pos, "need more than %d value%s to unpack"
707
+ % (rhs_size, (rhs_size != 1) and 's' or ''))
708
+ output.append([lhs,rhs])
709
+ continue
710
+ elif starred_targets:
711
+ map_starred_assignment(lhs_targets, starred_assignments,
712
+ lhs.args, rhs_args)
713
+ elif lhs_size < rhs_size:
714
+ error(lhs.pos, "too many values to unpack (expected %d, got %d)"
715
+ % (lhs_size, rhs_size))
716
+ output.append([lhs,rhs])
717
+ continue
718
+ else:
719
+ for targets, expr in zip(lhs_targets, lhs.args):
720
+ targets.append(expr)
721
+
722
+ if complete_assignments:
723
+ complete_assignments.append(rhs)
724
+ output.append(complete_assignments)
725
+
726
+ # recursively flatten partial assignments
727
+ for cascade, rhs in zip(lhs_targets, rhs_args):
728
+ if cascade:
729
+ cascade.append(rhs)
730
+ flatten_parallel_assignments(cascade, output)
731
+
732
+ # recursively flatten starred assignments
733
+ for cascade in starred_assignments:
734
+ if cascade[0].is_sequence_constructor:
735
+ flatten_parallel_assignments(cascade, output)
736
+ else:
737
+ output.append(cascade)
738
+
739
+
740
+ @cython.cfunc
741
+ def map_starred_assignment(lhs_targets: list, starred_assignments: list, lhs_args: list, rhs_args: list):
742
+ # Appends the fixed-position LHS targets to the target list that
743
+ # appear left and right of the starred argument.
744
+ #
745
+ # The starred_assignments list receives a new tuple
746
+ # (lhs_target, rhs_values_list) that maps the remaining arguments
747
+ # (those that match the starred target) to a list.
748
+
749
+ # left side of the starred target
750
+ i: cython.Py_ssize_t
751
+ starred: cython.Py_ssize_t
752
+ lhs_remaining: cython.Py_ssize_t
753
+ for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
754
+ if expr.is_starred:
755
+ starred = i
756
+ lhs_remaining = len(lhs_args) - i - 1
757
+ break
758
+ targets.append(expr)
759
+ else:
760
+ raise InternalError("no starred arg found when splitting starred assignment")
761
+
762
+ # right side of the starred target
763
+ for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
764
+ lhs_args[starred + 1:])):
765
+ targets.append(expr)
766
+
767
+ # the starred target itself, must be assigned a (potentially empty) list
768
+ target = lhs_args[starred].target # unpack starred node
769
+ starred_rhs = rhs_args[starred:]
770
+ if lhs_remaining:
771
+ starred_rhs = starred_rhs[:-lhs_remaining]
772
+ if starred_rhs:
773
+ pos = starred_rhs[0].pos
774
+ else:
775
+ pos = target.pos
776
+ starred_assignments.append([
777
+ target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])
778
+
779
+
780
+ class PxdPostParse(CythonTransform, SkipDeclarations):
781
+ """
782
+ Basic interpretation/validity checking that should only be
783
+ done on pxd trees.
784
+
785
+ A lot of this checking currently happens in the parser; but
786
+ what is listed below happens here.
787
+
788
+ - "def" functions are let through only if they fill the
789
+ getbuffer/releasebuffer slots
790
+
791
+ - cdef functions are let through only if they are on the
792
+ top level and are declared "inline"
793
+ """
794
+ ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
795
+ ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
796
+
797
+ def __call__(self, node):
798
+ self.scope_type = 'pxd'
799
+ return super().__call__(node)
800
+
801
+ def visit_CClassDefNode(self, node):
802
+ old = self.scope_type
803
+ self.scope_type = 'cclass'
804
+ self.visitchildren(node)
805
+ self.scope_type = old
806
+ return node
807
+
808
+ def visit_FuncDefNode(self, node):
809
+ # FuncDefNode always come with an implementation (without
810
+ # an imp they are CVarDefNodes..)
811
+ err = self.ERR_INLINE_ONLY
812
+
813
+ if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
814
+ and node.name in ('__getbuffer__', '__releasebuffer__')):
815
+ err = None # allow these slots
816
+
817
+ if isinstance(node, Nodes.CFuncDefNode):
818
+ if ('inline' in node.modifiers and
819
+ self.scope_type in ('pxd', 'cclass')):
820
+ node.inline_in_pxd = True
821
+ if node.visibility != 'private':
822
+ err = self.ERR_NOGO_WITH_INLINE % node.visibility
823
+ elif node.api:
824
+ err = self.ERR_NOGO_WITH_INLINE % 'api'
825
+ else:
826
+ err = None # allow inline function
827
+ else:
828
+ err = self.ERR_INLINE_ONLY
829
+
830
+ if err:
831
+ self.context.nonfatal_error(PostParseError(node.pos, err))
832
+ return None
833
+ else:
834
+ return node
835
+
836
+
837
+ class TrackNumpyAttributes(VisitorTransform, SkipDeclarations):
838
+ # TODO: Make name handling as good as in InterpretCompilerDirectives() below - probably best to merge the two.
839
+ def __init__(self):
840
+ super().__init__()
841
+ self.numpy_module_names = set()
842
+
843
+ def visit_CImportStatNode(self, node):
844
+ if node.module_name == "numpy":
845
+ self.numpy_module_names.add(node.as_name or "numpy")
846
+ return node
847
+
848
+ def visit_AttributeNode(self, node):
849
+ self.visitchildren(node)
850
+ obj = node.obj
851
+ if (obj.is_name and obj.name in self.numpy_module_names) or obj.is_numpy_attribute:
852
+ node.is_numpy_attribute = True
853
+ return node
854
+
855
+ visit_Node = VisitorTransform.recurse_to_children
856
+
857
+
858
+ class InterpretCompilerDirectives(CythonTransform):
859
+ """
860
+ After parsing, directives can be stored in a number of places:
861
+ - #cython-comments at the top of the file (stored in ModuleNode)
862
+ - Command-line arguments overriding these
863
+ - @cython.directivename decorators
864
+ - with cython.directivename: statements
865
+ - replaces "cython.compiled" with BoolNode(value=True)
866
+ allowing unreachable blocks to be removed at a fairly early stage
867
+ before cython typing rules are forced on applied
868
+
869
+ This transform is responsible for interpreting these various sources
870
+ and store the directive in two ways:
871
+ - Set the directives attribute of the ModuleNode for global directives.
872
+ - Use a CompilerDirectivesNode to override directives for a subtree.
873
+
874
+ (The first one is primarily to not have to modify with the tree
875
+ structure, so that ModuleNode stay on top.)
876
+
877
+ The directives are stored in dictionaries from name to value in effect.
878
+ Each such dictionary is always filled in for all possible directives,
879
+ using default values where no value is given by the user.
880
+
881
+ The available directives are controlled in Options.py.
882
+
883
+ Note that we have to run this prior to analysis, and so some minor
884
+ duplication of functionality has to occur: We manually track cimports
885
+ and which names the "cython" module may have been imported to.
886
+ """
887
+ unop_method_nodes = {
888
+ 'typeof': ExprNodes.TypeofNode,
889
+
890
+ 'operator.address': ExprNodes.AmpersandNode,
891
+ 'operator.dereference': ExprNodes.DereferenceNode,
892
+ 'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
893
+ 'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
894
+ 'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
895
+ 'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
896
+ 'operator.typeid' : ExprNodes.TypeidNode,
897
+
898
+ # For backwards compatibility.
899
+ 'address': ExprNodes.AmpersandNode,
900
+ }
901
+
902
+ binop_method_nodes = {
903
+ 'operator.comma' : ExprNodes.c_binop_constructor(','),
904
+ }
905
+
906
+ special_methods = {
907
+ 'declare', 'union', 'struct', 'typedef',
908
+ 'sizeof', 'cast', 'pointer', 'compiled',
909
+ 'NULL', 'fused_type', 'parallel', 'likely',
910
+ 'unlikely'
911
+ }
912
+ special_methods.update(unop_method_nodes)
913
+
914
+ valid_cython_submodules = {
915
+ 'cimports',
916
+ 'dataclasses',
917
+ 'operator',
918
+ 'parallel',
919
+ 'view',
920
+ }
921
+
922
+ valid_parallel_directives = {
923
+ "parallel",
924
+ "prange",
925
+ "threadid",
926
+ #"threadsavailable",
927
+ }
928
+
929
+ def __init__(self, context, compilation_directive_defaults):
930
+ super().__init__(context)
931
+ self.cython_module_names = set()
932
+ self.directive_names = {'staticmethod': 'staticmethod'}
933
+ self.parallel_directives = {}
934
+ directives = copy.deepcopy(Options.get_directive_defaults())
935
+ for key, value in compilation_directive_defaults.items():
936
+ directives[str(key)] = copy.deepcopy(value)
937
+ self.directives = directives
938
+
939
+ def check_directive_scope(self, pos, directive, scope):
940
+ legal_scopes = Options.directive_scopes.get(directive, None)
941
+ if legal_scopes and scope not in legal_scopes:
942
+ self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
943
+ 'is not allowed in %s scope' % (directive, scope)))
944
+ return False
945
+ else:
946
+ if directive not in Options.directive_types:
947
+ error(pos, "Invalid directive: '%s'." % (directive,))
948
+ return True
949
+
950
+ def _check_valid_cython_module(self, pos, module_name):
951
+ if not module_name.startswith("cython."):
952
+ return
953
+ submodule = module_name.split('.', 2)[1]
954
+ if submodule in self.valid_cython_submodules:
955
+ return
956
+
957
+ extra = ""
958
+ # This is very rarely used, so don't waste space on static tuples.
959
+ hints = [
960
+ line.split() for line in """\
961
+ imp cimports
962
+ cimp cimports
963
+ para parallel
964
+ parra parallel
965
+ dataclass dataclasses
966
+ """.splitlines()[:-1]
967
+ ]
968
+ for wrong, correct in hints:
969
+ if module_name.startswith("cython." + wrong):
970
+ extra = "Did you mean 'cython.%s' ?" % correct
971
+ break
972
+ if not extra:
973
+ is_simple_cython_name = submodule in Options.directive_types
974
+ if not is_simple_cython_name and not submodule.startswith("_"):
975
+ # Try to find it in the Shadow module (i.e. the pure Python namespace of cython.*).
976
+ # FIXME: use an internal reference of "cython.*" names instead of Shadow.py
977
+ from .. import Shadow
978
+ is_simple_cython_name = hasattr(Shadow, submodule)
979
+ if is_simple_cython_name:
980
+ extra = "Instead, use 'import cython' and then 'cython.%s'." % submodule
981
+
982
+ error(pos, "'%s' is not a valid cython.* module%s%s" % (
983
+ module_name,
984
+ ". " if extra else "",
985
+ extra,
986
+ ))
987
+
988
+ # Set up processing and handle the cython: comments.
989
+ def visit_ModuleNode(self, node):
990
+ for key in sorted(node.directive_comments):
991
+ if not self.check_directive_scope(node.pos, key, 'module'):
992
+ self.wrong_scope_error(node.pos, key, 'module')
993
+ del node.directive_comments[key]
994
+
995
+ self.module_scope = node.scope
996
+
997
+ self.directives.update(node.directive_comments)
998
+ node.directives = self.directives
999
+ node.parallel_directives = self.parallel_directives
1000
+ self.visitchildren(node)
1001
+ node.cython_module_names = self.cython_module_names
1002
+ return node
1003
+
1004
+ def visit_CompilerDirectivesMixin(self, node):
1005
+ with node.apply_directives(self):
1006
+ self.visitchildren(node)
1007
+ return node
1008
+
1009
+ # The following four functions track imports and cimports that
1010
+ # begin with "cython"
1011
+ def is_cython_directive(self, name):
1012
+ return (name in Options.directive_types or
1013
+ name in self.special_methods or
1014
+ PyrexTypes.parse_basic_type(name))
1015
+
1016
+ def is_parallel_directive(self, full_name, pos):
1017
+ """
1018
+ Checks to see if fullname (e.g. cython.parallel.prange) is a valid
1019
+ parallel directive. If it is a star import it also updates the
1020
+ parallel_directives.
1021
+ """
1022
+ result = (full_name + ".").startswith("cython.parallel.")
1023
+
1024
+ if result:
1025
+ directive = full_name.split('.')
1026
+ if full_name == "cython.parallel":
1027
+ self.parallel_directives["parallel"] = "cython.parallel"
1028
+ elif full_name == "cython.parallel.*":
1029
+ for name in self.valid_parallel_directives:
1030
+ self.parallel_directives[name] = "cython.parallel.%s" % name
1031
+ elif (len(directive) != 3 or
1032
+ directive[-1] not in self.valid_parallel_directives):
1033
+ error(pos, "No such directive: %s" % full_name)
1034
+
1035
+ return result
1036
+
1037
+ def visit_CImportStatNode(self, node):
1038
+ module_name = node.module_name
1039
+ if module_name == "cython.cimports":
1040
+ error(node.pos, "Cannot cimport the 'cython.cimports' package directly, only submodules.")
1041
+ if module_name.startswith("cython.cimports."):
1042
+ if node.as_name and node.as_name != 'cython':
1043
+ node.module_name = module_name[len("cython.cimports."):]
1044
+ return node
1045
+ error(node.pos,
1046
+ "Python cimports must use 'from cython.cimports... import ...'"
1047
+ " or 'import ... as ...', not just 'import ...'")
1048
+
1049
+ if module_name == "cython":
1050
+ self.cython_module_names.add(node.as_name or "cython")
1051
+ elif module_name.startswith("cython."):
1052
+ if module_name.startswith("cython.parallel."):
1053
+ error(node.pos, node.module_name + " is not a module")
1054
+ else:
1055
+ self._check_valid_cython_module(node.pos, module_name)
1056
+
1057
+ if module_name == "cython.parallel":
1058
+ if node.as_name and node.as_name != "cython":
1059
+ self.parallel_directives[node.as_name] = module_name
1060
+ else:
1061
+ self.cython_module_names.add("cython")
1062
+ self.parallel_directives[
1063
+ "cython.parallel"] = module_name
1064
+ elif node.as_name:
1065
+ self.directive_names[node.as_name] = module_name[7:]
1066
+ else:
1067
+ self.cython_module_names.add("cython")
1068
+ # if this cimport was a compiler directive, we don't
1069
+ # want to leave the cimport node sitting in the tree
1070
+ return None
1071
+ return node
1072
+
1073
+ def visit_FromCImportStatNode(self, node):
1074
+ module_name = node.module_name
1075
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1076
+ # only supported for convenience
1077
+ return self._create_cimport_from_import(
1078
+ node.pos, module_name, node.relative_level, node.imported_names)
1079
+ elif not node.relative_level and (
1080
+ module_name == "cython" or module_name.startswith("cython.")):
1081
+ self._check_valid_cython_module(node.pos, module_name)
1082
+ submodule = (module_name + ".")[7:]
1083
+ newimp = []
1084
+ for pos, name, as_name in node.imported_names:
1085
+ full_name = submodule + name
1086
+ qualified_name = "cython." + full_name
1087
+ if self.is_parallel_directive(qualified_name, node.pos):
1088
+ # from cython cimport parallel, or
1089
+ # from cython.parallel cimport parallel, prange, ...
1090
+ self.parallel_directives[as_name or name] = qualified_name
1091
+ elif self.is_cython_directive(full_name):
1092
+ self.directive_names[as_name or name] = full_name
1093
+ elif full_name in ['dataclasses', 'typing']:
1094
+ self.directive_names[as_name or name] = full_name
1095
+ # unlike many directives, still treat it as a regular module
1096
+ newimp.append((pos, name, as_name))
1097
+ else:
1098
+ newimp.append((pos, name, as_name))
1099
+
1100
+ if not newimp:
1101
+ return None
1102
+
1103
+ node.imported_names = newimp
1104
+ return node
1105
+
1106
+ def visit_FromImportStatNode(self, node):
1107
+ import_node = node.module
1108
+ module_name = import_node.module_name.value
1109
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1110
+ imported_names = []
1111
+ for name, name_node in node.items:
1112
+ imported_names.append(
1113
+ (name_node.pos, name, None if name == name_node.name else name_node.name))
1114
+ return self._create_cimport_from_import(
1115
+ node.pos, module_name, import_node.level, imported_names)
1116
+ elif module_name == "cython" or module_name.startswith("cython."):
1117
+ self._check_valid_cython_module(import_node.module_name.pos, module_name)
1118
+ submodule = (module_name + ".")[7:]
1119
+ newimp = []
1120
+ for name, name_node in node.items:
1121
+ full_name = submodule + name
1122
+ qualified_name = "cython." + full_name
1123
+ if self.is_parallel_directive(qualified_name, node.pos):
1124
+ self.parallel_directives[name_node.name] = qualified_name
1125
+ elif self.is_cython_directive(full_name):
1126
+ self.directive_names[name_node.name] = full_name
1127
+ else:
1128
+ newimp.append((name, name_node))
1129
+ if not newimp:
1130
+ return None
1131
+ node.items = newimp
1132
+ return node
1133
+
1134
+ def _create_cimport_from_import(self, node_pos, module_name, level, imported_names):
1135
+ if module_name == "cython.cimports" or module_name.startswith("cython.cimports."):
1136
+ module_name = EncodedString(module_name[len("cython.cimports."):]) # may be empty
1137
+
1138
+ if module_name:
1139
+ # from cython.cimports.a.b import x, y, z => from a.b cimport x, y, z
1140
+ return Nodes.FromCImportStatNode(
1141
+ node_pos, module_name=module_name,
1142
+ relative_level=level,
1143
+ imported_names=imported_names)
1144
+ else:
1145
+ # from cython.cimports import x, y, z => cimport x; cimport y; cimport z
1146
+ return [
1147
+ Nodes.CImportStatNode(
1148
+ pos,
1149
+ module_name=dotted_name,
1150
+ as_name=as_name,
1151
+ is_absolute=level == 0)
1152
+ for pos, dotted_name, as_name in imported_names
1153
+ ]
1154
+
1155
+ def visit_SingleAssignmentNode(self, node):
1156
+ if isinstance(node.rhs, ExprNodes.ImportNode):
1157
+ module_name = node.rhs.module_name.value
1158
+ if module_name != "cython" and not module_name.startswith("cython."):
1159
+ return node
1160
+
1161
+ node = Nodes.CImportStatNode(node.pos, module_name=module_name, as_name=node.lhs.name)
1162
+ node = self.visit_CImportStatNode(node)
1163
+ else:
1164
+ self.visitchildren(node)
1165
+
1166
+ return node
1167
+
1168
+ def visit_NameNode(self, node):
1169
+ if node.annotation:
1170
+ self.visitchild(node, 'annotation')
1171
+ if node.name in self.cython_module_names:
1172
+ node.is_cython_module = True
1173
+ else:
1174
+ directive = self.directive_names.get(node.name)
1175
+ if directive is not None:
1176
+ node.cython_attribute = directive
1177
+ if node.as_cython_attribute() == "compiled":
1178
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1179
+ # before they have a chance to cause compile-errors
1180
+ return node
1181
+
1182
+ def visit_AttributeNode(self, node):
1183
+ self.visitchildren(node)
1184
+ if node.as_cython_attribute() == "compiled":
1185
+ return ExprNodes.BoolNode(node.pos, value=True) # replace early so unused branches can be dropped
1186
+ # before they have a chance to cause compile-errors
1187
+ return node
1188
+
1189
+ def visit_AnnotationNode(self, node):
1190
+ # for most transforms annotations are left unvisited (because they're unevaluated)
1191
+ # however, it is important to pick up compiler directives from them
1192
+ if node.expr:
1193
+ self.visit(node.expr)
1194
+ return node
1195
+
1196
+ def visit_NewExprNode(self, node):
1197
+ self.visitchild(node, 'cppclass')
1198
+ self.visitchildren(node)
1199
+ return node
1200
+
1201
+ def try_to_parse_directives(self, node):
1202
+ # If node is the contents of an directive (in a with statement or
1203
+ # decorator), returns a list of (directivename, value) pairs.
1204
+ # Otherwise, returns None
1205
+ if isinstance(node, ExprNodes.CallNode):
1206
+ self.visitchild(node, 'function')
1207
+ optname = node.function.as_cython_attribute()
1208
+ if optname:
1209
+ directivetype = Options.directive_types.get(optname)
1210
+ if directivetype:
1211
+ args, kwds = node.explicit_args_kwds()
1212
+ directives = []
1213
+ key_value_pairs = []
1214
+ if kwds is not None and directivetype is not dict:
1215
+ for keyvalue in kwds.key_value_pairs:
1216
+ key, value = keyvalue
1217
+ sub_optname = "%s.%s" % (optname, key.value)
1218
+ if Options.directive_types.get(sub_optname):
1219
+ directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
1220
+ else:
1221
+ key_value_pairs.append(keyvalue)
1222
+ if not key_value_pairs:
1223
+ kwds = None
1224
+ else:
1225
+ kwds.key_value_pairs = key_value_pairs
1226
+ if directives and not kwds and not args:
1227
+ return directives
1228
+ directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
1229
+ return directives
1230
+ elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
1231
+ self.visit(node)
1232
+ optname = node.as_cython_attribute()
1233
+ if optname:
1234
+ directivetype = Options.directive_types.get(optname)
1235
+ if directivetype is bool:
1236
+ arg = ExprNodes.BoolNode(node.pos, value=True)
1237
+ return [self.try_to_parse_directive(optname, [arg], None, node.pos)]
1238
+ elif directivetype is None or directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1239
+ return [(optname, None)]
1240
+ else:
1241
+ raise PostParseError(
1242
+ node.pos, "The '%s' directive should be used as a function call." % optname)
1243
+ return None
1244
+
1245
+ def try_to_parse_directive(self, optname, args, kwds, pos):
1246
+ if optname == 'np_pythran' and not self.context.cpp:
1247
+ raise PostParseError(pos, 'The %s directive can only be used in C++ mode.' % optname)
1248
+ elif optname == 'exceptval':
1249
+ # default: exceptval(None, check=True)
1250
+ arg_error = len(args) > 1
1251
+ check = True
1252
+ if kwds and kwds.key_value_pairs:
1253
+ kw = kwds.key_value_pairs[0]
1254
+ if (len(kwds.key_value_pairs) == 1 and
1255
+ kw.key.is_string_literal and kw.key.value == 'check' and
1256
+ isinstance(kw.value, ExprNodes.BoolNode)):
1257
+ check = kw.value.value
1258
+ else:
1259
+ arg_error = True
1260
+ if arg_error:
1261
+ raise PostParseError(
1262
+ pos, 'The exceptval directive takes 0 or 1 positional arguments and the boolean keyword "check"')
1263
+ return ('exceptval', (args[0] if args else None, check))
1264
+
1265
+ directivetype = Options.directive_types.get(optname)
1266
+ if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
1267
+ return optname, Options.get_directive_defaults()[optname]
1268
+ elif directivetype is bool:
1269
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1270
+ raise PostParseError(pos,
1271
+ 'The %s directive takes one compile-time boolean argument' % optname)
1272
+ return (optname, args[0].value)
1273
+ elif directivetype is int:
1274
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
1275
+ raise PostParseError(pos,
1276
+ 'The %s directive takes one compile-time integer argument' % optname)
1277
+ return (optname, int(args[0].value))
1278
+ elif directivetype is str:
1279
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1280
+ raise PostParseError(pos,
1281
+ 'The %s directive takes one compile-time string argument' % optname)
1282
+ return (optname, str(args[0].value))
1283
+ elif directivetype is type:
1284
+ if kwds is not None or len(args) != 1:
1285
+ raise PostParseError(pos,
1286
+ 'The %s directive takes one type argument' % optname)
1287
+ return (optname, args[0])
1288
+ elif directivetype is dict:
1289
+ if len(args) != 0:
1290
+ raise PostParseError(pos,
1291
+ 'The %s directive takes no prepositional arguments' % optname)
1292
+ return optname, kwds.as_python_dict()
1293
+ elif directivetype is list:
1294
+ if kwds and len(kwds.key_value_pairs) != 0:
1295
+ raise PostParseError(pos,
1296
+ 'The %s directive takes no keyword arguments' % optname)
1297
+ return optname, [ str(arg.value) for arg in args ]
1298
+ elif callable(directivetype):
1299
+ if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.UnicodeNode):
1300
+ raise PostParseError(pos,
1301
+ 'The %s directive takes one compile-time string argument' % optname)
1302
+ return (optname, directivetype(optname, str(args[0].value)))
1303
+ elif directivetype is Options.DEFER_ANALYSIS_OF_ARGUMENTS:
1304
+ # signal to pass things on without processing
1305
+ return (optname, (args, kwds.as_python_dict() if kwds else {}))
1306
+ else:
1307
+ assert False
1308
+
1309
+ def visit_with_directives(self, node, directives, contents_directives):
1310
+ # contents_directives may be None
1311
+ if not directives:
1312
+ assert not contents_directives
1313
+ return self.visit_Node(node)
1314
+
1315
+ old_directives = self.directives
1316
+ new_directives = Options.copy_inherited_directives(old_directives, **directives)
1317
+ if contents_directives is not None:
1318
+ new_contents_directives = Options.copy_inherited_directives(
1319
+ old_directives, **contents_directives)
1320
+ else:
1321
+ new_contents_directives = new_directives
1322
+
1323
+ if new_directives == old_directives:
1324
+ return self.visit_Node(node)
1325
+
1326
+ self.directives = new_directives
1327
+ if (contents_directives is not None and
1328
+ new_contents_directives != new_directives):
1329
+ # we need to wrap the node body in a compiler directives node
1330
+ node.body = Nodes.StatListNode(
1331
+ node.body.pos,
1332
+ stats=[
1333
+ Nodes.CompilerDirectivesNode(
1334
+ node.body.pos,
1335
+ directives=new_contents_directives,
1336
+ body=node.body)
1337
+ ]
1338
+ )
1339
+ retbody = self.visit_Node(node)
1340
+ self.directives = old_directives
1341
+
1342
+ if isinstance(retbody, Nodes.CompilerDirectivesNode):
1343
+ new_directives.update(retbody.directives)
1344
+ retbody = retbody.body
1345
+ if not isinstance(retbody, Nodes.StatListNode):
1346
+ retbody = Nodes.StatListNode(node.pos, stats=[retbody])
1347
+ return Nodes.CompilerDirectivesNode(
1348
+ retbody.pos, body=retbody, directives=new_directives, is_terminator=retbody.is_terminator)
1349
+
1350
+ # Handle decorators
1351
+ def visit_FuncDefNode(self, node):
1352
+ directives, contents_directives = self._extract_directives(node, 'function')
1353
+ return self.visit_with_directives(node, directives, contents_directives)
1354
+
1355
+ def visit_CVarDefNode(self, node):
1356
+ directives, _ = self._extract_directives(node, 'function')
1357
+ for name, value in directives.items():
1358
+ if name == 'locals':
1359
+ node.directive_locals = value
1360
+ elif name not in ('final', 'staticmethod'):
1361
+ self.context.nonfatal_error(PostParseError(
1362
+ node.pos,
1363
+ "Cdef functions can only take cython.locals(), "
1364
+ "staticmethod, or final decorators, got %s." % name))
1365
+ return self.visit_with_directives(node, directives, contents_directives=None)
1366
+
1367
+ def visit_CClassDefNode(self, node):
1368
+ directives, contents_directives = self._extract_directives(node, 'cclass')
1369
+ return self.visit_with_directives(node, directives, contents_directives)
1370
+
1371
+ def visit_CppClassNode(self, node):
1372
+ directives, contents_directives = self._extract_directives(node, 'cppclass')
1373
+ return self.visit_with_directives(node, directives, contents_directives)
1374
+
1375
+ def visit_PyClassDefNode(self, node):
1376
+ directives, contents_directives = self._extract_directives(node, 'class')
1377
+ return self.visit_with_directives(node, directives, contents_directives)
1378
+
1379
+ def _extract_directives(self, node, scope_name):
1380
+ """
1381
+ Returns two dicts - directives applied to this function/class
1382
+ and directives applied to its contents. They aren't always the
1383
+ same (since e.g. cfunc should not be applied to inner functions)
1384
+ """
1385
+ if not node.decorators:
1386
+ return {}, {}
1387
+ # Split the decorators into two lists -- real decorators and directives
1388
+ directives = []
1389
+ realdecs = []
1390
+ both = []
1391
+ current_opt_dict = dict(self.directives)
1392
+ missing = object()
1393
+ # Decorators coming first take precedence.
1394
+ for dec in node.decorators[::-1]:
1395
+ new_directives = self.try_to_parse_directives(dec.decorator)
1396
+ if new_directives is not None:
1397
+ for directive in new_directives:
1398
+ if self.check_directive_scope(node.pos, directive[0], scope_name):
1399
+ name, value = directive
1400
+ if name in ('nogil', 'with_gil'):
1401
+ if value is None:
1402
+ value = True
1403
+ else:
1404
+ args, kwds = value
1405
+ if kwds or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
1406
+ raise PostParseError(dec.pos, 'The %s directive takes one compile-time boolean argument' % name)
1407
+ value = args[0].value
1408
+ directive = (name, value)
1409
+ if current_opt_dict.get(name, missing) != value:
1410
+ if name == 'cfunc' and 'ufunc' in current_opt_dict:
1411
+ error(dec.pos, "Cannot apply @cfunc to @ufunc, please reverse the decorators.")
1412
+ directives.append(directive)
1413
+ current_opt_dict[name] = value
1414
+ elif name not in ['profile', 'linetrace']:
1415
+ # Exclude some decorators that people may leave around, but warn about useless ones.
1416
+ warning(dec.pos, "Directive does not change previous value (%s%s)" % (
1417
+ name, '=%r' % value if value is not None else ''))
1418
+ if directive[0] == 'staticmethod':
1419
+ both.append(dec)
1420
+ # Adapt scope type based on decorators that change it.
1421
+ if directive[0] == 'cclass' and scope_name == 'class':
1422
+ scope_name = 'cclass'
1423
+ else:
1424
+ realdecs.append(dec)
1425
+ node.decorators = realdecs[::-1] + both[::-1]
1426
+ # merge or override repeated directives
1427
+ optdict = {}
1428
+ contents_optdict = {}
1429
+ for name, value in directives:
1430
+ if name in optdict:
1431
+ old_value = optdict[name]
1432
+ # keywords and arg lists can be merged, everything
1433
+ # else overrides completely
1434
+ if isinstance(old_value, dict):
1435
+ old_value.update(value)
1436
+ elif isinstance(old_value, list):
1437
+ old_value.extend(value)
1438
+ else:
1439
+ if name == "collection_type" and value != optdict[name]:
1440
+ error(node.pos, "Multiple values of collection_type are not supported")
1441
+ optdict[name] = value
1442
+ else:
1443
+ optdict[name] = value
1444
+ if name not in Options.immediate_decorator_directives:
1445
+ contents_optdict[name] = value
1446
+ return optdict, contents_optdict
1447
+
1448
+ # Handle with-statements
1449
+ def visit_WithStatNode(self, node):
1450
+ directive_dict = {}
1451
+ for directive in self.try_to_parse_directives(node.manager) or []:
1452
+ if directive is None:
1453
+ continue
1454
+ if node.target is not None:
1455
+ self.context.nonfatal_error(
1456
+ PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
1457
+ continue
1458
+ name, value = directive
1459
+ if name in ('nogil', 'gil'):
1460
+ # special case: in pure mode, "with nogil" spells "with cython.nogil"
1461
+ return self._transform_with_gil(node, name)
1462
+ elif name == "critical_section":
1463
+ args, kwds = value
1464
+ return self._transform_critical_section(node, args, kwds)
1465
+ elif self.check_directive_scope(node.pos, name, 'with statement'):
1466
+ directive_dict[name] = value
1467
+ if directive_dict:
1468
+ return self.visit_with_directives(node.body, directive_dict, contents_directives=None)
1469
+ return self.visit_Node(node)
1470
+
1471
+ def _transform_with_gil(self, node, state):
1472
+ assert state in ('gil', 'nogil')
1473
+ manager = node.manager
1474
+ condition = None
1475
+ if isinstance(manager, ExprNodes.SimpleCallNode) and manager.args:
1476
+ if len(manager.args) > 1:
1477
+ self.context.nonfatal_error(
1478
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % state))
1479
+ condition = manager.args[0]
1480
+ elif isinstance(manager, ExprNodes.GeneralCallNode):
1481
+ self.context.nonfatal_error(
1482
+ PostParseError(node.pos, "Compiler directive %s accepts one positional argument." % state))
1483
+ node = Nodes.GILStatNode(node.pos, state=state, body=node.body, condition=condition)
1484
+ return self.visit_Node(node)
1485
+
1486
+ def _transform_critical_section(self, node, args, kwds):
1487
+ if len(args) < 1 or len(args) > 2 or kwds:
1488
+ self.context.nonfatal_error(
1489
+ PostParseError(node.pos, "critical_section directive accepts one or two positional arguments")
1490
+ )
1491
+ node = Nodes.CriticalSectionStatNode(
1492
+ node.pos, args=args, body=node.body
1493
+ )
1494
+ return self.visit_Node(node)
1495
+
1496
+
1497
+ class ParallelRangeTransform(CythonTransform, SkipDeclarations):
1498
+ """
1499
+ Transform cython.parallel stuff. The parallel_directives come from the
1500
+ module node, set there by InterpretCompilerDirectives.
1501
+
1502
+ x = cython.parallel.threadavailable() -> ParallelThreadAvailableNode
1503
+ with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
1504
+ print cython.parallel.threadid() -> ParallelThreadIdNode
1505
+ for i in cython.parallel.prange(...): -> ParallelRangeNode
1506
+ ...
1507
+ """
1508
+
1509
+ # a list of names, maps 'cython.parallel.prange' in the code to
1510
+ # ['cython', 'parallel', 'prange']
1511
+ parallel_directive = None
1512
+
1513
+ # Indicates whether a namenode in an expression is the cython module
1514
+ namenode_is_cython_module = False
1515
+
1516
+ # Keep track of whether we are the context manager of a 'with' statement
1517
+ in_context_manager_section = False
1518
+
1519
+ # One of 'prange' or 'with parallel'. This is used to disallow closely
1520
+ # nested 'with parallel:' blocks
1521
+ state = None
1522
+
1523
+ directive_to_node = {
1524
+ "cython.parallel.parallel": Nodes.ParallelWithBlockNode,
1525
+ # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
1526
+ "cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
1527
+ "cython.parallel.prange": Nodes.ParallelRangeNode,
1528
+ }
1529
+
1530
+ def node_is_parallel_directive(self, node):
1531
+ return node.name in self.parallel_directives or node.is_cython_module
1532
+
1533
+ def get_directive_class_node(self, node):
1534
+ """
1535
+ Figure out which parallel directive was used and return the associated
1536
+ Node class.
1537
+
1538
+ E.g. for a cython.parallel.prange() call we return ParallelRangeNode
1539
+ """
1540
+ if self.namenode_is_cython_module:
1541
+ directive = '.'.join(self.parallel_directive)
1542
+ else:
1543
+ directive = self.parallel_directives[self.parallel_directive[0]]
1544
+ directive = '%s.%s' % (directive,
1545
+ '.'.join(self.parallel_directive[1:]))
1546
+ directive = directive.rstrip('.')
1547
+
1548
+ cls = self.directive_to_node.get(directive)
1549
+ if cls is None and not (self.namenode_is_cython_module and
1550
+ self.parallel_directive[0] != 'parallel'):
1551
+ error(node.pos, "Invalid directive: %s" % directive)
1552
+
1553
+ self.namenode_is_cython_module = False
1554
+ self.parallel_directive = None
1555
+
1556
+ return cls
1557
+
1558
+ def visit_ModuleNode(self, node):
1559
+ """
1560
+ If any parallel directives were imported, copy them over and visit
1561
+ the AST
1562
+ """
1563
+ if node.parallel_directives:
1564
+ self.parallel_directives = node.parallel_directives
1565
+ return self.visit_Node(node)
1566
+
1567
+ # No parallel directives were imported, so they can't be used :)
1568
+ return node
1569
+
1570
+ def visit_NameNode(self, node):
1571
+ if self.node_is_parallel_directive(node):
1572
+ self.parallel_directive = [node.name]
1573
+ self.namenode_is_cython_module = node.is_cython_module
1574
+ return node
1575
+
1576
+ def visit_AttributeNode(self, node):
1577
+ self.visitchildren(node)
1578
+ if self.parallel_directive:
1579
+ self.parallel_directive.append(node.attribute)
1580
+ return node
1581
+
1582
+ def visit_CallNode(self, node):
1583
+ self.visitchild(node, 'function')
1584
+ if not self.parallel_directive:
1585
+ self.visitchildren(node, exclude=('function',))
1586
+ return node
1587
+
1588
+ # We are a parallel directive, replace this node with the
1589
+ # corresponding ParallelSomethingSomething node
1590
+
1591
+ if isinstance(node, ExprNodes.GeneralCallNode):
1592
+ args = node.positional_args.args
1593
+ kwargs = node.keyword_args
1594
+ else:
1595
+ args = node.args
1596
+ kwargs = {}
1597
+
1598
+ parallel_directive_class = self.get_directive_class_node(node)
1599
+ if parallel_directive_class:
1600
+ # Note: in case of a parallel() the body is set by
1601
+ # visit_WithStatNode
1602
+ node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)
1603
+
1604
+ return node
1605
+
1606
+ def visit_WithStatNode(self, node):
1607
+ "Rewrite with cython.parallel.parallel() blocks"
1608
+ newnode = self.visit(node.manager)
1609
+
1610
+ if isinstance(newnode, Nodes.ParallelWithBlockNode):
1611
+ if self.state == 'parallel with':
1612
+ error(node.manager.pos,
1613
+ "Nested parallel with blocks are disallowed")
1614
+
1615
+ self.state = 'parallel with'
1616
+ body = self.visitchild(node, 'body')
1617
+ self.state = None
1618
+
1619
+ newnode.body = body
1620
+ return newnode
1621
+ elif self.parallel_directive:
1622
+ parallel_directive_class = self.get_directive_class_node(node)
1623
+
1624
+ if not parallel_directive_class:
1625
+ # There was an error, stop here and now
1626
+ return None
1627
+
1628
+ if parallel_directive_class is Nodes.ParallelWithBlockNode:
1629
+ error(node.pos, "The parallel directive must be called")
1630
+ return None
1631
+
1632
+ self.visitchild(node, 'body')
1633
+ return node
1634
+
1635
+ def visit_ForInStatNode(self, node):
1636
+ "Rewrite 'for i in cython.parallel.prange(...):'"
1637
+ self.visitchild(node, 'iterator')
1638
+ self.visitchild(node, 'target')
1639
+
1640
+ in_prange = isinstance(node.iterator.sequence,
1641
+ Nodes.ParallelRangeNode)
1642
+ previous_state = self.state
1643
+
1644
+ if in_prange:
1645
+ # This will replace the entire ForInStatNode, so copy the
1646
+ # attributes
1647
+ parallel_range_node = node.iterator.sequence
1648
+
1649
+ parallel_range_node.target = node.target
1650
+ parallel_range_node.body = node.body
1651
+ parallel_range_node.else_clause = node.else_clause
1652
+
1653
+ node = parallel_range_node
1654
+
1655
+ if not isinstance(node.target, ExprNodes.NameNode):
1656
+ error(node.target.pos,
1657
+ "Can only iterate over an iteration variable")
1658
+
1659
+ self.state = 'prange'
1660
+
1661
+ self.visitchild(node, 'body')
1662
+ self.state = previous_state
1663
+ self.visitchild(node, 'else_clause')
1664
+ return node
1665
+
1666
+ def visit(self, node):
1667
+ "Visit a node that may be None"
1668
+ if node is not None:
1669
+ return super().visit(node)
1670
+
1671
+
1672
+ class WithTransform(VisitorTransform, SkipDeclarations):
1673
+ def visit_WithStatNode(self, node):
1674
+ self.visitchildren(node, ['body'])
1675
+ pos = node.pos
1676
+ is_async = node.is_async
1677
+ body, target, manager = node.body, node.target, node.manager
1678
+ manager = node.manager = ExprNodes.ProxyNode(manager)
1679
+ node.enter_call = ExprNodes.SimpleCallNode(
1680
+ pos, function=ExprNodes.AttributeNode(
1681
+ pos, obj=ExprNodes.CloneNode(manager),
1682
+ attribute=EncodedString('__aenter__' if is_async else '__enter__'),
1683
+ is_special_lookup=True),
1684
+ args=[],
1685
+ is_temp=True)
1686
+
1687
+ if is_async:
1688
+ node.enter_call = ExprNodes.AwaitExprNode(pos, arg=node.enter_call)
1689
+
1690
+ if target is not None:
1691
+ body = Nodes.StatListNode(
1692
+ pos, stats=[
1693
+ Nodes.WithTargetAssignmentStatNode(
1694
+ pos, lhs=target, with_node=node),
1695
+ body])
1696
+
1697
+ excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
1698
+ ExprNodes.ExcValueNode(pos) for _ in range(3)])
1699
+ except_clause = Nodes.ExceptClauseNode(
1700
+ pos, body=Nodes.IfStatNode(
1701
+ pos, if_clauses=[
1702
+ Nodes.IfClauseNode(
1703
+ pos, condition=ExprNodes.NotNode(
1704
+ pos, operand=ExprNodes.WithExitCallNode(
1705
+ pos, with_stat=node,
1706
+ test_if_run=False,
1707
+ args=excinfo_target,
1708
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1709
+ body=Nodes.ReraiseStatNode(pos),
1710
+ ),
1711
+ ],
1712
+ else_clause=None),
1713
+ pattern=None,
1714
+ target=None,
1715
+ excinfo_target=excinfo_target,
1716
+ )
1717
+
1718
+ node.body = Nodes.TryFinallyStatNode(
1719
+ pos, body=Nodes.TryExceptStatNode(
1720
+ pos, body=body,
1721
+ except_clauses=[except_clause],
1722
+ else_clause=None,
1723
+ ),
1724
+ finally_clause=Nodes.ExprStatNode(
1725
+ pos, expr=ExprNodes.WithExitCallNode(
1726
+ pos, with_stat=node,
1727
+ test_if_run=True,
1728
+ args=ExprNodes.TupleNode(
1729
+ pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]),
1730
+ await_expr=ExprNodes.AwaitExprNode(pos, arg=None) if is_async else None)),
1731
+ handle_error_case=False,
1732
+ )
1733
+ return node
1734
+
1735
+ def visit_ExprNode(self, node):
1736
+ # With statements are never inside expressions.
1737
+ return node
1738
+
1739
+ visit_Node = VisitorTransform.recurse_to_children
1740
+
1741
+
1742
+ class _GeneratorExpressionArgumentsMarker(TreeVisitor, SkipDeclarations):
1743
+ # called from "MarkClosureVisitor"
1744
+ def __init__(self, gen_expr):
1745
+ super().__init__()
1746
+ self.gen_expr = gen_expr
1747
+
1748
+ def visit_ExprNode(self, node):
1749
+ if not node.is_literal:
1750
+ # Don't bother tagging literal nodes
1751
+ assert (not node.generator_arg_tag) # nobody has tagged this first
1752
+ node.generator_arg_tag = self.gen_expr
1753
+ self.visitchildren(node)
1754
+
1755
+ def visit_Node(self, node):
1756
+ # We're only interested in the expressions that make up the iterator sequence,
1757
+ # so don't go beyond ExprNodes (e.g. into ForFromStatNode).
1758
+ return
1759
+
1760
+ def visit_GeneratorExpressionNode(self, node):
1761
+ node.generator_arg_tag = self.gen_expr
1762
+ # don't visit children, can't handle overlapping tags
1763
+ # (and assume generator expressions don't end up optimized out in a way
1764
+ # that would require overlapping tags)
1765
+
1766
+
1767
+ class _HandleGeneratorArguments(VisitorTransform, SkipDeclarations):
1768
+ # used from within CreateClosureClasses
1769
+
1770
+ def __call__(self, node):
1771
+ from . import Visitor
1772
+ assert isinstance(node, ExprNodes.GeneratorExpressionNode)
1773
+ self.gen_node = node
1774
+
1775
+ self.args = list(node.def_node.args)
1776
+ self.call_parameters = list(node.call_parameters)
1777
+ self.tag_count = 0
1778
+ self.substitutions = {}
1779
+
1780
+ self.visitchildren(node)
1781
+
1782
+ for k, v in self.substitutions.items():
1783
+ # doing another search for replacements here (at the end) allows us to sweep up
1784
+ # CloneNodes too (which are often generated by the optimizer)
1785
+ # (it could arguably be done more efficiently with a single traversal though)
1786
+ Visitor.recursively_replace_node(node, k, v)
1787
+
1788
+ node.def_node.args = self.args
1789
+ node.call_parameters = self.call_parameters
1790
+ return node
1791
+
1792
+ def visit_GeneratorExpressionNode(self, node):
1793
+ # a generator can also be substituted itself, so handle that case
1794
+ new_node = self._handle_ExprNode(node, do_visit_children=False)
1795
+ # However do not traverse into it. A new _HandleGeneratorArguments visitor will be used
1796
+ # elsewhere to do that.
1797
+ return node
1798
+
1799
+ def _handle_ExprNode(self, node, do_visit_children):
1800
+ if (node.generator_arg_tag is not None and self.gen_node is not None and
1801
+ self.gen_node == node.generator_arg_tag):
1802
+ pos = node.pos
1803
+ # The reason for using ".x" as the name is that this is how CPython
1804
+ # tracks internal variables in loops (e.g.
1805
+ # { locals() for v in range(10) }
1806
+ # will produce "v" and ".0"). We don't replicate this behaviour completely
1807
+ # but use it as a starting point
1808
+ name_source = self.tag_count
1809
+ self.tag_count += 1
1810
+ name = EncodedString(".{}".format(name_source))
1811
+ def_node = self.gen_node.def_node
1812
+ if not def_node.local_scope.lookup_here(name):
1813
+ from . import Symtab
1814
+ cname = EncodedString(Naming.genexpr_arg_prefix + Symtab.punycodify_name(str(name_source)))
1815
+ name_decl = Nodes.CNameDeclaratorNode(pos=pos, name=name)
1816
+ type = node.type
1817
+
1818
+ # strip away cv types - they shouldn't be applied to the
1819
+ # function argument or to the closure struct.
1820
+ # It isn't obvious whether the right thing to do would be to capture by reference or by
1821
+ # value (C++ itself doesn't know either for lambda functions and forces a choice).
1822
+ # However, capture by reference involves converting to FakeReference which would require
1823
+ # re-analysing AttributeNodes. Therefore I've picked capture-by-value out of convenience
1824
+ # TODO - could probably be optimized by making the arg a reference but the closure not
1825
+ # (see https://github.com/cython/cython/issues/2468)
1826
+ type = PyrexTypes.remove_cv_ref(type, remove_fakeref=False)
1827
+
1828
+ name_decl.type = type
1829
+ new_arg = Nodes.CArgDeclNode(pos=pos, declarator=name_decl,
1830
+ base_type=None, default=None, annotation=None)
1831
+ new_arg.name = name_decl.name
1832
+ new_arg.type = type
1833
+
1834
+ self.args.append(new_arg)
1835
+ node.generator_arg_tag = None # avoid the possibility of this being caught again
1836
+ self.call_parameters.append(node)
1837
+ new_arg.entry = def_node.declare_argument(def_node.local_scope, new_arg)
1838
+ new_arg.entry.cname = cname
1839
+ new_arg.entry.in_closure = True
1840
+
1841
+ if do_visit_children:
1842
+ # now visit the Nodes's children (but remove self.gen_node to not to further
1843
+ # argument substitution)
1844
+ gen_node, self.gen_node = self.gen_node, None
1845
+ self.visitchildren(node)
1846
+ self.gen_node = gen_node
1847
+
1848
+ # replace the node inside the generator with a looked-up name
1849
+ # (initialized_check can safely be False because the source variable will be checked
1850
+ # before it is captured if the check is required)
1851
+ name_node = ExprNodes.NameNode(pos, name=name, initialized_check=False)
1852
+ name_node.entry = self.gen_node.def_node.gbody.local_scope.lookup(name_node.name)
1853
+ name_node.type = name_node.entry.type
1854
+ self.substitutions[node] = name_node
1855
+ return name_node
1856
+ if do_visit_children:
1857
+ self.visitchildren(node)
1858
+ return node
1859
+
1860
+ def visit_ExprNode(self, node):
1861
+ return self._handle_ExprNode(node, True)
1862
+
1863
+ visit_Node = VisitorTransform.recurse_to_children
1864
+
1865
+
1866
+ class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
1867
+ """
1868
+ Transforms method decorators in cdef classes into nested calls or properties.
1869
+
1870
+ Python-style decorator properties are transformed into a PropertyNode
1871
+ with up to the three getter, setter and deleter DefNodes.
1872
+ The functional style isn't supported yet.
1873
+ """
1874
+ _properties = None
1875
+
1876
+ _map_property_attribute = {
1877
+ 'getter': EncodedString('__get__'),
1878
+ 'setter': EncodedString('__set__'),
1879
+ 'deleter': EncodedString('__del__'),
1880
+ }.get
1881
+
1882
+ def visit_CClassDefNode(self, node):
1883
+ if self._properties is None:
1884
+ self._properties = []
1885
+ self._properties.append({})
1886
+ node = super().visit_CClassDefNode(node)
1887
+ self._properties.pop()
1888
+ return node
1889
+
1890
+ def visit_PropertyNode(self, node):
1891
+ # Low-level warning for other code until we can convert all our uses over.
1892
+ level = 2 if isinstance(node.pos[0], str) else 0
1893
+ warning(node.pos, "'property %s:' syntax is deprecated, use '@property'" % node.name, level)
1894
+ return node
1895
+
1896
+ def visit_CFuncDefNode(self, node):
1897
+ node = self.visit_FuncDefNode(node)
1898
+ if not node.decorators:
1899
+ return node
1900
+ elif self.scope_type != 'cclass' or self.scope_node.visibility != "extern":
1901
+ # at the moment cdef functions are very restricted in what decorators they can take
1902
+ # so it's simple to test for the small number of allowed decorators....
1903
+ if not (len(node.decorators) == 1 and node.decorators[0].decorator.is_name and
1904
+ node.decorators[0].decorator.name == "staticmethod"):
1905
+ error(node.decorators[0].pos, "Cdef functions cannot take arbitrary decorators.")
1906
+ return node
1907
+
1908
+ ret_node = node
1909
+ decorator_node = self._find_property_decorator(node)
1910
+ if decorator_node and (name := node.declared_name()):
1911
+ if decorator_node.decorator.is_name:
1912
+ ret_node = self._add_property(node, name, decorator_node)
1913
+ else:
1914
+ handler_name = self._map_property_attribute(decorator_node.decorator.attribute)
1915
+ if handler_name:
1916
+ if handler_name == "__del__":
1917
+ error(decorator_node.pos,
1918
+ "Cannot have deleter for C property")
1919
+ ret_node = None
1920
+ elif decorator_node.decorator.obj.name != name:
1921
+ # CPython does not generate an error or warning, but not something useful either.
1922
+ error(decorator_node.pos,
1923
+ "Mismatching C property names, expected '%s', got '%s'" % (
1924
+ decorator_node.decorator.obj.name, name))
1925
+ ret_node = None
1926
+ else:
1927
+ ret_node = self._add_to_property(node, handler_name, decorator_node)
1928
+
1929
+ if node.decorators:
1930
+ return self._reject_decorated_property(node, node.decorators[0])
1931
+ return ret_node
1932
+
1933
+ def visit_DefNode(self, node):
1934
+ scope_type = self.scope_type
1935
+ node = self.visit_FuncDefNode(node)
1936
+ if scope_type != 'cclass' or not node.decorators:
1937
+ return node
1938
+
1939
+ # transform @property decorators
1940
+ decorator_node = self._find_property_decorator(node)
1941
+ if decorator_node is not None:
1942
+ decorator = decorator_node.decorator
1943
+ if decorator.is_name:
1944
+ return self._add_property(node, node.name, decorator_node)
1945
+ else:
1946
+ handler_name = self._map_property_attribute(decorator.attribute)
1947
+ if handler_name:
1948
+ if decorator.obj.name != node.name:
1949
+ # CPython does not generate an error or warning, but not something useful either.
1950
+ error(decorator_node.pos,
1951
+ "Mismatching property names, expected '%s', got '%s'" % (
1952
+ decorator.obj.name, node.name))
1953
+ elif len(node.decorators) > 1:
1954
+ return self._reject_decorated_property(node, decorator_node)
1955
+ else:
1956
+ return self._add_to_property(node, handler_name, decorator_node)
1957
+
1958
+ # we clear node.decorators, so we need to set the
1959
+ # is_staticmethod/is_classmethod attributes now
1960
+ for decorator in node.decorators:
1961
+ func = decorator.decorator
1962
+ if func.is_name:
1963
+ node.is_classmethod |= func.name == 'classmethod'
1964
+ node.is_staticmethod |= func.name == 'staticmethod'
1965
+
1966
+ # transform normal decorators
1967
+ decs = node.decorators
1968
+ node.decorators = None
1969
+ return self.chain_decorators(node, decs, node.name)
1970
+
1971
+ def _find_property_decorator(self, node):
1972
+ properties = self._properties[-1]
1973
+ for decorator_node in node.decorators[::-1]:
1974
+ decorator = decorator_node.decorator
1975
+ if decorator.is_name and decorator.name == 'property':
1976
+ # @property
1977
+ return decorator_node
1978
+ elif decorator.is_attribute and decorator.obj.name in properties:
1979
+ # @prop.setter etc.
1980
+ return decorator_node
1981
+ return None
1982
+
1983
+ @staticmethod
1984
+ def _reject_decorated_property(node, decorator_node):
1985
+ # restrict transformation to outermost decorator as wrapped properties will probably not work
1986
+ for deco in node.decorators:
1987
+ if deco != decorator_node:
1988
+ error(deco.pos, "Property methods with additional decorators are not supported")
1989
+ return node
1990
+
1991
+ def _get_property_function_name(self, node):
1992
+ if isinstance(node, Nodes.CFuncDefNode):
1993
+ return node.declared_name()
1994
+ return node.name
1995
+
1996
+ def _rename_property_function(self, node, name):
1997
+ if isinstance(node, Nodes.CFuncDefNode):
1998
+ node.declarator.set_declared_name(name)
1999
+ else:
2000
+ node.name = name
2001
+
2002
+ def _add_property(self, node, name, decorator_node):
2003
+ if len(node.decorators) > 1:
2004
+ return self._reject_decorated_property(node, decorator_node)
2005
+ node.decorators.remove(decorator_node)
2006
+ properties = self._properties[-1]
2007
+ is_cproperty = isinstance(node, Nodes.CFuncDefNode)
2008
+ body = Nodes.StatListNode(node.pos, stats=[node])
2009
+ node_type = Nodes.PropertyNode
2010
+ if is_cproperty:
2011
+ if 'inline' not in node.modifiers:
2012
+ error(node.pos, "C property method must be declared 'inline'")
2013
+ node_type = Nodes.CPropertyNode
2014
+ if name in properties:
2015
+ prop = properties[name]
2016
+ if prop.is_cproperty:
2017
+ error(node.pos, "C property redeclared")
2018
+ else:
2019
+ self._rename_property_function(node, EncodedString("__get__"))
2020
+ prop.pos = node.pos
2021
+ prop.doc = node.doc
2022
+ prop.body.stats = [node]
2023
+ return None
2024
+ else:
2025
+ self._rename_property_function(node, EncodedString("__get__"))
2026
+ prop = node_type(
2027
+ node.pos, name=name, doc=node.doc, body=body)
2028
+ properties[name] = prop
2029
+ return prop
2030
+
2031
+ def _add_to_property(self, node, name, decorator):
2032
+ properties = self._properties[-1]
2033
+ prop = properties[self._get_property_function_name(node)]
2034
+ self._rename_property_function(node, name)
2035
+ if isinstance(node, Nodes.CFuncDefNode):
2036
+ if 'inline' not in node.modifiers:
2037
+ error(node.pos, "C property method must be declared 'inline'")
2038
+ node.decorators.remove(decorator)
2039
+ stats = prop.body.stats
2040
+ for i, stat in enumerate(stats):
2041
+ if self._get_property_function_name(stat) == name:
2042
+ if prop.is_cproperty:
2043
+ error(node.pos, "C property redeclared")
2044
+ return None
2045
+ stats[i] = node
2046
+ break
2047
+ else:
2048
+ stats.append(node)
2049
+ return None
2050
+
2051
+ @staticmethod
2052
+ def chain_decorators(node, decorators, name):
2053
+ """
2054
+ Decorators are applied directly in DefNode and PyClassDefNode to avoid
2055
+ reassignments to the function/class name - except for cdef class methods.
2056
+ For those, the reassignment is required as methods are originally
2057
+ defined in the PyMethodDef struct.
2058
+
2059
+ The IndirectionNode allows DefNode to override the decorator.
2060
+ """
2061
+ decorator_result = ExprNodes.NameNode(node.pos, name=name)
2062
+ for decorator in decorators[::-1]:
2063
+ decorator_result = ExprNodes.SimpleCallNode(
2064
+ decorator.pos,
2065
+ function=decorator.decorator,
2066
+ args=[decorator_result])
2067
+
2068
+ name_node = ExprNodes.NameNode(node.pos, name=name)
2069
+ reassignment = Nodes.SingleAssignmentNode(
2070
+ node.pos,
2071
+ lhs=name_node,
2072
+ rhs=decorator_result)
2073
+
2074
+ reassignment = Nodes.IndirectionNode([reassignment])
2075
+ node.decorator_indirection = reassignment
2076
+ return [node, reassignment]
2077
+
2078
+
2079
+ class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
2080
+ """
2081
+ Only part of the CythonUtilityCode pipeline. Must be run before
2082
+ DecoratorTransform in case this is a decorator for a cdef class.
2083
+ It filters out @cname('my_cname') decorators and rewrites them to
2084
+ CnameDecoratorNodes.
2085
+ """
2086
+
2087
+ def handle_function(self, node):
2088
+ if not getattr(node, 'decorators', None):
2089
+ return self.visit_Node(node)
2090
+
2091
+ for i, decorator in enumerate(node.decorators):
2092
+ decorator = decorator.decorator
2093
+
2094
+ if (isinstance(decorator, ExprNodes.CallNode) and
2095
+ decorator.function.is_name and
2096
+ decorator.function.name == 'cname'):
2097
+ args, kwargs = decorator.explicit_args_kwds()
2098
+
2099
+ if kwargs:
2100
+ raise AssertionError(
2101
+ "cname decorator does not take keyword arguments")
2102
+
2103
+ if len(args) != 1:
2104
+ raise AssertionError(
2105
+ "cname decorator takes exactly one argument")
2106
+
2107
+ if not (args[0].is_literal and args[0].type is Builtin.unicode_type):
2108
+ raise AssertionError(
2109
+ "argument to cname decorator must be a string literal")
2110
+
2111
+ cname = args[0].compile_time_value(None)
2112
+ del node.decorators[i]
2113
+ node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
2114
+ cname=cname)
2115
+ break
2116
+
2117
+ return self.visit_Node(node)
2118
+
2119
+ visit_FuncDefNode = handle_function
2120
+ visit_CClassDefNode = handle_function
2121
+ visit_CEnumDefNode = handle_function
2122
+ visit_CStructOrUnionDefNode = handle_function
2123
+ visit_CVarDefNode = handle_function
2124
+
2125
+
2126
+ class ForwardDeclareTypes(CythonTransform):
2127
+ """
2128
+ Declare all global cdef names that we allow referencing in other places,
2129
+ before declaring everything (else) in source code order.
2130
+ """
2131
+
2132
+ def visit_CompilerDirectivesMixin(self, node):
2133
+ env = self.module_scope
2134
+ with node.apply_directives(env):
2135
+ self.visitchildren(node)
2136
+ return node
2137
+
2138
+ def visit_ModuleNode(self, node):
2139
+ self.module_scope = node.scope
2140
+ self.module_scope.directives = node.directives
2141
+ self.visitchildren(node)
2142
+ return node
2143
+
2144
+ def visit_CDefExternNode(self, node):
2145
+ old_cinclude_flag = self.module_scope.in_cinclude
2146
+ self.module_scope.in_cinclude = 1
2147
+ self.visitchildren(node)
2148
+ self.module_scope.in_cinclude = old_cinclude_flag
2149
+ return node
2150
+
2151
+ def visit_CEnumDefNode(self, node):
2152
+ node.declare(self.module_scope)
2153
+ return node
2154
+
2155
+ def visit_CStructOrUnionDefNode(self, node):
2156
+ if node.name not in self.module_scope.entries:
2157
+ node.declare(self.module_scope)
2158
+ return node
2159
+
2160
+ def visit_CClassDefNode(self, node):
2161
+ if node.class_name not in self.module_scope.entries:
2162
+ node.declare(self.module_scope)
2163
+ # Expand fused methods of .pxd declared types to construct the final vtable order.
2164
+ type = self.module_scope.entries[node.class_name].type
2165
+ if type is not None and type.is_extension_type and not type.is_builtin_type and type.scope:
2166
+ scope = type.scope
2167
+ for entry in scope.cfunc_entries:
2168
+ if entry.type and entry.type.is_fused:
2169
+ entry.type.get_all_specialized_function_types()
2170
+ return node
2171
+
2172
+ def visit_FuncDefNode(self, node):
2173
+ # no traversal needed
2174
+ return node
2175
+
2176
+ def visit_PyClassDefNode(self, node):
2177
+ # no traversal needed
2178
+ return node
2179
+
2180
+
2181
+ class AnalyseDeclarationsTransform(EnvTransform):
2182
+
2183
+ basic_property = TreeFragment("""
2184
+ property NAME:
2185
+ def __get__(self):
2186
+ with CRITICAL_SECTION(self):
2187
+ return ATTR
2188
+ def __set__(self, value):
2189
+ with CRITICAL_SECTION(self):
2190
+ ATTR = value
2191
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2192
+ basic_pyobject_property = TreeFragment("""
2193
+ property NAME:
2194
+ def __get__(self):
2195
+ with CRITICAL_SECTION(self):
2196
+ return ATTR
2197
+ def __set__(self, value):
2198
+ with CRITICAL_SECTION(self):
2199
+ ATTR = value
2200
+ def __del__(self):
2201
+ with CRITICAL_SECTION(self):
2202
+ ATTR = None
2203
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2204
+ basic_property_ro = TreeFragment("""
2205
+ property NAME:
2206
+ def __get__(self):
2207
+ with CRITICAL_SECTION(self):
2208
+ return ATTR
2209
+ """, level='c_class', pipeline=[NormalizeTree(None)])
2210
+
2211
+ struct_or_union_wrapper = TreeFragment("""
2212
+ cdef class NAME:
2213
+ cdef TYPE value
2214
+ def __init__(self, MEMBER=None):
2215
+ cdef int count
2216
+ count = 0
2217
+ INIT_ASSIGNMENTS
2218
+ if IS_UNION and count > 1:
2219
+ raise ValueError, "At most one union member should be specified."
2220
+ def __str__(self):
2221
+ return STR_FORMAT % MEMBER_TUPLE
2222
+ def __repr__(self):
2223
+ return REPR_FORMAT % MEMBER_TUPLE
2224
+ """, pipeline=[NormalizeTree(None)])
2225
+
2226
+ init_assignment = TreeFragment("""
2227
+ if VALUE is not None:
2228
+ ATTR = VALUE
2229
+ count += 1
2230
+ """, pipeline=[NormalizeTree(None)])
2231
+
2232
+ fused_function = None
2233
+ in_lambda = 0
2234
+
2235
+ def __call__(self, root):
2236
+ # needed to determine if a cdef var is declared after it's used.
2237
+ self.seen_vars_stack = []
2238
+ self.fused_error_funcs = set()
2239
+ super_class = super()
2240
+ self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
2241
+ return super_class.__call__(root)
2242
+
2243
+ def visit_NameNode(self, node):
2244
+ self.seen_vars_stack[-1].add(node.name)
2245
+ return node
2246
+
2247
+ def visit_ModuleNode(self, node):
2248
+ # Pickling support requires injecting module-level nodes.
2249
+ self.extra_module_declarations = []
2250
+ self.seen_vars_stack.append(set())
2251
+ node.analyse_declarations(self.current_env())
2252
+ self.visitchildren(node)
2253
+ self.seen_vars_stack.pop()
2254
+ node.body.stats.extend(self.extra_module_declarations)
2255
+ return node
2256
+
2257
+ def visit_LambdaNode(self, node):
2258
+ self.in_lambda += 1
2259
+ node.analyse_declarations(self.current_env())
2260
+ self.visitchildren(node)
2261
+ self.in_lambda -= 1
2262
+ return node
2263
+
2264
+ def visit_CClassDefNode(self, node):
2265
+ node = self.visit_ClassDefNode(node)
2266
+ if node.scope and 'dataclasses.dataclass' in node.scope.directives:
2267
+ from .Dataclass import handle_cclass_dataclass
2268
+ handle_cclass_dataclass(node, node.scope.directives['dataclasses.dataclass'], self)
2269
+ if node.scope and node.scope.implemented and node.body:
2270
+ stats = []
2271
+ for entry in node.scope.var_entries:
2272
+ if entry.needs_property:
2273
+ property = self.create_Property(entry)
2274
+ property.analyse_declarations(node.scope)
2275
+ self.visit(property)
2276
+ stats.append(property)
2277
+ if stats:
2278
+ node.body.stats += stats
2279
+ if (node.visibility != 'extern'
2280
+ and not node.scope.lookup('__reduce__')
2281
+ and not node.scope.lookup('__reduce_ex__')):
2282
+ self._inject_pickle_methods(node)
2283
+ return node
2284
+
2285
+ def _inject_pickle_methods(self, node):
2286
+ env = self.current_env()
2287
+ if node.scope.directives['auto_pickle'] is False: # None means attempt it.
2288
+ # Old behavior of not doing anything.
2289
+ return
2290
+ auto_pickle_forced = node.scope.directives['auto_pickle'] is True
2291
+
2292
+ all_members = []
2293
+ cls = node.entry.type
2294
+ cinit = None
2295
+ inherited_reduce = None
2296
+ while cls is not None:
2297
+ all_members.extend(e for e in cls.scope.var_entries if e.name not in ('__weakref__', '__dict__'))
2298
+ cinit = cinit or cls.scope.lookup('__cinit__')
2299
+ inherited_reduce = inherited_reduce or cls.scope.lookup('__reduce__') or cls.scope.lookup('__reduce_ex__')
2300
+ cls = cls.base_type
2301
+ all_members.sort(key=lambda e: e.name)
2302
+
2303
+ if inherited_reduce:
2304
+ # This is not failsafe, as we may not know whether a cimported class defines a __reduce__.
2305
+ # This is why we define __reduce_cython__ and only replace __reduce__
2306
+ # (via ExtensionTypes.SetupReduce utility code) at runtime on class creation.
2307
+ return
2308
+
2309
+ non_py = [
2310
+ e for e in all_members
2311
+ if not e.type.is_pyobject and (not e.type.can_coerce_to_pyobject(env)
2312
+ or not e.type.can_coerce_from_pyobject(env))
2313
+ ]
2314
+
2315
+ structs = [e for e in all_members if e.type.is_struct_or_union]
2316
+
2317
+ if cinit or non_py or (structs and not auto_pickle_forced):
2318
+ if cinit:
2319
+ # TODO(robertwb): We could allow this if __cinit__ has no require arguments.
2320
+ msg = 'no default __reduce__ due to non-trivial __cinit__'
2321
+ elif non_py:
2322
+ msg = "%s cannot be converted to a Python object for pickling" % ','.join("self.%s" % e.name for e in non_py)
2323
+ else:
2324
+ # Extern structs may be only partially defined.
2325
+ # TODO(robertwb): Limit the restriction to extern
2326
+ # (and recursively extern-containing) structs.
2327
+ msg = ("Pickling of struct members such as %s must be explicitly requested "
2328
+ "with @auto_pickle(True)" % ','.join("self.%s" % e.name for e in structs))
2329
+
2330
+ if auto_pickle_forced:
2331
+ error(node.pos, msg)
2332
+
2333
+ pickle_func = TreeFragment("""
2334
+ def __reduce_cython__(self):
2335
+ raise TypeError, "%(msg)s"
2336
+ def __setstate_cython__(self, __pyx_state):
2337
+ raise TypeError, "%(msg)s"
2338
+ """ % {'msg': msg},
2339
+ level='c_class', pipeline=[NormalizeTree(None)]).substitute({})
2340
+ pickle_func.analyse_declarations(node.scope)
2341
+ self.visit(pickle_func)
2342
+ node.body.stats.append(pickle_func)
2343
+
2344
+ else:
2345
+ for e in all_members:
2346
+ if not e.type.is_pyobject:
2347
+ e.type.create_to_py_utility_code(env)
2348
+ e.type.create_from_py_utility_code(env)
2349
+
2350
+ all_members_names = [e.name for e in all_members]
2351
+ assignments = '; '.join([
2352
+ '__pyx_result.%s = __pyx_state[%s]' % (v, ix)
2353
+ for ix, v in enumerate(all_members_names)
2354
+ ])
2355
+ checksums = _calculate_pickle_checksums(all_members_names)
2356
+ if len(checksums) != 3:
2357
+ # If we don't have enough checksums to call the check function, we just repeat the last one.
2358
+ checksums = (checksums + [checksums[-1] * 2])[:3]
2359
+
2360
+ unpickle_func_name = f'__pyx_unpickle_{node.punycode_class_name}'
2361
+ num_members = len(all_members_names)
2362
+
2363
+ env.use_utility_code(Code.UtilityCode.load_cached("UpdateUnpickledDict", "ExtensionTypes.c"))
2364
+
2365
+ # TODO(robertwb): Move the state into the third argument
2366
+ # so it can be pickled *after* self is memoized.
2367
+ unpickle_code = f"""
2368
+ cdef extern from *:
2369
+ int __Pyx_CheckUnpickleChecksum(long, long, long, long, const char*) except -1
2370
+ int __Pyx_UpdateUnpickledDict(object, object, Py_ssize_t) except -1
2371
+
2372
+ def {unpickle_func_name}(__pyx_type, long __pyx_checksum, tuple __pyx_state):
2373
+ cdef object __pyx_result
2374
+ __Pyx_CheckUnpickleChecksum(__pyx_checksum, {', '.join(checksums)}, {', '.join(all_members_names).encode('UTF-8')!r})
2375
+ __pyx_result = {node.class_name}.__new__(__pyx_type)
2376
+ if __pyx_state is not None:
2377
+ {unpickle_func_name}__set_state(<{node.class_name}> __pyx_result, __pyx_state)
2378
+ return __pyx_result
2379
+
2380
+ cdef {unpickle_func_name}__set_state({node.class_name} __pyx_result, __pyx_state: tuple):
2381
+ {assignments}
2382
+ __Pyx_UpdateUnpickledDict(__pyx_result, __pyx_state, {num_members:d})
2383
+ """
2384
+
2385
+ env.use_utility_code(Code.UtilityCode.load_cached("CheckUnpickleChecksum", "ExtensionTypes.c"))
2386
+
2387
+ unpickle_func = TreeFragment(unpickle_code, level='module', pipeline=[NormalizeTree(None)]).substitute({})
2388
+ unpickle_func.analyse_declarations(node.entry.scope)
2389
+
2390
+ self.visit(unpickle_func)
2391
+ self.extra_module_declarations.append(unpickle_func)
2392
+
2393
+ members = ', '.join(f'self.{v}' for v in all_members_names) + (',' if len(all_members_names) == 1 else '')
2394
+ # Even better, we could check PyType_IS_GC.
2395
+ any_notnone_members = ' or '.join([f'self.{e.name} is not None' for e in all_members if e.type.is_pyobject] or ['False'])
2396
+
2397
+ pickle_code = f"""
2398
+ def __reduce_cython__(self):
2399
+ cdef tuple state
2400
+ cdef object _dict
2401
+ cdef bint use_setstate
2402
+ with CRITICAL_SECTION(self):
2403
+ state = ({members})
2404
+ _dict = getattr(self, '__dict__', None)
2405
+ if _dict is not None and _dict:
2406
+ state += (_dict,)
2407
+ use_setstate = True
2408
+ else:
2409
+ use_setstate = {any_notnone_members}
2410
+ if use_setstate:
2411
+ return {unpickle_func_name}, (type(self), {checksums[0]}, None), state
2412
+ else:
2413
+ return {unpickle_func_name}, (type(self), {checksums[0]}, state)
2414
+
2415
+ def __setstate_cython__(self, __pyx_state):
2416
+ {unpickle_func_name}__set_state(self, __pyx_state)
2417
+ """
2418
+
2419
+ pickle_func = TreeFragment(pickle_code, level='c_class', pipeline=[NormalizeTree(None)]).substitute(
2420
+ {'CRITICAL_SECTION': self._create_critical_section_name_node(node.scope, node.pos)}
2421
+ )
2422
+ pickle_func = InterpretCompilerDirectives(None, {})(pickle_func)
2423
+ pickle_func.analyse_declarations(node.scope)
2424
+
2425
+ self.enter_scope(node, node.scope) # functions should be visited in the class scope
2426
+ self.visit(pickle_func)
2427
+ self.exit_scope()
2428
+ node.body.stats.append(pickle_func)
2429
+
2430
+ def _handle_fused_def_decorators(self, old_decorators, env, node):
2431
+ """
2432
+ Create function calls to the decorators and reassignments to
2433
+ the function.
2434
+ """
2435
+ # Delete staticmethod and classmethod decorators, this is
2436
+ # handled directly by the fused function object.
2437
+ decorators = []
2438
+ for decorator in old_decorators:
2439
+ func = decorator.decorator
2440
+ if (not func.is_name or
2441
+ func.name not in ('staticmethod', 'classmethod') or
2442
+ env.lookup_here(func.name)):
2443
+ # not a static or classmethod
2444
+ decorators.append(decorator)
2445
+
2446
+ if decorators:
2447
+ transform = DecoratorTransform(self.context)
2448
+ def_node = node.node
2449
+ _, reassignments = transform.chain_decorators(
2450
+ def_node, decorators, def_node.name)
2451
+ reassignments.analyse_declarations(env)
2452
+ node = [node, reassignments]
2453
+
2454
+ return node
2455
+
2456
+ def _handle_def(self, decorators, env, node):
2457
+ "Handle def or cpdef fused functions"
2458
+ # Create PyCFunction nodes for each specialization
2459
+ node.stats.insert(0, node.py_func)
2460
+ self.visitchild(node, 'py_func')
2461
+ node.update_fused_defnode_entry(env)
2462
+ # For the moment, fused functions do not support METH_FASTCALL
2463
+ call_signature = node.py_func.entry.signature
2464
+ call_signature.use_fastcall = call_signature.FastcallUsed.NO
2465
+ pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func, binding=True)
2466
+ pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
2467
+ node.resulting_fused_function = pycfunc
2468
+ # Create assignment node for our def function
2469
+ node.fused_func_assignment = self._create_assignment(
2470
+ node.py_func, ExprNodes.CloneNode(pycfunc), env)
2471
+
2472
+ if decorators:
2473
+ node = self._handle_fused_def_decorators(decorators, env, node)
2474
+
2475
+ return node
2476
+
2477
+ def _create_fused_function(self, env, node):
2478
+ "Create a fused function for a DefNode with fused arguments"
2479
+ from . import FusedNode
2480
+
2481
+ if self.fused_function or self.in_lambda:
2482
+ if self.fused_function not in self.fused_error_funcs:
2483
+ if self.in_lambda:
2484
+ error(node.pos, "Fused lambdas not allowed")
2485
+ else:
2486
+ error(node.pos, "Cannot nest fused functions")
2487
+
2488
+ self.fused_error_funcs.add(self.fused_function)
2489
+
2490
+ node.body = Nodes.PassStatNode(node.pos)
2491
+ for arg in node.args:
2492
+ if arg.type.is_fused:
2493
+ arg.type = arg.type.get_fused_types()[0]
2494
+
2495
+ return node
2496
+
2497
+ decorators = getattr(node, 'decorators', None)
2498
+ node = FusedNode.FusedCFuncDefNode(node, env)
2499
+ if node.py_func:
2500
+ node.attach_fused_py_funcs()
2501
+ self.fused_function = node
2502
+ self.visitchildren(node)
2503
+ self.fused_function = None
2504
+ if node.py_func:
2505
+ node = self._handle_def(decorators, env, node)
2506
+
2507
+ return node
2508
+
2509
+ def _handle_fused(self, node):
2510
+ if node.is_generator and node.has_fused_arguments:
2511
+ error(node.pos, "Fused generators not supported")
2512
+ node.has_fused_arguments = False
2513
+ node.gbody.body = Nodes.StatListNode(node.pos, stats=[])
2514
+
2515
+ return node.has_fused_arguments
2516
+
2517
+ def visit_FuncDefNode(self, node):
2518
+ """
2519
+ Analyse a function and its body, as that hasn't happened yet. Also
2520
+ analyse the directive_locals set by @cython.locals().
2521
+
2522
+ Then, if we are a function with fused arguments, replace the function
2523
+ (after it has declared itself in the symbol table!) with a
2524
+ FusedCFuncDefNode, and analyse its children (which are in turn normal
2525
+ functions). If we're a normal function, just analyse the body of the
2526
+ function.
2527
+ """
2528
+ env = self.current_env()
2529
+
2530
+ self.seen_vars_stack.append(set())
2531
+ lenv = node.local_scope
2532
+ node.declare_arguments(lenv)
2533
+
2534
+ # @cython.locals(...)
2535
+ for var, type_node in node.directive_locals.items():
2536
+ if not lenv.lookup_here(var): # don't redeclare args
2537
+ type = type_node.analyse_as_type(lenv)
2538
+ if type and type.is_fused and lenv.fused_to_specific:
2539
+ type = type.specialize(lenv.fused_to_specific)
2540
+ if type:
2541
+ lenv.declare_var(var, type, type_node.pos)
2542
+ else:
2543
+ error(type_node.pos, "Not a type")
2544
+
2545
+ if self._handle_fused(node):
2546
+ node = self._create_fused_function(env, node)
2547
+ else:
2548
+ node.body.analyse_declarations(lenv)
2549
+ node = self._super_visit_FuncDefNode(node)
2550
+
2551
+ self.seen_vars_stack.pop()
2552
+
2553
+ if "ufunc" in lenv.directives:
2554
+ from . import UFuncs
2555
+ return UFuncs.convert_to_ufunc(node)
2556
+ return node
2557
+
2558
+ def visit_DefNode(self, node):
2559
+ node = self.visit_FuncDefNode(node)
2560
+ if not isinstance(node, Nodes.DefNode):
2561
+ return node
2562
+ env = self.current_env()
2563
+ if node.code_object is None:
2564
+ node.code_object = ExprNodes.CodeObjectNode(node)
2565
+ node.code_object.analyse_declarations(env)
2566
+ if node.fused_py_func or node.is_generator_body:
2567
+ return node
2568
+ if not node.needs_assignment_synthesis(env):
2569
+ return node
2570
+ return [node, self._synthesize_assignment(node, env)]
2571
+
2572
+ def visit_CFuncDefNode(self, node):
2573
+ if node.code_object is None and node.py_func is None:
2574
+ node.code_object = ExprNodes.CodeObjectNode.for_cfunc(node)
2575
+ node.code_object.analyse_declarations(self.current_env())
2576
+ return self.visit_FuncDefNode(node)
2577
+
2578
+ def visit_GeneratorBodyDefNode(self, node):
2579
+ return self.visit_FuncDefNode(node)
2580
+
2581
+ def visit_GeneratorDefNode(self, node):
2582
+ # The generator body should use the same code object as the (user facing) generator function that creates it.
2583
+ result = self.visit_DefNode(node)
2584
+ # 'result' will usually be a list of statements, but we still have the original node.
2585
+ node.gbody.code_object = node.code_object
2586
+ return result
2587
+
2588
+ def _synthesize_assignment(self, node, env):
2589
+ # Synthesize assignment node and put it right after defnode
2590
+ genv = env
2591
+ while genv.is_py_class_scope or genv.is_c_class_scope:
2592
+ genv = genv.outer_scope
2593
+
2594
+ binding = env.is_py_class_scope or self.current_directives.get('binding')
2595
+ if genv.is_closure_scope:
2596
+ rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode.from_defnode(node, binding)
2597
+ else:
2598
+ rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
2599
+
2600
+ node.is_cyfunction = rhs.binding
2601
+ return self._create_assignment(node, rhs, env)
2602
+
2603
+ def _create_assignment(self, def_node, rhs, env):
2604
+ if def_node.decorators:
2605
+ for decorator in def_node.decorators[::-1]:
2606
+ rhs = ExprNodes.SimpleCallNode(
2607
+ decorator.pos,
2608
+ function = decorator.decorator,
2609
+ args = [rhs])
2610
+ def_node.decorators = None
2611
+
2612
+ assmt = Nodes.SingleAssignmentNode(
2613
+ def_node.pos,
2614
+ lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
2615
+ rhs=rhs)
2616
+ assmt.analyse_declarations(env)
2617
+ return assmt
2618
+
2619
+ def visit_func_outer_attrs(self, node):
2620
+ # any names in the outer attrs should not be looked up in the function "seen_vars_stack"
2621
+ stack = self.seen_vars_stack.pop()
2622
+ super().visit_func_outer_attrs(node)
2623
+ self.seen_vars_stack.append(stack)
2624
+
2625
+ def visit_ScopedExprNode(self, node):
2626
+ env = self.current_env()
2627
+ node.analyse_declarations(env)
2628
+ # the node may or may not have a local scope
2629
+ if node.expr_scope:
2630
+ self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
2631
+ self.enter_scope(node, node.expr_scope)
2632
+ node.analyse_scoped_declarations(node.expr_scope)
2633
+ self.visitchildren(node)
2634
+ self.exit_scope()
2635
+ self.seen_vars_stack.pop()
2636
+ else:
2637
+
2638
+ node.analyse_scoped_declarations(env)
2639
+ self.visitchildren(node)
2640
+ return node
2641
+
2642
+ def visit_TempResultFromStatNode(self, node):
2643
+ self.visitchildren(node)
2644
+ node.analyse_declarations(self.current_env())
2645
+ return node
2646
+
2647
+ def visit_CppClassNode(self, node):
2648
+ if node.visibility == 'extern':
2649
+ return None
2650
+ else:
2651
+ return self.visit_ClassDefNode(node)
2652
+
2653
+ def visit_CStructOrUnionDefNode(self, node):
2654
+ # Create a wrapper node if needed.
2655
+ # We want to use the struct type information (so it can't happen
2656
+ # before this phase) but also create new objects to be declared
2657
+ # (so it can't happen later).
2658
+ # Note that we don't return the original node, as it is
2659
+ # never used after this phase.
2660
+ if True: # private (default)
2661
+ return None
2662
+
2663
+ self_value = ExprNodes.AttributeNode(
2664
+ pos = node.pos,
2665
+ obj = ExprNodes.NameNode(pos=node.pos, name="self"),
2666
+ attribute = EncodedString("value"))
2667
+ var_entries = node.entry.type.scope.var_entries
2668
+ attributes = []
2669
+ for entry in var_entries:
2670
+ attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
2671
+ obj = self_value,
2672
+ attribute = entry.name))
2673
+ # __init__ assignments
2674
+ init_assignments = []
2675
+ for entry, attr in zip(var_entries, attributes):
2676
+ # TODO: branch on visibility
2677
+ init_assignments.append(
2678
+ self.init_assignment.substitute(
2679
+ {
2680
+ "VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
2681
+ "ATTR": attr,
2682
+ },
2683
+ pos=entry.pos,
2684
+ )
2685
+ )
2686
+
2687
+ # create the class
2688
+ str_format = "%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
2689
+ wrapper_class = self.struct_or_union_wrapper.substitute({
2690
+ "INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
2691
+ "IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
2692
+ "MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
2693
+ "STR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format)),
2694
+ "REPR_FORMAT": ExprNodes.UnicodeNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
2695
+ }, pos = node.pos).stats[0]
2696
+ wrapper_class.class_name = node.name
2697
+ wrapper_class.shadow = True
2698
+ class_body = wrapper_class.body.stats
2699
+
2700
+ # fix value type
2701
+ assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
2702
+ class_body[0].base_type.name = node.name
2703
+
2704
+ # fix __init__ arguments
2705
+ init_method = class_body[1]
2706
+ assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
2707
+ arg_template = init_method.args[1]
2708
+ if not node.entry.type.is_struct:
2709
+ arg_template.kw_only = True
2710
+ del init_method.args[1]
2711
+ for entry, attr in zip(var_entries, attributes):
2712
+ arg = copy.deepcopy(arg_template)
2713
+ arg.declarator.name = entry.name
2714
+ init_method.args.append(arg)
2715
+
2716
+ # setters/getters
2717
+ for entry, attr in zip(var_entries, attributes):
2718
+ # TODO: branch on visibility
2719
+ if entry.type.is_pyobject:
2720
+ template = self.basic_pyobject_property
2721
+ else:
2722
+ template = self.basic_property
2723
+ property = template.substitute(
2724
+ {
2725
+ "ATTR": attr,
2726
+ },
2727
+ pos=entry.pos,
2728
+ ).stats[0]
2729
+ property.name = entry.name
2730
+ wrapper_class.body.stats.append(property)
2731
+
2732
+ wrapper_class.analyse_declarations(self.current_env())
2733
+ return self.visit_CClassDefNode(wrapper_class)
2734
+
2735
+ # Some nodes are no longer needed after declaration
2736
+ # analysis and can be dropped. The analysis was performed
2737
+ # on these nodes in a separate recursive process from the
2738
+ # enclosing function or module, so we can simply drop them.
2739
+ def visit_CDeclaratorNode(self, node):
2740
+ # necessary to ensure that all CNameDeclaratorNodes are visited.
2741
+ self.visitchildren(node)
2742
+ return node
2743
+
2744
+ def visit_CTypeDefNode(self, node):
2745
+ return node
2746
+
2747
+ def visit_CBaseTypeNode(self, node):
2748
+ return None
2749
+
2750
+ def visit_CEnumDefNode(self, node):
2751
+ if node.visibility == 'public':
2752
+ return node
2753
+ else:
2754
+ return None
2755
+
2756
+ def visit_CNameDeclaratorNode(self, node):
2757
+ if node.name in self.seen_vars_stack[-1]:
2758
+ entry = self.current_env().lookup(node.name)
2759
+ if (entry is None or entry.visibility != 'extern'
2760
+ and not entry.scope.is_c_class_scope):
2761
+ error(node.pos, "cdef variable '%s' declared after it is used" % node.name)
2762
+ self.visitchildren(node)
2763
+ return node
2764
+
2765
+ def visit_CVarDefNode(self, node):
2766
+ # to ensure all CNameDeclaratorNodes are visited.
2767
+ self.visitchildren(node)
2768
+ return None
2769
+
2770
+ def visit_CnameDecoratorNode(self, node):
2771
+ child_node = self.visitchild(node, 'node')
2772
+ if not child_node:
2773
+ return None
2774
+ if type(child_node) is list: # Assignment synthesized
2775
+ node.node = child_node[0]
2776
+ return [node] + child_node[1:]
2777
+ return node
2778
+
2779
+ def create_Property(self, entry):
2780
+ if entry.visibility == 'public':
2781
+ if entry.type.is_pyobject:
2782
+ template = self.basic_pyobject_property
2783
+ else:
2784
+ template = self.basic_property
2785
+ elif entry.visibility == 'readonly':
2786
+ template = self.basic_property_ro
2787
+ property = template.substitute(
2788
+ {
2789
+ "ATTR": ExprNodes.AttributeNode(pos=entry.pos,
2790
+ obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
2791
+ attribute=entry.name),
2792
+ "CRITICAL_SECTION": self._create_critical_section_name_node(self.current_env(), entry.pos)
2793
+ },
2794
+ pos=entry.pos
2795
+ ).stats[0]
2796
+ property = InterpretCompilerDirectives(None, {})(property)
2797
+ property.name = entry.name
2798
+ property.doc = entry.doc
2799
+ return property
2800
+
2801
+ def visit_AssignmentExpressionNode(self, node):
2802
+ self.visitchildren(node)
2803
+ node.analyse_declarations(self.current_env())
2804
+ return node
2805
+
2806
+ def _create_critical_section_name_node(self, env, pos):
2807
+ return ExprNodes.NameNode(
2808
+ pos,
2809
+ name="critical_section",
2810
+ cython_attribute="critical_section"
2811
+ )
2812
+
2813
+
2814
+ def _calculate_pickle_checksums(member_names):
2815
+ # Cython 0.x used MD5 for the checksum, which a few Python installations remove for security reasons.
2816
+ # SHA-256 should be ok for years to come, but early Cython 3.0 alpha releases used SHA-1,
2817
+ # which may not be.
2818
+ member_names_string = ' '.join(member_names).encode('utf-8')
2819
+ checksums = []
2820
+ for algo_name in ['sha256', 'sha1', 'md5']:
2821
+ try:
2822
+ mkchecksum = getattr(hashlib, algo_name)
2823
+ checksum = mkchecksum(member_names_string, usedforsecurity=False).hexdigest()
2824
+ except (AttributeError, ValueError):
2825
+ # The algorithm (i.e. MD5) might not be there at all, or might be blocked at runtime.
2826
+ continue
2827
+ checksums.append('0x' + checksum[:7])
2828
+ return checksums
2829
+
2830
+
2831
+ class CalculateQualifiedNamesTransform(EnvTransform):
2832
+ """
2833
+ Calculate and store the '__qualname__' and the global
2834
+ module name on some nodes.
2835
+ """
2836
+ needs_qualname_assignment = False
2837
+ needs_module_assignment = False
2838
+
2839
+ def visit_ModuleNode(self, node):
2840
+ self.module_name = self.global_scope().qualified_name
2841
+ self.qualified_name = []
2842
+ _super = super()
2843
+ self._super_visit_FuncDefNode = _super.visit_FuncDefNode
2844
+ self._super_visit_ClassDefNode = _super.visit_ClassDefNode
2845
+ self.visitchildren(node)
2846
+ return node
2847
+
2848
+ def _set_qualname(self, node, name=None):
2849
+ if name:
2850
+ qualname = self.qualified_name[:]
2851
+ qualname.append(name)
2852
+ else:
2853
+ qualname = self.qualified_name
2854
+ node.qualname = EncodedString('.'.join(qualname))
2855
+ node.module_name = self.module_name
2856
+
2857
+ def _append_entry(self, entry):
2858
+ if entry.is_pyglobal and not entry.is_pyclass_attr:
2859
+ self.qualified_name = [entry.name]
2860
+ else:
2861
+ self.qualified_name.append(entry.name)
2862
+
2863
+ def visit_ClassNode(self, node):
2864
+ self._set_qualname(node, node.name)
2865
+ self.visitchildren(node)
2866
+ return node
2867
+
2868
+ def visit_PyClassNamespaceNode(self, node):
2869
+ # class name was already added by parent node
2870
+ self._set_qualname(node)
2871
+ self.visitchildren(node)
2872
+ return node
2873
+
2874
+ def visit_PyCFunctionNode(self, node):
2875
+ orig_qualified_name = self.qualified_name[:]
2876
+ if node.def_node.is_wrapper and self.qualified_name and self.qualified_name[-1] == '<locals>':
2877
+ self.qualified_name.pop()
2878
+ self._set_qualname(node)
2879
+ else:
2880
+ self._set_qualname(node, node.def_node.name)
2881
+ self.visitchildren(node)
2882
+ self.qualified_name = orig_qualified_name
2883
+ return node
2884
+
2885
+ def visit_DefNode(self, node):
2886
+ if node.is_wrapper and self.qualified_name:
2887
+ assert self.qualified_name[-1] == '<locals>', self.qualified_name
2888
+ orig_qualified_name = self.qualified_name[:]
2889
+ self.qualified_name.pop()
2890
+ self._set_qualname(node)
2891
+ self._super_visit_FuncDefNode(node)
2892
+ self.qualified_name = orig_qualified_name
2893
+ else:
2894
+ self._set_qualname(node, node.name)
2895
+ self.visit_FuncDefNode(node)
2896
+ return node
2897
+
2898
+ def visit_FuncDefNode(self, node):
2899
+ orig_qualified_name = self.qualified_name[:]
2900
+ if getattr(node, 'name', None) == '<lambda>':
2901
+ self.qualified_name.append('<lambda>')
2902
+ else:
2903
+ self._append_entry(node.entry)
2904
+ self.qualified_name.append('<locals>')
2905
+ self._super_visit_FuncDefNode(node)
2906
+ self.qualified_name = orig_qualified_name
2907
+ return node
2908
+
2909
+ def generate_assignment(self, node, name, value):
2910
+ entry = node.scope.lookup_here(name)
2911
+ lhs = ExprNodes.NameNode(
2912
+ node.pos,
2913
+ name=EncodedString(name),
2914
+ entry=entry,
2915
+ is_target=True)
2916
+ rhs = ExprNodes.UnicodeNode(node.pos, value=value)
2917
+ node.body.stats.insert(0, Nodes.SingleAssignmentNode(
2918
+ node.pos,
2919
+ lhs=lhs,
2920
+ rhs=rhs,
2921
+ ).analyse_expressions(self.current_env()))
2922
+
2923
+ def visit_ClassDefNode(self, node):
2924
+ orig_needs_qualname_assignment = self.needs_qualname_assignment
2925
+ self.needs_qualname_assignment = False
2926
+ orig_needs_module_assignment = self.needs_module_assignment
2927
+ self.needs_module_assignment = False
2928
+ orig_qualified_name = self.qualified_name[:]
2929
+ entry = (getattr(node, 'entry', None) or # PyClass
2930
+ self.current_env().lookup_here(node.target.name)) # CClass
2931
+ self._append_entry(entry)
2932
+ self._super_visit_ClassDefNode(node)
2933
+ if self.needs_qualname_assignment:
2934
+ self.generate_assignment(node, "__qualname__",
2935
+ EncodedString(".".join(self.qualified_name)))
2936
+ if self.needs_module_assignment:
2937
+ self.generate_assignment(node, "__module__",
2938
+ EncodedString(self.module_name))
2939
+ self.qualified_name = orig_qualified_name
2940
+ self.needs_qualname_assignment = orig_needs_qualname_assignment
2941
+ self.needs_module_assignment = orig_needs_module_assignment
2942
+ return node
2943
+
2944
+ def visit_NameNode(self, node):
2945
+ scope = self.current_env()
2946
+ if scope.is_c_class_scope:
2947
+ # unlike for a PyClass scope, these attributes aren't defined in the
2948
+ # dictionary when the class definition is executed, therefore we ask
2949
+ # the compiler to generate an assignment to them at the start of the
2950
+ # body.
2951
+ # NOTE: this doesn't put them in locals()
2952
+ if node.name == "__qualname__":
2953
+ self.needs_qualname_assignment = True
2954
+ elif node.name == "__module__":
2955
+ self.needs_module_assignment = True
2956
+ return node
2957
+
2958
+
2959
+ class AnalyseExpressionsTransform(CythonTransform):
2960
+
2961
+ def visit_ModuleNode(self, node):
2962
+ node.scope.infer_types()
2963
+ node.body = node.body.analyse_expressions(node.scope)
2964
+ self.positions = [{node.pos}]
2965
+ self.visitchildren(node)
2966
+ self._build_positions(node)
2967
+ return node
2968
+
2969
+ def visit_FuncDefNode(self, node):
2970
+ node.local_scope.infer_types()
2971
+ node.body = node.body.analyse_expressions(node.local_scope)
2972
+ self.positions[-1].add(node.pos)
2973
+
2974
+ if node.is_wrapper:
2975
+ # Share positions between function and Python wrapper.
2976
+ local_positions = self.positions[-1]
2977
+ else:
2978
+ local_positions = {node.pos}
2979
+ self.positions.append(local_positions)
2980
+
2981
+ self.visitchildren(node)
2982
+ self._build_positions(node)
2983
+ return node
2984
+
2985
+ def visit_ScopedExprNode(self, node):
2986
+ if node.has_local_scope:
2987
+ node.expr_scope.infer_types()
2988
+ node = node.analyse_scoped_expressions(node.expr_scope)
2989
+ self.visit_ExprNode(node)
2990
+ return node
2991
+
2992
+ def visit_IndexNode(self, node):
2993
+ """
2994
+ Replace index nodes used to specialize cdef functions with fused
2995
+ argument types with the Attribute- or NameNode referring to the
2996
+ function. We then need to copy over the specialization properties to
2997
+ the attribute or name node.
2998
+
2999
+ Because the indexing might be a Python indexing operation on a fused
3000
+ function, or (usually) a Cython indexing operation, we need to
3001
+ re-analyse the types.
3002
+ """
3003
+ self.visit_ExprNode(node)
3004
+ if node.is_fused_index and not node.type.is_error:
3005
+ node = node.base
3006
+ return node
3007
+
3008
+ # Build the line table according to PEP-626.
3009
+ # We mostly just do this here to avoid yet another transform traversal.
3010
+
3011
+ def visit_ExprNode(self, node):
3012
+ self.positions[-1].add(node.pos)
3013
+ self.visitchildren(node)
3014
+ return node
3015
+
3016
+ def visit_StatNode(self, node):
3017
+ self.positions[-1].add(node.pos)
3018
+ self.visitchildren(node)
3019
+ return node
3020
+
3021
+ def _build_positions(self, func_node):
3022
+ """
3023
+ Build the PEP-626 line table and "bytecode-to-position" mapping used for CodeObjects.
3024
+ """
3025
+ # Code can originate from different source files and string code fragments, even within a single function.
3026
+ # Thus, it's not completely correct to just ignore the source files when sorting the line numbers,
3027
+ # but it also doesn't hurt much for the moment. Eventually, we might need different CodeObjects
3028
+ # even within a single function if it uses code from different sources / line number ranges.
3029
+ positions: list = sorted(
3030
+ self.positions.pop(),
3031
+ key=itemgetter(1, 2), # (line, column)
3032
+ # Build ranges backwards to know the end column before we see the start column in the same line.
3033
+ reverse=True,
3034
+ )
3035
+
3036
+ next_line = -1
3037
+ next_column_in_line = 0
3038
+
3039
+ ranges = []
3040
+ for _, line, start_column in positions:
3041
+ ranges.append((line, line, start_column, next_column_in_line if line == next_line else start_column + 1))
3042
+ next_line, next_column_in_line = line, start_column
3043
+
3044
+ ranges.reverse()
3045
+ func_node.node_positions = ranges
3046
+
3047
+ positions.reverse()
3048
+ i: cython.Py_ssize_t
3049
+ func_node.local_scope.node_positions_to_offset = {
3050
+ position: i
3051
+ for i, position in enumerate(positions)
3052
+ }
3053
+
3054
+
3055
+ class FindInvalidUseOfFusedTypes(TreeVisitor):
3056
+
3057
+ def __call__(self, tree):
3058
+ self._in_fused_function = False
3059
+ self.visit(tree)
3060
+ return tree
3061
+
3062
+ def visit_Node(self, node):
3063
+ self.visitchildren(node)
3064
+
3065
+ def visit_FuncDefNode(self, node):
3066
+ outer_status = self._in_fused_function
3067
+ self._in_fused_function = node.has_fused_arguments
3068
+
3069
+ if not self._in_fused_function:
3070
+ # Errors related to use in functions with fused args will already
3071
+ # have been detected.
3072
+ if not node.is_generator_body and node.return_type.is_fused:
3073
+ error(node.pos, "Return type is not specified as argument type")
3074
+
3075
+ self.visitchildren(node)
3076
+ self._in_fused_function = outer_status
3077
+
3078
+ def visit_ExprNode(self, node):
3079
+ if not self._in_fused_function and node.type and node.type.is_fused:
3080
+ error(node.pos, "Invalid use of fused types, type cannot be specialized")
3081
+ # Errors in subtrees are likely related, so do not recurse.
3082
+ else:
3083
+ self.visitchildren(node)
3084
+
3085
+
3086
+ class ExpandInplaceOperators(EnvTransform):
3087
+ """Expand in-place operators into separate read/write operations.
3088
+ """
3089
+
3090
+ def visit_InPlaceAssignmentNode(self, node):
3091
+ lhs = node.lhs
3092
+ rhs = node.rhs
3093
+ if lhs.type.is_cpp_class:
3094
+ # No getting around this exact operator here.
3095
+ return node
3096
+ if isinstance(lhs, ExprNodes.BufferIndexNode):
3097
+ # There is code to handle this case in InPlaceAssignmentNode
3098
+ return node
3099
+
3100
+ env = self.current_env()
3101
+ def side_effect_free_reference(node, setting=False):
3102
+ if node.is_name:
3103
+ return node, []
3104
+ elif node.type.is_pyobject and not setting:
3105
+ node = LetRefNode(node)
3106
+ return node, [node]
3107
+ elif node.is_subscript:
3108
+ base, temps = side_effect_free_reference(node.base)
3109
+ index = LetRefNode(node.index)
3110
+ return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
3111
+ elif node.is_attribute:
3112
+ obj, temps = side_effect_free_reference(node.obj, setting=setting)
3113
+ return ExprNodes.AttributeNode.from_node(
3114
+ node, obj=obj, attribute=node.attribute, constant_result=ExprNodes.not_a_constant), temps
3115
+ elif isinstance(node, ExprNodes.BufferIndexNode):
3116
+ raise ValueError("Don't allow things like attributes of buffer indexing operations")
3117
+ else:
3118
+ node = LetRefNode(node)
3119
+ return node, [node]
3120
+
3121
+ try:
3122
+ lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
3123
+ except ValueError:
3124
+ return node
3125
+
3126
+ dup = lhs.__class__(**lhs.__dict__)
3127
+ dup = dup.analyse_types(env) # FIXME: no need to reanalyse the copy, right?
3128
+ binop = ExprNodes.binop_node(node.pos,
3129
+ operator = node.operator,
3130
+ operand1 = dup,
3131
+ operand2 = rhs,
3132
+ inplace=True)
3133
+
3134
+ # Manually analyse types for new node.
3135
+ lhs.is_target = True
3136
+ lhs = lhs.analyse_target_types(env)
3137
+ binop.analyse_operation(env)
3138
+ node = Nodes.SingleAssignmentNode(
3139
+ node.pos,
3140
+ lhs = lhs,
3141
+ rhs=binop.coerce_to(lhs.type, env))
3142
+ # Use LetRefNode to avoid side effects.
3143
+ let_ref_nodes.reverse()
3144
+ for t in let_ref_nodes:
3145
+ node = LetNode(t, node)
3146
+ return node
3147
+
3148
+ def visit_ExprNode(self, node):
3149
+ # In-place assignments can't happen within an expression.
3150
+ return node
3151
+
3152
+
3153
+ class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
3154
+ """
3155
+ Adjust function and class definitions by the decorator directives:
3156
+
3157
+ @cython.cfunc
3158
+ @cython.cclass
3159
+ @cython.ccall
3160
+ @cython.inline
3161
+ @cython.nogil
3162
+ @cython.critical_section
3163
+ """
3164
+ # list of directives that cause conversion to cclass
3165
+ converts_to_cclass = ('cclass', 'total_ordering', 'dataclasses.dataclass')
3166
+
3167
+ def visit_ModuleNode(self, node):
3168
+ self.directives = node.directives
3169
+ self.in_py_class = False
3170
+ self.visitchildren(node)
3171
+ return node
3172
+
3173
+ def visit_CompilerDirectivesMixin(self, node):
3174
+ with node.apply_directives(self):
3175
+ self.visitchildren(node)
3176
+ return node
3177
+
3178
+ def visit_DefNode(self, node):
3179
+ modifiers = []
3180
+ if 'inline' in self.directives:
3181
+ modifiers.append('inline')
3182
+ nogil = self.directives.get('nogil')
3183
+ with_gil = self.directives.get('with_gil')
3184
+ except_val = self.directives.get('exceptval')
3185
+ has_explicit_exc_clause = False if except_val is None else True
3186
+ return_type_node = self.directives.get('returns')
3187
+ if return_type_node is None and self.directives['annotation_typing']:
3188
+ return_type_node = node.return_type_annotation
3189
+ # for Python annotations, prefer safe exception handling by default
3190
+ if return_type_node is not None and except_val is None:
3191
+ except_val = (None, True) # except *
3192
+ elif except_val is None:
3193
+ # backward compatible default: no exception check, unless there's also a "@returns" declaration
3194
+ except_val = (None, True if return_type_node else False)
3195
+ if self.directives.get('c_compile_guard') and 'cfunc' not in self.directives:
3196
+ error(node.pos, "c_compile_guard only allowed on C functions")
3197
+ if 'ccall' in self.directives:
3198
+ if 'cfunc' in self.directives:
3199
+ error(node.pos, "cfunc and ccall directives cannot be combined")
3200
+ if with_gil:
3201
+ error(node.pos, "ccall functions cannot be declared 'with_gil'")
3202
+ node = node.as_cfunction(
3203
+ overridable=True, modifiers=modifiers, nogil=nogil,
3204
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3205
+ return self.visit(node)
3206
+ if 'cfunc' in self.directives:
3207
+ if self.in_py_class:
3208
+ error(node.pos, "cfunc directive is not allowed here")
3209
+ else:
3210
+ node = node.as_cfunction(
3211
+ overridable=False, modifiers=modifiers, nogil=nogil, with_gil=with_gil,
3212
+ returns=return_type_node, except_val=except_val, has_explicit_exc_clause=has_explicit_exc_clause)
3213
+ return self.visit(node)
3214
+ if 'inline' in modifiers:
3215
+ error(node.pos, "Python functions cannot be declared 'inline'")
3216
+ if nogil:
3217
+ # TODO: turn this into a "with gil" declaration.
3218
+ error(node.pos, "Python functions cannot be declared 'nogil'")
3219
+ if with_gil:
3220
+ error(node.pos, "Python functions cannot be declared 'with_gil'")
3221
+ self.visit_FuncDefNode(node)
3222
+ return node
3223
+
3224
+ def visit_FuncDefNode(self, node):
3225
+ if "critical_section" in self.directives:
3226
+ value = self.directives["critical_section"]
3227
+ if value is not None:
3228
+ error(node.pos, "critical_section decorator does not take arguments")
3229
+ if self.in_py_class:
3230
+ warning(
3231
+ node.pos,
3232
+ "@critical_section on method of a class that is not an extension type is unlikely to be useful",
3233
+ 2)
3234
+ new_body = Nodes.CriticalSectionStatNode(
3235
+ node.pos,
3236
+ args=[ExprNodes.FirstArgumentForCriticalSectionNode(node.pos, func_node=node)],
3237
+ body=node.body
3238
+ )
3239
+ node.body = new_body
3240
+ self.visitchildren(node)
3241
+ return node
3242
+
3243
+ def visit_LambdaNode(self, node):
3244
+ # No directives should modify lambdas or generator expressions (and also nothing in them).
3245
+ return node
3246
+
3247
+ def visit_PyClassDefNode(self, node):
3248
+ if any(directive in self.directives for directive in self.converts_to_cclass):
3249
+ node = node.as_cclass()
3250
+ return self.visit(node)
3251
+ else:
3252
+ old_in_pyclass = self.in_py_class
3253
+ self.in_py_class = True
3254
+ self.visitchildren(node)
3255
+ self.in_py_class = old_in_pyclass
3256
+ return node
3257
+
3258
+ def visit_CClassDefNode(self, node):
3259
+ old_in_pyclass = self.in_py_class
3260
+ self.in_py_class = False
3261
+ self.visitchildren(node)
3262
+ self.in_py_class = old_in_pyclass
3263
+ return node
3264
+
3265
+
3266
+ class AlignFunctionDefinitions(CythonTransform):
3267
+ """
3268
+ This class takes the signatures from a .pxd file and applies them to
3269
+ the def methods in a .py file.
3270
+ """
3271
+
3272
+ def visit_ModuleNode(self, node):
3273
+ self.scope = node.scope
3274
+ self.visitchildren(node)
3275
+ return node
3276
+
3277
+ def visit_PyClassDefNode(self, node):
3278
+ pxd_def = self.scope.lookup(node.name)
3279
+ if pxd_def:
3280
+ if pxd_def.is_cclass:
3281
+ return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
3282
+ elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
3283
+ error(node.pos, "'%s' redeclared" % node.name)
3284
+ if pxd_def.pos:
3285
+ error(pxd_def.pos, "previous declaration here")
3286
+ return None
3287
+ return node
3288
+
3289
+ def visit_CClassDefNode(self, node, pxd_def=None):
3290
+ if pxd_def is None:
3291
+ pxd_def = self.scope.lookup(node.class_name)
3292
+ if pxd_def:
3293
+ if not pxd_def.defined_in_pxd:
3294
+ return node
3295
+ outer_scope = self.scope
3296
+ self.scope = pxd_def.type.scope
3297
+ self.visitchildren(node)
3298
+ if pxd_def:
3299
+ self.scope = outer_scope
3300
+ return node
3301
+
3302
+ def visit_DefNode(self, node):
3303
+ pxd_def = self.scope.lookup(node.name)
3304
+ if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
3305
+ if not pxd_def.is_cfunction:
3306
+ error(node.pos, "'%s' redeclared" % node.name)
3307
+ if pxd_def.pos:
3308
+ error(pxd_def.pos, "previous declaration here")
3309
+ return None
3310
+ node = node.as_cfunction(pxd_def)
3311
+ # Enable this when nested cdef functions are allowed.
3312
+ # self.visitchildren(node)
3313
+ return node
3314
+
3315
+ def visit_ExprNode(self, node):
3316
+ # ignore lambdas and everything else that appears in expressions
3317
+ return node
3318
+
3319
+
3320
+ class AutoCpdefFunctionDefinitions(CythonTransform):
3321
+
3322
+ def visit_ModuleNode(self, node):
3323
+ self.directives = node.directives
3324
+ self.imported_names = set() # hack, see visit_FromImportStatNode()
3325
+ self.scope = node.scope
3326
+ self.visitchildren(node)
3327
+ return node
3328
+
3329
+ def visit_DefNode(self, node):
3330
+ if (self.scope.is_module_scope and self.directives['auto_cpdef']
3331
+ and node.name not in self.imported_names
3332
+ and node.is_cdef_func_compatible()):
3333
+ # FIXME: cpdef-ing should be done in analyse_declarations()
3334
+ node = node.as_cfunction(scope=self.scope)
3335
+ return node
3336
+
3337
+ def visit_CClassDefNode(self, node, pxd_def=None):
3338
+ if pxd_def is None:
3339
+ pxd_def = self.scope.lookup(node.class_name)
3340
+ if pxd_def:
3341
+ if not pxd_def.defined_in_pxd:
3342
+ return node
3343
+ outer_scope = self.scope
3344
+ self.scope = pxd_def.type.scope
3345
+ self.visitchildren(node)
3346
+ if pxd_def:
3347
+ self.scope = outer_scope
3348
+ return node
3349
+
3350
+ def visit_FromImportStatNode(self, node):
3351
+ # hack to prevent conditional import fallback functions from
3352
+ # being cdpef-ed (global Python variables currently conflict
3353
+ # with imports)
3354
+ if self.scope.is_module_scope:
3355
+ for name, _ in node.items:
3356
+ self.imported_names.add(name)
3357
+ return node
3358
+
3359
+ def visit_ExprNode(self, node):
3360
+ # ignore lambdas and everything else that appears in expressions
3361
+ return node
3362
+
3363
+
3364
+ class RemoveUnreachableCode(CythonTransform):
3365
+
3366
+ def visit_StatListNode(self, node):
3367
+ if not self.current_directives['remove_unreachable']:
3368
+ return node
3369
+ self.visitchildren(node)
3370
+ if len(node.stats) == 1 and isinstance(node.stats[0], Nodes.StatListNode) and not node.stats[0].stats:
3371
+ del node.stats[:]
3372
+ for idx, stat in enumerate(node.stats, 1):
3373
+ if stat.is_terminator:
3374
+ if idx < len(node.stats):
3375
+ if self.current_directives['warn.unreachable']:
3376
+ warning(node.stats[idx].pos, "Unreachable code", 2)
3377
+ node.stats = node.stats[:idx]
3378
+ node.is_terminator = True
3379
+ break
3380
+ return node
3381
+
3382
+ def visit_IfClauseNode(self, node):
3383
+ self.visitchildren(node)
3384
+ if node.body.is_terminator:
3385
+ node.is_terminator = True
3386
+ return node
3387
+
3388
+ def visit_IfStatNode(self, node):
3389
+ self.visitchildren(node)
3390
+ if node.else_clause and node.else_clause.is_terminator:
3391
+ for clause in node.if_clauses:
3392
+ if not clause.is_terminator:
3393
+ break
3394
+ else:
3395
+ node.is_terminator = True
3396
+ return node
3397
+
3398
+ def visit_TryExceptStatNode(self, node):
3399
+ self.visitchildren(node)
3400
+ if node.body.is_terminator and node.else_clause:
3401
+ if self.current_directives['warn.unreachable']:
3402
+ warning(node.else_clause.pos, "Unreachable code", 2)
3403
+ node.else_clause = None
3404
+ return node
3405
+
3406
+ def visit_TryFinallyStatNode(self, node):
3407
+ self.visitchildren(node)
3408
+ if node.finally_clause.is_terminator:
3409
+ node.is_terminator = True
3410
+ return node
3411
+
3412
+ def visit_PassStatNode(self, node):
3413
+ """Eliminate useless PassStatNode"""
3414
+ # 'pass' statements often appear in a separate line and must be traced.
3415
+ if not self.current_directives['linetrace']:
3416
+ node = Nodes.StatListNode(pos=node.pos, stats=[])
3417
+ return node
3418
+
3419
+
3420
+ class YieldNodeCollector(TreeVisitor):
3421
+
3422
+ def __init__(self, excludes=[]):
3423
+ super().__init__()
3424
+ self.yields = []
3425
+ self.returns = []
3426
+ self.finallys = []
3427
+ self.excepts = []
3428
+ self.has_return_value = False
3429
+ self.has_yield = False
3430
+ self.has_await = False
3431
+ self.excludes = excludes
3432
+
3433
+ def visit_Node(self, node):
3434
+ if node not in self.excludes:
3435
+ self.visitchildren(node)
3436
+
3437
+ def visit_YieldExprNode(self, node):
3438
+ self.yields.append(node)
3439
+ self.has_yield = True
3440
+ self.visitchildren(node)
3441
+
3442
+ def visit_AwaitExprNode(self, node):
3443
+ self.yields.append(node)
3444
+ self.has_await = True
3445
+ self.visitchildren(node)
3446
+
3447
+ def visit_ReturnStatNode(self, node):
3448
+ self.visitchildren(node)
3449
+ if node.value:
3450
+ self.has_return_value = True
3451
+ self.returns.append(node)
3452
+
3453
+ def visit_TryFinallyStatNode(self, node):
3454
+ self.visitchildren(node)
3455
+ self.finallys.append(node)
3456
+
3457
+ def visit_TryExceptStatNode(self, node):
3458
+ self.visitchildren(node)
3459
+ self.excepts.append(node)
3460
+
3461
+ def visit_ClassDefNode(self, node):
3462
+ pass
3463
+
3464
+ def visit_FuncDefNode(self, node):
3465
+ pass
3466
+
3467
+ def visit_LambdaNode(self, node):
3468
+ pass
3469
+
3470
+ def visit_GeneratorExpressionNode(self, node):
3471
+ # node.loop iterator is evaluated outside the generator expression
3472
+ if isinstance(node.loop, Nodes._ForInStatNode):
3473
+ # Possibly should handle ForFromStatNode
3474
+ # but for now do nothing
3475
+ self.visit(node.loop.iterator)
3476
+
3477
+ def visit_CArgDeclNode(self, node):
3478
+ # do not look into annotations
3479
+ # FIXME: support (yield) in default arguments (currently crashes)
3480
+ pass
3481
+
3482
+
3483
+ class MarkClosureVisitor(CythonTransform):
3484
+ # In addition to marking closures this is also responsible to finding parts of the
3485
+ # generator iterable and marking them
3486
+
3487
+ def visit_ModuleNode(self, node):
3488
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.NO_CLOSURE
3489
+ self.excludes = []
3490
+ self.visitchildren(node)
3491
+ return node
3492
+
3493
+ def visit_FuncDefNode(self, node):
3494
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.NO_CLOSURE
3495
+ self.visitchildren(node)
3496
+ node.needs_closure = self.needs_closure
3497
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.FULL_CLOSURE
3498
+
3499
+ collector = YieldNodeCollector(self.excludes)
3500
+ collector.visitchildren(node)
3501
+
3502
+ if node.is_async_def:
3503
+ coroutine_type = Nodes.AsyncDefNode
3504
+ if collector.has_yield:
3505
+ coroutine_type = Nodes.AsyncGenNode
3506
+ for yield_expr in collector.yields + collector.returns:
3507
+ yield_expr.in_async_gen = True
3508
+ elif self.current_directives['iterable_coroutine']:
3509
+ coroutine_type = Nodes.IterableAsyncDefNode
3510
+ elif collector.has_await:
3511
+ found = next(y for y in collector.yields if y.is_await)
3512
+ error(found.pos, "'await' not allowed in generators (use 'yield')")
3513
+ return node
3514
+ elif collector.has_yield:
3515
+ coroutine_type = Nodes.GeneratorDefNode
3516
+ else:
3517
+ return node
3518
+
3519
+ for i, yield_expr in enumerate(collector.yields, 1):
3520
+ yield_expr.label_num = i
3521
+ for retnode in collector.returns + collector.finallys + collector.excepts:
3522
+ retnode.in_generator = True
3523
+
3524
+ gbody = Nodes.GeneratorBodyDefNode(
3525
+ pos=node.pos, name=node.name, body=node.body,
3526
+ is_coroutine_body=node.is_async_def,
3527
+ is_async_gen_body=node.is_async_def and collector.has_yield)
3528
+ coroutine = coroutine_type(
3529
+ pos=node.pos, name=node.name, args=node.args,
3530
+ star_arg=node.star_arg, starstar_arg=node.starstar_arg,
3531
+ doc=node.doc, decorators=node.decorators,
3532
+ gbody=gbody, lambda_name=node.lambda_name,
3533
+ return_type_annotation=node.return_type_annotation,
3534
+ is_generator_expression=node.is_generator_expression)
3535
+ if node.needs_closure:
3536
+ # We may have determined that we need a "full closure"
3537
+ # so upgrade the coroutine to signal that
3538
+ coroutine.needs_closure = node.needs_closure
3539
+ return coroutine
3540
+
3541
+ def visit_CFuncDefNode(self, node):
3542
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.NO_CLOSURE
3543
+ self.visitchildren(node)
3544
+ node.needs_closure = self.needs_closure
3545
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.FULL_CLOSURE
3546
+ if node.needs_closure and node.overridable:
3547
+ error(node.pos, "closures inside cpdef functions not yet supported")
3548
+ return node
3549
+
3550
+ def visit_LambdaNode(self, node):
3551
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.NO_CLOSURE
3552
+ self.visitchildren(node)
3553
+ node.needs_closure = self.needs_closure
3554
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.FULL_CLOSURE
3555
+ return node
3556
+
3557
+ def visit_ClassDefNode(self, node):
3558
+ self.visitchildren(node)
3559
+ self.needs_closure = Nodes.FuncDefNode.NeedsClosure.FULL_CLOSURE
3560
+ return node
3561
+
3562
+ def visit_GeneratorExpressionNode(self, node):
3563
+ excludes = self.excludes
3564
+ if isinstance(node.loop, Nodes._ForInStatNode):
3565
+ self.excludes = [node.loop.iterator]
3566
+ node = self.visit_LambdaNode(node)
3567
+ self.excludes = excludes
3568
+ if not isinstance(node.loop, Nodes._ForInStatNode):
3569
+ # Possibly should handle ForFromStatNode
3570
+ # but for now do nothing
3571
+ return node
3572
+ itseq = node.loop.iterator.sequence
3573
+ # literals do not need replacing with an argument
3574
+ if itseq.is_literal:
3575
+ return node
3576
+ _GeneratorExpressionArgumentsMarker(node).visit(itseq)
3577
+ return node
3578
+
3579
+
3580
+ class CreateClosureClasses(CythonTransform):
3581
+ # Output closure classes in module scope for all functions
3582
+ # that really need it.
3583
+
3584
+ def __init__(self, context):
3585
+ super().__init__(context)
3586
+ self.path = []
3587
+ self.in_lambda = False
3588
+
3589
+ def visit_ModuleNode(self, node):
3590
+ self.module_scope = node.scope
3591
+ self.visitchildren(node)
3592
+ return node
3593
+
3594
+ def find_entries_used_in_closures(self, node):
3595
+ from_closure = []
3596
+ in_closure = []
3597
+ for scope in node.local_scope.iter_local_scopes():
3598
+ for name, entry in scope.entries.items():
3599
+ if not name:
3600
+ continue
3601
+ if entry.from_closure:
3602
+ from_closure.append((name, entry))
3603
+ elif entry.in_closure:
3604
+ in_closure.append((name, entry))
3605
+ return from_closure, in_closure
3606
+
3607
+ def create_class_from_scope(self, node, target_module_scope, inner_node=None):
3608
+ # move local variables into closure
3609
+ if node.is_generator:
3610
+ for scope in node.local_scope.iter_local_scopes():
3611
+ for entry in scope.entries.values():
3612
+ if not (entry.from_closure or entry.is_pyglobal or entry.is_cglobal):
3613
+ entry.in_closure = True
3614
+
3615
+ from_closure, in_closure = self.find_entries_used_in_closures(node)
3616
+ in_closure.sort()
3617
+
3618
+ # Now from the beginning
3619
+ node.needs_closure = Nodes.FuncDefNode.NeedsClosure.NO_CLOSURE
3620
+ node.needs_outer_scope = False
3621
+
3622
+ func_scope = node.local_scope
3623
+ cscope = node.entry.scope
3624
+ while cscope.is_py_class_scope or cscope.is_c_class_scope:
3625
+ cscope = cscope.outer_scope
3626
+
3627
+ if not from_closure and (self.path or inner_node):
3628
+ if not inner_node:
3629
+ if not node.py_cfunc_node:
3630
+ raise InternalError("DefNode does not have assignment node")
3631
+ inner_node = node.py_cfunc_node
3632
+ inner_node.needs_closure_code = False
3633
+ node.needs_outer_scope = False
3634
+
3635
+ if node.is_generator:
3636
+ pass
3637
+ elif not in_closure and not from_closure:
3638
+ return
3639
+ elif not in_closure:
3640
+ func_scope.is_passthrough = True
3641
+ func_scope.scope_class = cscope.scope_class
3642
+ node.needs_outer_scope = True
3643
+ return
3644
+
3645
+ # entry.cname can contain periods (eg. a derived C method of a class).
3646
+ # We want to use the cname as part of a C struct name, so we replace
3647
+ # periods with double underscores.
3648
+ as_name = '%s_%s' % (
3649
+ target_module_scope.next_id(Naming.closure_class_prefix),
3650
+ node.entry.cname.replace('.','__'))
3651
+ as_name = EncodedString(as_name)
3652
+
3653
+ entry = target_module_scope.declare_c_class(
3654
+ name=as_name, pos=node.pos, defining=True,
3655
+ implementing=True)
3656
+ entry.type.is_final_type = True
3657
+
3658
+ func_scope.scope_class = entry
3659
+ class_scope = entry.type.scope
3660
+ class_scope.is_internal = True
3661
+ class_scope.is_closure_class_scope = True
3662
+ if node.is_async_def or node.is_generator:
3663
+ # Generators need their closure intact during cleanup as they resume to handle GeneratorExit
3664
+ class_scope.directives['no_gc_clear'] = True
3665
+ if Options.closure_freelist_size:
3666
+ class_scope.directives['freelist'] = Options.closure_freelist_size
3667
+
3668
+ if from_closure:
3669
+ assert cscope.is_closure_scope
3670
+ class_scope.declare_var(pos=node.pos,
3671
+ name=Naming.outer_scope_cname,
3672
+ cname=Naming.outer_scope_cname,
3673
+ type=cscope.scope_class.type,
3674
+ is_cdef=True)
3675
+ node.needs_outer_scope = True
3676
+ for name, entry in in_closure:
3677
+ closure_entry = class_scope.declare_var(
3678
+ pos=entry.pos,
3679
+ name=entry.name if not entry.in_subscope else None,
3680
+ cname=entry.cname,
3681
+ type=entry.type,
3682
+ is_cdef=True)
3683
+ if entry.is_declared_generic:
3684
+ closure_entry.is_declared_generic = 1
3685
+ node.needs_closure = Nodes.FuncDefNode.NeedsClosure.FULL_CLOSURE
3686
+ # Do it here because other classes are already checked
3687
+ target_module_scope.check_c_class(func_scope.scope_class)
3688
+
3689
+ def visit_LambdaNode(self, node):
3690
+ if not isinstance(node.def_node, Nodes.DefNode):
3691
+ # fused function, an error has been previously issued
3692
+ return node
3693
+
3694
+ was_in_lambda = self.in_lambda
3695
+ self.in_lambda = True
3696
+ self.create_class_from_scope(node.def_node, self.module_scope, node)
3697
+ self.visitchildren(node)
3698
+ self.in_lambda = was_in_lambda
3699
+ return node
3700
+
3701
+ def visit_FuncDefNode(self, node):
3702
+ if self.in_lambda:
3703
+ self.visitchildren(node)
3704
+ return node
3705
+ if node.needs_closure or self.path:
3706
+ self.create_class_from_scope(node, self.module_scope)
3707
+ self.path.append(node)
3708
+ self.visitchildren(node)
3709
+ self.path.pop()
3710
+ return node
3711
+
3712
+ def visit_GeneratorBodyDefNode(self, node):
3713
+ self.visitchildren(node)
3714
+ return node
3715
+
3716
+ def visit_CFuncDefNode(self, node):
3717
+ if not node.overridable:
3718
+ return self.visit_FuncDefNode(node)
3719
+ else:
3720
+ self.visitchildren(node)
3721
+ return node
3722
+
3723
+ def visit_GeneratorExpressionNode(self, node):
3724
+ node = _HandleGeneratorArguments()(node)
3725
+ return self.visit_LambdaNode(node)
3726
+
3727
+
3728
+ class InjectGilHandling(VisitorTransform, SkipDeclarations):
3729
+ """
3730
+ Allow certain Python operations inside of nogil blocks by implicitly acquiring the GIL.
3731
+
3732
+ Must run before the AnalyseDeclarationsTransform to make sure the GILStatNodes get
3733
+ set up, parallel sections know that the GIL is acquired inside of them, etc.
3734
+ """
3735
+ nogil = False
3736
+
3737
+ # special node handling
3738
+
3739
+ def _inject_gil_in_nogil(self, node):
3740
+ """Allow the (Python statement) node in nogil sections by wrapping it in a 'with gil' block."""
3741
+ if self.nogil:
3742
+ node = Nodes.GILStatNode(node.pos, state='gil', body=node, internally_generated=True)
3743
+ return node
3744
+
3745
+ visit_RaiseStatNode = _inject_gil_in_nogil
3746
+ visit_PrintStatNode = _inject_gil_in_nogil # sadly, not the function
3747
+
3748
+ # further candidates:
3749
+ # def visit_ReraiseStatNode(self, node):
3750
+
3751
+ # nogil tracking
3752
+
3753
+ def visit_GILStatNode(self, node):
3754
+ was_nogil = self.nogil
3755
+ self.nogil = (node.state == 'nogil')
3756
+ self.visitchildren(node)
3757
+ self.nogil = was_nogil
3758
+ return node
3759
+
3760
+ def visit_CFuncDefNode(self, node):
3761
+ was_nogil = self.nogil
3762
+ if isinstance(node.declarator, Nodes.CFuncDeclaratorNode):
3763
+ self.nogil = node.declarator.nogil and not node.declarator.with_gil
3764
+ self.visitchildren(node)
3765
+ self.nogil = was_nogil
3766
+ return node
3767
+
3768
+ def visit_ParallelRangeNode(self, node):
3769
+ was_nogil = self.nogil
3770
+ self.nogil = node.nogil
3771
+ self.visitchildren(node)
3772
+ self.nogil = was_nogil
3773
+ return node
3774
+
3775
+ def visit_ExprNode(self, node):
3776
+ # No special GIL handling inside of expressions for now.
3777
+ return node
3778
+
3779
+ visit_Node = VisitorTransform.recurse_to_children
3780
+
3781
+
3782
+ class GilCheck(VisitorTransform):
3783
+ """
3784
+ Call `node.gil_check(env)` on each node to make sure we hold the
3785
+ GIL when we need it. Raise an error when on Python operations
3786
+ inside a `nogil` environment.
3787
+
3788
+ Additionally, raise exceptions for closely nested with gil or with nogil
3789
+ statements. The latter would abort Python.
3790
+ """
3791
+
3792
+ def __call__(self, root):
3793
+ self.env_stack = [root.scope]
3794
+ self.nogil_state = Nodes.NoGilState.HasGil
3795
+ self.in_critical_section = False
3796
+
3797
+ self.nogil_state_at_current_gilstatnode = Nodes.NoGilState.HasGil
3798
+ return super().__call__(root)
3799
+
3800
+ def _visit_scoped_children(self, node, nogil_state):
3801
+ was_nogil = self.nogil_state
3802
+ outer_attrs = node.outer_attrs
3803
+ if outer_attrs and len(self.env_stack) > 1:
3804
+ self.nogil_state = (
3805
+ Nodes.NoGilState.NoGil if self.env_stack[-2].nogil else Nodes.NoGilState.HasGil)
3806
+ self.visitchildren(node, outer_attrs)
3807
+
3808
+ self.nogil_state = nogil_state
3809
+ self.visitchildren(node, attrs=None, exclude=outer_attrs)
3810
+ self.nogil_state = was_nogil
3811
+
3812
+ def visit_FuncDefNode(self, node):
3813
+ self.env_stack.append(node.local_scope)
3814
+ inner_nogil = node.local_scope.nogil
3815
+
3816
+ nogil_state = self.nogil_state
3817
+ if inner_nogil:
3818
+ self.nogil_state = Nodes.NoGilState.NoGilScope
3819
+
3820
+ if inner_nogil and node.nogil_check:
3821
+ node.nogil_check(node.local_scope)
3822
+
3823
+ in_critical_section, self.in_critical_section = self.in_critical_section, False
3824
+ self._visit_scoped_children(node, self.nogil_state)
3825
+ self.in_critical_section = in_critical_section
3826
+
3827
+ # FuncDefNodes can be nested, because a cpdef function contains a def function
3828
+ # inside it. Therefore restore to previous state
3829
+ self.nogil_state = nogil_state
3830
+
3831
+ self.env_stack.pop()
3832
+ return node
3833
+
3834
+ def visit_GILStatNode(self, node):
3835
+ if node.condition is not None:
3836
+ error(node.condition.pos,
3837
+ "Non-constant condition in a "
3838
+ "`with %s(<condition>)` statement" % node.state)
3839
+ return node
3840
+
3841
+ if self.nogil_state and node.nogil_check:
3842
+ node.nogil_check()
3843
+
3844
+ was_nogil = self.nogil_state
3845
+ is_nogil = (node.state == 'nogil')
3846
+
3847
+ if was_nogil == is_nogil and not self.nogil_state == Nodes.NoGilState.NoGilScope:
3848
+ if not was_nogil:
3849
+ error(node.pos, "Trying to acquire the GIL while it is "
3850
+ "already held.")
3851
+ else:
3852
+ error(node.pos, "Trying to release the GIL while it was "
3853
+ "previously released.")
3854
+ if self.nogil_state == Nodes.NoGilState.NoGilScope:
3855
+ node.scope_gil_state_known = False
3856
+
3857
+ if isinstance(node.finally_clause, Nodes.StatListNode):
3858
+ # The finally clause of the GILStatNode is a GILExitNode,
3859
+ # which is wrapped in a StatListNode. Just unpack that.
3860
+ node.finally_clause, = node.finally_clause.stats
3861
+
3862
+ nogil_state_at_current_gilstatnode = self.nogil_state_at_current_gilstatnode
3863
+ self.nogil_state_at_current_gilstatnode = self.nogil_state
3864
+ nogil_state = Nodes.NoGilState.NoGil if is_nogil else Nodes.NoGilState.HasGil
3865
+ self._visit_scoped_children(node, nogil_state)
3866
+ self.nogil_state_at_current_gilstatnode = nogil_state_at_current_gilstatnode
3867
+
3868
+ # Drop pointless nested `with gil: with nogil:` blocks.
3869
+ # These can occur during the generation of prange/parallel sections for example.
3870
+ node_body = node.body
3871
+ if (isinstance(node_body, Nodes.StatListNode) and len(node_body.stats) == 1):
3872
+ node_body = node_body.stats[0]
3873
+ if isinstance(node_body, Nodes.GILStatNode) and node_body.state != node.state and (
3874
+ # Don't optimize out user-inserted `with nogil` for now. They have the
3875
+ # side-effect of deliberately allowing a thread-switch even if they might
3876
+ # appear useless, so they might be deliberate.
3877
+ (node_body.state == "nogil" and node_body.internally_generated) or
3878
+ (node.state == "nogil" and node.internally_generated)):
3879
+ if not node.scope_gil_state_known:
3880
+ node_body.scope_gil_state_known = False
3881
+ node_body.finally_clause.scope_gil_state_known = False
3882
+ if node_body.finally_except_clause is not None:
3883
+ node_body.finally_except_clause.scope_gil_state_known = False
3884
+ return node_body
3885
+ else:
3886
+ return node_body.body
3887
+
3888
+ return node
3889
+
3890
+ def visit_ParallelRangeNode(self, node):
3891
+ if self.nogil_state == Nodes.NoGilState.HasGil and not node.nogil:
3892
+ # We still release the GIL then reacquire it to avoid deadlocks.
3893
+ node.acquire_gil = True
3894
+ if (node.nogil or self.nogil_state == Nodes.NoGilState.NoGilScope or
3895
+ (self.nogil_state == Nodes.NoGilState.HasGil and not node.parent)):
3896
+ node_was_nogil, node.nogil = node.nogil, False
3897
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node, internally_generated=True)
3898
+ if not node_was_nogil and self.nogil_state == Nodes.NoGilState.NoGilScope:
3899
+ # We're in a "nogil" function, but that doesn't prove we
3900
+ # didn't have the gil
3901
+ node.scope_gil_state_known = False
3902
+ return self.visit_GILStatNode(node)
3903
+
3904
+ if node.acquire_gil:
3905
+ if not self.env_stack[-1].directives['freethreading_compatible']:
3906
+ warning(
3907
+ node.pos,
3908
+ "prange without releasing the GIL will only work well on freethreaded Python",
3909
+ level=2)
3910
+ was_nogil = self.nogil_state
3911
+ self.nogil_state = Nodes.NoGilState.HasGil
3912
+
3913
+ node.nogil_check(self.env_stack[-1])
3914
+ self.visitchildren(node)
3915
+
3916
+ if node.acquire_gil:
3917
+ self.nogil_state = was_nogil
3918
+ return node
3919
+
3920
+ def visit_ParallelWithBlockNode(self, node):
3921
+ if self.nogil_state != Nodes.NoGilState.NoGil:
3922
+ # Ensure that the GIL is released
3923
+ if self.nogil_state == Nodes.NoGilState.HasGil:
3924
+ # Even if we intend the block to have the GIL it's easier to release
3925
+ # and reacquire it to void deadlocks.
3926
+ node.acquire_gil = True
3927
+ node = Nodes.GILStatNode(node.pos, state='nogil', body=node, internally_generated=True)
3928
+ if self.nogil_state == Nodes.NoGilState.NoGilScope:
3929
+ node.scope_gil_state_known = False
3930
+ return self.visit_GILStatNode(node)
3931
+
3932
+ if node.acquire_gil:
3933
+ if not self.env_stack[-1].directives['freethreading_compatible']:
3934
+ warning(
3935
+ node.pos,
3936
+ "Parallel section without releasing the GIL will only work well on freethreaded Python",
3937
+ level=2)
3938
+ was_nogil = self.nogil_state
3939
+ self.nogil_state = Nodes.NoGilState.HasGil
3940
+
3941
+ if node.nogil_check:
3942
+ # It does not currently implement this, but test for it anyway to
3943
+ # avoid potential future surprises
3944
+ node.nogil_check(self.env_stack[-1])
3945
+
3946
+ self.visitchildren(node)
3947
+ if node.acquire_gil:
3948
+ self.nogil_state = was_nogil
3949
+ return node
3950
+
3951
+ def visit_TryFinallyStatNode(self, node):
3952
+ """
3953
+ Take care of try/finally statements in nogil code sections.
3954
+ """
3955
+ if not self.nogil_state:
3956
+ return self.visit_Node(node)
3957
+
3958
+ node.nogil_check = None
3959
+ node.is_try_finally_in_nogil = True
3960
+ self.visitchildren(node)
3961
+ return node
3962
+
3963
+ def visit_CriticalSectionStatNode(self, node):
3964
+ # skip normal "try/finally node" handling
3965
+ in_critical_section, self.in_critical_section = self.in_critical_section, True
3966
+ result = self.visit_Node(node)
3967
+ self.in_critical_section = in_critical_section
3968
+ return result
3969
+
3970
+ def visit_CythonLockStatNode(self, node):
3971
+ # skip normal "try/finally node" handling
3972
+ return self.visit_Node(node)
3973
+
3974
+ def visit_GILExitNode(self, node):
3975
+ if self.nogil_state_at_current_gilstatnode == Nodes.NoGilState.NoGilScope:
3976
+ node.scope_gil_state_known = False
3977
+ self.visitchildren(node)
3978
+ return node
3979
+
3980
+ def visit_Node(self, node):
3981
+ if self.env_stack and self.nogil_state and node.nogil_check:
3982
+ node.nogil_check(self.env_stack[-1])
3983
+ if node.outer_attrs:
3984
+ self._visit_scoped_children(node, self.nogil_state)
3985
+ else:
3986
+ self.visitchildren(node)
3987
+ if self.nogil_state:
3988
+ node.in_nogil_context = self.nogil_state
3989
+ return node
3990
+
3991
+ def visit_SimpleCallNode(self, node):
3992
+ if (node.self and node.self.type.is_cython_lock_type and
3993
+ node.function.is_attribute and node.function.attribute == "acquire" and
3994
+ len(node.args) == 1):
3995
+ # For the cython lock types we can optimize if we know the GIL state.
3996
+ # (Remove this in the distant future when it's all PyMutexes because for these
3997
+ # it doesn't matter)
3998
+ suffix = None
3999
+ if self.nogil_state == Nodes.NoGilState.NoGil:
4000
+ suffix = "Nogil"
4001
+ elif self.nogil_state == Nodes.NoGilState.HasGil:
4002
+ suffix = "Gil"
4003
+ if suffix:
4004
+ node = ExprNodes.PythonCapiCallNode(
4005
+ node.pos,
4006
+ node.function.entry.cname + suffix,
4007
+ node.function.type,
4008
+ args=[node.self],
4009
+ )
4010
+ return self.visit_Node(node)
4011
+
4012
+ def visit_AttributeNode(self, node):
4013
+ if self.in_critical_section and node.is_py_attr:
4014
+ warning(
4015
+ node.pos,
4016
+ "Python attribute access is not usefully protected by critical_section",
4017
+ 1)
4018
+ return self.visit_Node(node)
4019
+
4020
+
4021
+ class CoerceCppTemps(EnvTransform, SkipDeclarations):
4022
+ """
4023
+ For temporary expression that are implemented using std::optional it's necessary the temps are
4024
+ assigned using `__pyx_t_x = value;` but accessed using `something = (*__pyx_t_x)`. This transform
4025
+ inserts a coercion node to take care of this, and runs absolutely last (once nothing else can be
4026
+ inserted into the tree)
4027
+
4028
+ TODO: a possible alternative would be to split ExprNode.result() into ExprNode.rhs_result() and ExprNode.lhs_result()???
4029
+ """
4030
+ def visit_ModuleNode(self, node):
4031
+ if self.current_env().cpp:
4032
+ # skipping this makes it essentially free for C files
4033
+ self.visitchildren(node)
4034
+ return node
4035
+
4036
+ def visit_ExprNode(self, node):
4037
+ self.visitchildren(node)
4038
+ if (self.current_env().directives['cpp_locals'] and
4039
+ node.result_in_temp() and node.type.is_cpp_class and
4040
+ # Fake references are not replaced with "std::optional()".
4041
+ not node.type.is_fake_reference):
4042
+ node = ExprNodes.CppOptionalTempCoercion(node)
4043
+
4044
+ return node
4045
+
4046
+ def visit_CppOptionalTempCoercion(self, node):
4047
+ return node
4048
+
4049
+ def visit_CppIteratorNode(self, node):
4050
+ return node
4051
+
4052
+ def visit_ExprStatNode(self, node):
4053
+ # Deliberately skip `expr` in ExprStatNode - we don't need to access it.
4054
+ self.visitchildren(node.expr)
4055
+ return node
4056
+
4057
+
4058
+ class TransformBuiltinMethods(EnvTransform):
4059
+ """
4060
+ Replace Cython's own cython.* builtins by the corresponding tree nodes.
4061
+ Also handle some Python special builtin functions (e.g. super()/locals())
4062
+ that require introspection by the compiler.
4063
+ """
4064
+ def __init__(self, *args, **kwds):
4065
+ super().__init__(*args, **kwds)
4066
+ self.def_node_body_insertions = {}
4067
+
4068
+ def visit_SingleAssignmentNode(self, node):
4069
+ if node.declaration_only:
4070
+ return None
4071
+ else:
4072
+ self.visitchildren(node)
4073
+ return node
4074
+
4075
+ def visit_AttributeNode(self, node):
4076
+ self.visitchildren(node)
4077
+ return self.visit_cython_attribute(node)
4078
+
4079
+ def visit_NameNode(self, node):
4080
+ if node.name == u'__class__':
4081
+ lenv = self.current_env()
4082
+ entry = lenv.lookup_here(u'__class__')
4083
+ if not entry:
4084
+ node = self._inject_class(node)
4085
+ return self.visit_cython_attribute(node)
4086
+
4087
+ def visit_cython_attribute(self, node):
4088
+ attribute = node.as_cython_attribute()
4089
+ if attribute:
4090
+ if attribute == '__version__':
4091
+ from .. import __version__ as version
4092
+ node = ExprNodes.UnicodeNode(node.pos, value=EncodedString(version))
4093
+ elif attribute == 'NULL':
4094
+ node = ExprNodes.NullNode(node.pos)
4095
+ elif attribute in ('set', 'frozenset', 'staticmethod'):
4096
+ node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
4097
+ entry=self.current_env().builtin_scope().lookup_here(attribute))
4098
+ elif PyrexTypes.parse_basic_type(attribute):
4099
+ pass
4100
+ elif self.context.cython_scope.lookup_qualified_name(attribute):
4101
+ pass
4102
+ else:
4103
+ error(node.pos, "'%s' not a valid cython attribute or is being used incorrectly" % attribute)
4104
+ return node
4105
+
4106
+ def visit_ExecStatNode(self, node):
4107
+ lenv = self.current_env()
4108
+ self.visitchildren(node)
4109
+ if len(node.args) == 1:
4110
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
4111
+ if not lenv.is_module_scope:
4112
+ node.args.append(
4113
+ ExprNodes.LocalsExprNode(
4114
+ node.pos, self.current_scope_node(), lenv))
4115
+ return node
4116
+
4117
+ def _inject_locals(self, node, func_name):
4118
+ # locals()/dir()/vars() builtins
4119
+ lenv = self.current_env()
4120
+ entry = lenv.lookup_here(func_name)
4121
+ if entry:
4122
+ # not the builtin
4123
+ return node
4124
+ pos = node.pos
4125
+ if func_name in ('locals', 'vars'):
4126
+ if func_name == 'locals' and len(node.args) > 0:
4127
+ error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
4128
+ % len(node.args))
4129
+ return node
4130
+ elif func_name == 'vars':
4131
+ if len(node.args) > 1:
4132
+ error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
4133
+ % len(node.args))
4134
+ if len(node.args) > 0:
4135
+ return node # nothing to do
4136
+ return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
4137
+ else: # dir()
4138
+ if len(node.args) > 1:
4139
+ error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
4140
+ % len(node.args))
4141
+ if len(node.args) > 0:
4142
+ # optimised in Builtin.py
4143
+ return node
4144
+ if lenv.is_py_class_scope or lenv.is_module_scope:
4145
+ if lenv.is_py_class_scope:
4146
+ pyclass = self.current_scope_node()
4147
+ locals_dict = ExprNodes.CloneNode(pyclass.dict)
4148
+ else:
4149
+ locals_dict = ExprNodes.GlobalsExprNode(pos)
4150
+ return ExprNodes.SortedDictKeysNode(locals_dict)
4151
+ local_names = sorted(var.name for var in lenv.entries.values() if var.name)
4152
+ items = [ExprNodes.IdentifierStringNode(pos, value=var)
4153
+ for var in local_names]
4154
+ return ExprNodes.ListNode(pos, args=items)
4155
+
4156
+ def visit_PrimaryCmpNode(self, node):
4157
+ # special case: for in/not-in test, we do not need to sort locals()
4158
+ self.visitchildren(node)
4159
+ if node.operator in 'not_in': # in/not_in
4160
+ if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
4161
+ arg = node.operand2.arg
4162
+ if isinstance(arg, ExprNodes.NoneCheckNode):
4163
+ arg = arg.arg
4164
+ node.operand2 = arg
4165
+ return node
4166
+
4167
+ def visit_CascadedCmpNode(self, node):
4168
+ return self.visit_PrimaryCmpNode(node)
4169
+
4170
+ def _inject_eval(self, node, func_name):
4171
+ lenv = self.current_env()
4172
+ entry = lenv.lookup(func_name)
4173
+ if len(node.args) != 1 or (entry and not entry.is_builtin):
4174
+ return node
4175
+ # Inject globals and locals
4176
+ node.args.append(ExprNodes.GlobalsExprNode(node.pos))
4177
+ if not lenv.is_module_scope:
4178
+ node.args.append(
4179
+ ExprNodes.LocalsExprNode(
4180
+ node.pos, self.current_scope_node(), lenv))
4181
+ return node
4182
+
4183
+ def _inject_class(self, node):
4184
+ # bare __class__ reference inside function
4185
+ current_def_node = self.current_scope_node()
4186
+
4187
+ if not isinstance(current_def_node, Nodes.FuncDefNode):
4188
+ return node
4189
+
4190
+ # Go up the stack, find the first class node and its direct method (i.e. function).
4191
+ fdef_node = class_node = generator_node = None
4192
+ for stack_node, stack_scope in reversed(self.env_stack):
4193
+ if isinstance(stack_node, Nodes.ClassDefNode):
4194
+ class_node = stack_node
4195
+ class_scope = stack_scope
4196
+ break
4197
+ elif isinstance(stack_node, Nodes.GeneratorDefNode):
4198
+ generator_node = stack_node
4199
+ fdef_node = stack_node.gbody
4200
+ fdef_scope = stack_scope
4201
+ elif isinstance(stack_node, Nodes.FuncDefNode):
4202
+ fdef_node = stack_node
4203
+ fdef_scope = stack_scope
4204
+
4205
+ if not fdef_node or not class_node:
4206
+ # failed to find a class or function
4207
+ return node
4208
+
4209
+ # now we arrange to inject:
4210
+ # __class__ = ... at the start of the def_node body
4211
+ # The advantage of doing it like this is that it automatically appears in locals()
4212
+ # and it can be captured by inner functions
4213
+ if fdef_node not in self.def_node_body_insertions:
4214
+ pos = fdef_node.body.pos
4215
+ if class_scope.is_c_class_scope:
4216
+ # c-classes can be resolved at compile-time, so they have a simpler
4217
+ # implementation
4218
+ rhs = ExprNodes.NameNode(
4219
+ pos, name=class_node.scope.name,
4220
+ entry=class_node.entry)
4221
+ elif class_scope.is_py_class_scope:
4222
+ rhs = ExprNodes.ClassCellNode(pos, is_generator=generator_node is not None)
4223
+ if generator_node:
4224
+ generator_node.requires_classobj = True
4225
+ else:
4226
+ fdef_node.requires_classobj = True
4227
+ class_node.class_cell.is_active = True
4228
+ else:
4229
+ return node # should never happen
4230
+
4231
+ assign_node = Nodes.SingleAssignmentNode(pos,
4232
+ lhs=ExprNodes.NameNode(pos, name=EncodedString("__class__")),
4233
+ rhs=rhs)
4234
+
4235
+ assign_node.analyse_declarations(fdef_scope)
4236
+
4237
+ assert fdef_node not in self.def_node_body_insertions
4238
+ self.def_node_body_insertions[fdef_node] = assign_node
4239
+
4240
+ return node
4241
+
4242
+ def _inject_super(self, node, func_name):
4243
+ lenv = self.current_env()
4244
+ entry = lenv.lookup_here(func_name)
4245
+ if entry or node.args:
4246
+ return node
4247
+ # Inject no-args super
4248
+ def_node = self.current_scope_node()
4249
+ if not isinstance(def_node, Nodes.DefNode) or not def_node.args or len(self.env_stack) < 2:
4250
+ return node
4251
+ class_node, class_scope = self.env_stack[-2]
4252
+ if class_scope.is_py_class_scope:
4253
+ def_node.requires_classobj = True
4254
+ class_node.class_cell.is_active = True
4255
+ node.args = [
4256
+ ExprNodes.ClassCellNode(
4257
+ node.pos, is_generator=def_node.is_generator),
4258
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4259
+ ]
4260
+ elif class_scope.is_c_class_scope:
4261
+ node.args = [
4262
+ ExprNodes.NameNode(
4263
+ node.pos, name=class_node.scope.name,
4264
+ entry=class_node.entry),
4265
+ ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
4266
+ ]
4267
+ return node
4268
+
4269
+ def _do_body_insertion(self, node):
4270
+ body_insertion = self.def_node_body_insertions.pop(node, None)
4271
+ if body_insertion:
4272
+ if isinstance(node.body, Nodes.StatListNode):
4273
+ node.body.stats.insert(0, body_insertion)
4274
+ else:
4275
+ node.body = Nodes.StatListNode(node.body.pos,
4276
+ stats=[body_insertion, node.body])
4277
+
4278
+ def visit_FuncDefNode(self, node):
4279
+ node = super().visit_FuncDefNode(node)
4280
+ self._do_body_insertion(node)
4281
+ return node
4282
+
4283
+ def visit_GeneratorBodyDefNode(self, node):
4284
+ node = super().visit_GeneratorBodyDefNode(node)
4285
+ self._do_body_insertion(node)
4286
+ return node
4287
+
4288
+ def _inject_branch_hint(self, node):
4289
+ condition = node.condition
4290
+ if isinstance(condition, ExprNodes.SimpleCallNode):
4291
+ function = condition.function.as_cython_attribute()
4292
+ if function in ('likely', 'unlikely'):
4293
+ node.branch_hint = function
4294
+ node.condition = condition.args[0]
4295
+ self.visitchildren(node)
4296
+ return node
4297
+
4298
+ def visit_IfClauseNode(self, node):
4299
+ return self._inject_branch_hint(node)
4300
+
4301
+ def visit_CondExprNode(self, node):
4302
+ return self._inject_branch_hint(node)
4303
+
4304
+ def visit_SimpleCallNode(self, node):
4305
+ # cython.foo
4306
+ function = node.function.as_cython_attribute()
4307
+ if function:
4308
+ if function in InterpretCompilerDirectives.unop_method_nodes:
4309
+ if len(node.args) != 1:
4310
+ error(node.function.pos, "%s() takes exactly one argument" % function)
4311
+ else:
4312
+ node = InterpretCompilerDirectives.unop_method_nodes[function](
4313
+ node.function.pos, operand=node.args[0])
4314
+ elif function in InterpretCompilerDirectives.binop_method_nodes:
4315
+ if len(node.args) != 2:
4316
+ error(node.function.pos, "%s() takes exactly two arguments" % function)
4317
+ else:
4318
+ node = InterpretCompilerDirectives.binop_method_nodes[function](
4319
+ node.function.pos, operand1=node.args[0], operand2=node.args[1])
4320
+ elif function == 'cast':
4321
+ if len(node.args) != 2:
4322
+ error(node.function.pos,
4323
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4324
+ else:
4325
+ type = node.args[0].analyse_as_type(self.current_env())
4326
+ if type:
4327
+ node = ExprNodes.TypecastNode(
4328
+ node.function.pos, type=type, operand=node.args[1], typecheck=False)
4329
+ else:
4330
+ error(node.args[0].pos, "Not a type")
4331
+ elif function == 'sizeof':
4332
+ if len(node.args) != 1:
4333
+ error(node.function.pos, "sizeof() takes exactly one argument")
4334
+ else:
4335
+ type = node.args[0].analyse_as_type(self.current_env())
4336
+ if type:
4337
+ node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
4338
+ else:
4339
+ node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
4340
+ elif function == 'cmod':
4341
+ if len(node.args) != 2:
4342
+ error(node.function.pos, "cmod() takes exactly two arguments")
4343
+ else:
4344
+ node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
4345
+ node.cdivision = True
4346
+ elif function == 'cdiv':
4347
+ if len(node.args) != 2:
4348
+ error(node.function.pos, "cdiv() takes exactly two arguments")
4349
+ else:
4350
+ node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
4351
+ node.cdivision = True
4352
+ elif function == 'set':
4353
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
4354
+ elif function == 'staticmethod':
4355
+ node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod'))
4356
+ elif self.context.cython_scope.lookup_qualified_name(function):
4357
+ pass
4358
+ else:
4359
+ error(node.function.pos,
4360
+ "'%s' not a valid cython language construct" % function)
4361
+
4362
+ self.visitchildren(node)
4363
+
4364
+ if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
4365
+ func_name = node.function.name
4366
+ if func_name in ('dir', 'locals', 'vars'):
4367
+ return self._inject_locals(node, func_name)
4368
+ if func_name == 'eval':
4369
+ return self._inject_eval(node, func_name)
4370
+ if func_name == 'super':
4371
+ return self._inject_super(node, func_name)
4372
+ return node
4373
+
4374
+ def visit_GeneralCallNode(self, node):
4375
+ function = node.function.as_cython_attribute()
4376
+ if function == 'cast':
4377
+ # NOTE: assuming simple tuple/dict nodes for positional_args and keyword_args
4378
+ args = node.positional_args.args
4379
+ kwargs = node.keyword_args.compile_time_value(None)
4380
+ if (len(args) != 2 or len(kwargs) > 1 or
4381
+ (len(kwargs) == 1 and 'typecheck' not in kwargs)):
4382
+ error(node.function.pos,
4383
+ "cast() takes exactly two arguments and an optional typecheck keyword")
4384
+ else:
4385
+ type = args[0].analyse_as_type(self.current_env())
4386
+ if type:
4387
+ typecheck = kwargs.get('typecheck', False)
4388
+ node = ExprNodes.TypecastNode(
4389
+ node.function.pos, type=type, operand=args[1], typecheck=typecheck)
4390
+ else:
4391
+ error(args[0].pos, "Not a type")
4392
+
4393
+ self.visitchildren(node)
4394
+ return node
4395
+
4396
+
4397
+ class ReplaceFusedTypeChecks(VisitorTransform):
4398
+ """
4399
+ This is not a transform in the pipeline. It is invoked on the specific
4400
+ versions of a cdef function with fused argument types. It filters out any
4401
+ type branches that don't match. e.g.
4402
+
4403
+ if fused_t is mytype:
4404
+ ...
4405
+ elif fused_t in other_fused_type:
4406
+ ...
4407
+ """
4408
+ def __init__(self, local_scope):
4409
+ super().__init__()
4410
+ self.local_scope = local_scope
4411
+ # defer the import until now to avoid circular import time dependencies
4412
+ from .Optimize import ConstantFolding
4413
+ self.transform = ConstantFolding(reevaluate=True)
4414
+
4415
+ def visit_IfStatNode(self, node):
4416
+ """
4417
+ Filters out any if clauses with false compile time type check
4418
+ expression.
4419
+ """
4420
+ self.visitchildren(node)
4421
+ return self.transform(node)
4422
+
4423
+ def visit_GILStatNode(self, node):
4424
+ """
4425
+ Fold constant condition of GILStatNode.
4426
+ """
4427
+ self.visitchildren(node)
4428
+ return self.transform(node)
4429
+
4430
+ def visit_PrimaryCmpNode(self, node):
4431
+ with Errors.local_errors(ignore=True):
4432
+ type1 = node.operand1.analyse_as_type(self.local_scope)
4433
+ type2 = node.operand2.analyse_as_type(self.local_scope)
4434
+
4435
+ if type1 and type2:
4436
+ false_node = ExprNodes.BoolNode(node.pos, value=False)
4437
+ true_node = ExprNodes.BoolNode(node.pos, value=True)
4438
+
4439
+ type1 = self.specialize_type(type1, node.operand1.pos)
4440
+ op = node.operator
4441
+
4442
+ if op in ('is', 'is_not', '==', '!='):
4443
+ type2 = self.specialize_type(type2, node.operand2.pos)
4444
+
4445
+ is_same = type1.same_as(type2)
4446
+ eq = op in ('is', '==')
4447
+
4448
+ if (is_same and eq) or (not is_same and not eq):
4449
+ return true_node
4450
+
4451
+ elif op in ('in', 'not_in'):
4452
+ # We have to do an instance check directly, as operand2
4453
+ # needs to be a fused type and not a type with a subtype
4454
+ # that is fused. First unpack the typedef
4455
+ if isinstance(type2, PyrexTypes.CTypedefType):
4456
+ type2 = type2.typedef_base_type
4457
+
4458
+ if type1.is_fused:
4459
+ error(node.operand1.pos, "Type is fused")
4460
+ elif not type2.is_fused:
4461
+ error(node.operand2.pos,
4462
+ "Can only use 'in' or 'not in' on a fused type")
4463
+ else:
4464
+ types = PyrexTypes.get_specialized_types(type2)
4465
+
4466
+ for specialized_type in types:
4467
+ if type1.same_as(specialized_type):
4468
+ if op == 'in':
4469
+ return true_node
4470
+ else:
4471
+ return false_node
4472
+
4473
+ if op == 'not_in':
4474
+ return true_node
4475
+
4476
+ return false_node
4477
+
4478
+ return node
4479
+
4480
+ def specialize_type(self, type, pos):
4481
+ try:
4482
+ return type.specialize(self.local_scope.fused_to_specific)
4483
+ except KeyError:
4484
+ error(pos, "Type is not specific")
4485
+ return type
4486
+
4487
+ def visit_Node(self, node):
4488
+ self.visitchildren(node)
4489
+ return node
4490
+
4491
+
4492
+ class DebugTransform(CythonTransform):
4493
+ """
4494
+ Write debug information for this Cython module.
4495
+ """
4496
+
4497
+ def __init__(self, context, options, result):
4498
+ super().__init__(context)
4499
+ self.visited = set()
4500
+ # our treebuilder and debug output writer
4501
+ # (see Cython.Debugger.debug_output.CythonDebugWriter)
4502
+ self.tb = self.context.gdb_debug_outputwriter
4503
+ #self.c_output_file = options.output_file
4504
+ self.c_output_file = result.c_file
4505
+
4506
+ # Closure support, basically treat nested functions as if the AST were
4507
+ # never nested
4508
+ self.nested_funcdefs = []
4509
+
4510
+ # tells visit_NameNode whether it should register step-into functions
4511
+ self.register_stepinto = False
4512
+
4513
+ def visit_ModuleNode(self, node):
4514
+ self.tb.module_name = node.full_module_name
4515
+ attrs = dict(
4516
+ module_name=node.full_module_name,
4517
+ filename=node.pos[0].filename,
4518
+ c_filename=self.c_output_file)
4519
+
4520
+ self.tb.start('Module', attrs)
4521
+
4522
+ # serialize functions
4523
+ self.tb.start('Functions')
4524
+ # First, serialize functions normally...
4525
+ self.visitchildren(node)
4526
+
4527
+ # ... then, serialize nested functions
4528
+ for nested_funcdef in self.nested_funcdefs:
4529
+ self.visit_FuncDefNode(nested_funcdef)
4530
+
4531
+ self.register_stepinto = True
4532
+ self.serialize_modulenode_as_function(node)
4533
+ self.register_stepinto = False
4534
+ self.tb.end('Functions')
4535
+
4536
+ # 2.3 compatibility. Serialize global variables
4537
+ self.tb.start('Globals')
4538
+ entries = {}
4539
+
4540
+ for k, v in node.scope.entries.items():
4541
+ if (v.qualified_name not in self.visited and not
4542
+ v.name.startswith('__pyx_') and not
4543
+ v.type.is_cfunction and not
4544
+ v.type.is_extension_type):
4545
+ entries[k]= v
4546
+
4547
+ self.serialize_local_variables(entries)
4548
+ self.tb.end('Globals')
4549
+ # self.tb.end('Module') # end Module after the line number mapping in
4550
+ # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
4551
+ return node
4552
+
4553
+ def visit_FuncDefNode(self, node):
4554
+ self.visited.add(node.local_scope.qualified_name)
4555
+
4556
+ if getattr(node, 'is_wrapper', False):
4557
+ return node
4558
+
4559
+ if self.register_stepinto:
4560
+ self.nested_funcdefs.append(node)
4561
+ return node
4562
+
4563
+ # node.entry.visibility = 'extern'
4564
+ if node.py_func is None:
4565
+ pf_cname = ''
4566
+ else:
4567
+ pf_cname = node.py_func.entry.func_cname
4568
+
4569
+ # For functions defined using def, cname will be pyfunc_cname=__pyx_pf_*
4570
+ # For functions defined using cpdef or cdef, cname will be func_cname=__pyx_f_*
4571
+ # In all cases, cname will be the name of the function containing the actual code
4572
+ cname = node.entry.pyfunc_cname or node.entry.func_cname
4573
+
4574
+ attrs = dict(
4575
+ name=node.entry.name or getattr(node, 'name', '<unknown>'),
4576
+ cname=cname,
4577
+ pf_cname=pf_cname,
4578
+ qualified_name=node.local_scope.qualified_name,
4579
+ lineno=str(node.pos[1]))
4580
+
4581
+ self.tb.start('Function', attrs=attrs)
4582
+
4583
+ self.tb.start('Locals')
4584
+ self.serialize_local_variables(node.local_scope.entries)
4585
+ self.tb.end('Locals')
4586
+
4587
+ self.tb.start('Arguments')
4588
+ for arg in node.local_scope.arg_entries:
4589
+ self.tb.start(arg.name)
4590
+ self.tb.end(arg.name)
4591
+ self.tb.end('Arguments')
4592
+
4593
+ self.tb.start('StepIntoFunctions')
4594
+ self.register_stepinto = True
4595
+ self.visitchildren(node)
4596
+ self.register_stepinto = False
4597
+ self.tb.end('StepIntoFunctions')
4598
+ self.tb.end('Function')
4599
+
4600
+ return node
4601
+
4602
+ def visit_NameNode(self, node):
4603
+ if (self.register_stepinto and
4604
+ node.type is not None and
4605
+ node.type.is_cfunction and
4606
+ getattr(node, 'is_called', False) and
4607
+ node.entry.func_cname is not None):
4608
+ # don't check node.entry.in_cinclude, as 'cdef extern: ...'
4609
+ # declared functions are not 'in_cinclude'.
4610
+ # This means we will list called 'cdef' functions as
4611
+ # "step into functions", but this is not an issue as they will be
4612
+ # recognized as Cython functions anyway.
4613
+ attrs = dict(name=node.entry.func_cname)
4614
+ self.tb.start('StepIntoFunction', attrs=attrs)
4615
+ self.tb.end('StepIntoFunction')
4616
+
4617
+ self.visitchildren(node)
4618
+ return node
4619
+
4620
+ def serialize_modulenode_as_function(self, node):
4621
+ """
4622
+ Serialize the module-level code as a function so the debugger will know
4623
+ it's a "relevant frame" and it will know where to set the breakpoint
4624
+ for 'break modulename'.
4625
+ """
4626
+ self._serialize_modulenode_as_function(node, dict(
4627
+ name=node.full_module_name.rpartition('.')[-1],
4628
+ cname=node.module_init_func_cname(),
4629
+ pf_cname='',
4630
+ # Ignore the qualified_name, breakpoints should be set using
4631
+ # `cy break modulename:lineno` for module-level breakpoints.
4632
+ qualified_name='',
4633
+ lineno='1',
4634
+ is_initmodule_function="True",
4635
+ ))
4636
+
4637
+ def _serialize_modulenode_as_function(self, node, attrs):
4638
+ self.tb.start('Function', attrs=attrs)
4639
+
4640
+ self.tb.start('Locals')
4641
+ self.serialize_local_variables(node.scope.entries)
4642
+ self.tb.end('Locals')
4643
+
4644
+ self.tb.start('Arguments')
4645
+ self.tb.end('Arguments')
4646
+
4647
+ self.tb.start('StepIntoFunctions')
4648
+ self.register_stepinto = True
4649
+ self.visitchildren(node)
4650
+ self.register_stepinto = False
4651
+ self.tb.end('StepIntoFunctions')
4652
+
4653
+ self.tb.end('Function')
4654
+
4655
+ def serialize_local_variables(self, entries):
4656
+ for entry in entries.values():
4657
+ if not entry.cname:
4658
+ # not a local variable
4659
+ continue
4660
+ if entry.type.is_pyobject:
4661
+ vartype = 'PythonObject'
4662
+ else:
4663
+ vartype = 'CObject'
4664
+
4665
+ if entry.from_closure:
4666
+ # We're dealing with a closure where a variable from an outer
4667
+ # scope is accessed, get it from the scope object.
4668
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4669
+ entry.outer_entry.cname)
4670
+
4671
+ qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
4672
+ entry.scope.name,
4673
+ entry.name)
4674
+ elif entry.in_closure:
4675
+ cname = '%s->%s' % (Naming.cur_scope_cname,
4676
+ entry.cname)
4677
+ qname = entry.qualified_name
4678
+ else:
4679
+ cname = entry.cname
4680
+ qname = entry.qualified_name
4681
+
4682
+ if not entry.pos:
4683
+ # this happens for variables that are not in the user's code,
4684
+ # e.g. for the global __builtins__, __doc__, etc. We can just
4685
+ # set the lineno to 0 for those.
4686
+ lineno = '0'
4687
+ else:
4688
+ lineno = str(entry.pos[1])
4689
+
4690
+ attrs = dict(
4691
+ name=entry.name,
4692
+ cname=cname,
4693
+ qualified_name=qname,
4694
+ type=vartype,
4695
+ lineno=lineno)
4696
+
4697
+ self.tb.start('LocalVar', attrs)
4698
+ self.tb.end('LocalVar')
4699
+
4700
+
4701
+ class HasNoExceptionHandlingVisitor(TreeVisitor):
4702
+ """
4703
+ Used by finalExceptClauseNode to work out if the body
4704
+ needs to handle exceptions at all. This includes:
4705
+
4706
+ 1. Can raise an exception.
4707
+ 2. May try to access the traceback.
4708
+ """
4709
+ def __init__(self):
4710
+ self.uses_no_exceptions = True
4711
+ self.assignment_lhs = None
4712
+ super().__init__()
4713
+
4714
+ def __call__(self, node) -> bool:
4715
+ self.visit(node)
4716
+ return self.uses_no_exceptions
4717
+
4718
+ def visit_Node(self, node):
4719
+ self.uses_no_exceptions = False # In general, nodes use exceptions
4720
+
4721
+ def visit_ExprStatNode(self, node):
4722
+ self.visitchildren(node)
4723
+
4724
+ def visit_StatListNode(self, node):
4725
+ self.visitchildren(node)
4726
+
4727
+ def visit_ExprNode(self, node):
4728
+ if not node.is_literal:
4729
+ self.uses_no_exceptions = False
4730
+
4731
+ def visit_CallNode(self, node):
4732
+ # Implement this to make the behaviour as explicit as possible.
4733
+ # Even noexcept functions might end up printing a traceback.
4734
+ self.uses_no_exceptions = False
4735
+
4736
+ def visit_PassStatNode(self, node):
4737
+ pass # Does nothing. Good.
4738
+
4739
+ def visit_ReturnStatNode(self, node):
4740
+ if not self.uses_no_exceptions:
4741
+ return # shortcut
4742
+ self.visitchildren(node)
4743
+
4744
+ def visit_SingleAssignmentNode(self, node):
4745
+ if not self.uses_no_exceptions:
4746
+ return # shortcut
4747
+ self.assignment_lhs = node.lhs
4748
+ self.visit(node.lhs)
4749
+ self.assignment_lhs = None
4750
+ rhs_type = node.rhs.type
4751
+ if not (rhs_type.is_numeric or rhs_type.is_pyobject or rhs_type.is_memoryviewslice):
4752
+ # Treat everything we haven't explicitly thought about as potentially dubious.
4753
+ # cpp classes may have non-trivial assignment operators for example.
4754
+ self.uses_no_exceptions = False
4755
+ if not self.uses_no_exceptions:
4756
+ return
4757
+ self.visitchildren(node, exclude=["lhs"])
4758
+
4759
+ def visit_NameNode(self, node):
4760
+ if not self.uses_no_exceptions:
4761
+ return # shortcut
4762
+ entry = node.entry
4763
+ if self.assignment_lhs is node:
4764
+ if not (entry.is_cglobal or entry.is_arg or
4765
+ entry.is_local or entry.in_closure or entry.from_closure):
4766
+ self.uses_no_exceptions = False
4767
+ return
4768
+ else:
4769
+ if entry.is_cglobal:
4770
+ if entry.is_cpp_optional and node.initialized_check:
4771
+ # Otherwise, reading C globals should be safe.
4772
+ self.uses_no_exceptions = False
4773
+ return
4774
+ elif entry.is_arg or entry.is_local or entry.in_closure or entry.from_closure:
4775
+ if (node.cf_is_null or node.cf_maybe_null) and not node.type.is_numeric:
4776
+ # The logic here is slightly simpler than for NameNode error checking.
4777
+ # This gives a few false negatives (which is always the safe thing to do)
4778
+ # for memoryviews and cpp_optionals
4779
+ self.uses_no_exceptions = False
4780
+ return
4781
+ else:
4782
+ # Probably a py_global.
4783
+ self.uses_no_exceptions = False
4784
+ return
4785
+
4786
+ def visit_AttributeNode(self, node):
4787
+ if node.is_py_attr:
4788
+ self.uses_no_exceptions = False
4789
+ elif (node.type.is_memoryviewslice or node.entry.is_cpp_optional) and self.assignment_lhs is not node:
4790
+ # Memoryviewslices and cpp_optional are OK as a target, but reading them involves checks.
4791
+ # (Although cpp optionals are currently banned elsewhere
4792
+ # because C++ classes may have non-trivial assignment).
4793
+ self.uses_no_exceptions = False
4794
+ # Python objects just need an incref and simple C types are fine, too. Others may not be.
4795
+ if not (node.type.is_pyobject or node.type.is_numeric or node.type.is_memoryviewslice):
4796
+ self.uses_no_exceptions = False
4797
+ if self.uses_no_exceptions:
4798
+ self.visitchildren(node)
4799
+
4800
+ def visit_IndexNode(self, node):
4801
+ if not (node.base.type.is_array or node.base.type.is_ptr):
4802
+ self.uses_no_exceptions = False
4803
+ if not self.uses_no_exceptions:
4804
+ return
4805
+ self.visitchildren(node)
4806
+
4807
+ def visit_CoerceToTempNode(self, node):
4808
+ self.visitchildren(node)