typenative 0.6.2 → 0.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1131) hide show
  1. package/README.md +1 -1
  2. package/bin/cli.js +18 -8
  3. package/bin/go/overrides.js +21 -1
  4. package/bin/index.js +48 -25
  5. package/bin/main.js +2 -1
  6. package/bin/node/assert.js +56 -17
  7. package/bin/node/async_hooks.js +69 -10
  8. package/bin/node/buffer.js +321 -7
  9. package/bin/node/child_process.js +220 -16
  10. package/bin/node/cluster.js +436 -32
  11. package/bin/node/crypto.js +814 -5
  12. package/bin/node/crypto_curves.js +142 -0
  13. package/bin/node/crypto_keys.js +1147 -48
  14. package/bin/node/crypto_pem.js +313 -0
  15. package/bin/node/crypto_pkcs12.js +17 -0
  16. package/bin/node/dgram.js +161 -10
  17. package/bin/node/diagnostics_channel.js +5 -1
  18. package/bin/node/events.js +358 -8
  19. package/bin/node/ffi.js +933 -0
  20. package/bin/node/fs.js +1346 -10
  21. package/bin/node/histogram.js +1991 -0
  22. package/bin/node/http.js +630 -55
  23. package/bin/node/http2.js +1141 -88
  24. package/bin/node/intl.js +4576 -0
  25. package/bin/node/intl_data.js +65412 -0
  26. package/bin/node/module_v8.js +155 -16
  27. package/bin/node/net.js +59 -10
  28. package/bin/node/path.js +5 -1
  29. package/bin/node/process.js +355 -4
  30. package/bin/node/readline.js +558 -8
  31. package/bin/node/registry.js +2 -0
  32. package/bin/node/sqlite.js +1539 -7
  33. package/bin/node/stream.js +371 -32
  34. package/bin/node/stream_web.js +422 -15
  35. package/bin/node/system_extra.js +40 -154
  36. package/bin/node/temporal.js +96 -0
  37. package/bin/node/temporal_runtime.js +5075 -0
  38. package/bin/node/temporal_zones.js +460 -0
  39. package/bin/node/test.js +191 -8
  40. package/bin/node/test_coverage.js +58 -0
  41. package/bin/node/test_report.js +50 -4
  42. package/bin/node/test_run.js +64 -3
  43. package/bin/node/timers.js +123 -0
  44. package/bin/node/tls.js +346 -12
  45. package/bin/node/tls_roots.js +122 -0
  46. package/bin/node/typed_arrays.js +867 -93
  47. package/bin/node/url.js +657 -0
  48. package/bin/node/util.js +449 -7
  49. package/bin/node/util_extra.js +6 -0
  50. package/bin/node/wasi.js +214 -0
  51. package/bin/node/webassembly.js +1137 -0
  52. package/bin/node/websocket.js +763 -0
  53. package/bin/node/webview.js +3257 -244
  54. package/bin/node/worker_threads.js +434 -52
  55. package/bin/node/zlib.js +455 -22
  56. package/bin/node/zlib_iter.js +585 -0
  57. package/bin/node/zlib_zip.js +2392 -0
  58. package/bin/transpiler.js +3417 -501
  59. package/bin/typecheck-node.js +49 -17
  60. package/bin/version.js +1 -1
  61. package/govendor/github.com/cloudflare/circl/.golangci.yaml +64 -0
  62. package/govendor/github.com/cloudflare/circl/.semgrepignore +1 -0
  63. package/govendor/github.com/cloudflare/circl/AGENTS.md +91 -0
  64. package/govendor/github.com/cloudflare/circl/CITATION.cff +30 -0
  65. package/govendor/github.com/cloudflare/circl/LICENSE +57 -0
  66. package/govendor/github.com/cloudflare/circl/Makefile +70 -0
  67. package/govendor/github.com/cloudflare/circl/README.md +206 -0
  68. package/govendor/github.com/cloudflare/circl/REVIEW.md +83 -0
  69. package/govendor/github.com/cloudflare/circl/codecov.yml +21 -0
  70. package/govendor/github.com/cloudflare/circl/dh/x448/curve.go +104 -0
  71. package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.go +30 -0
  72. package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.h +111 -0
  73. package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.s +194 -0
  74. package/govendor/github.com/cloudflare/circl/dh/x448/curve_generic.go +100 -0
  75. package/govendor/github.com/cloudflare/circl/dh/x448/curve_noasm.go +11 -0
  76. package/govendor/github.com/cloudflare/circl/dh/x448/doc.go +19 -0
  77. package/govendor/github.com/cloudflare/circl/dh/x448/key.go +46 -0
  78. package/govendor/github.com/cloudflare/circl/dh/x448/table.go +460 -0
  79. package/govendor/github.com/cloudflare/circl/doc.go +8 -0
  80. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/constants.go +71 -0
  81. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/curve.go +90 -0
  82. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/isogeny.go +52 -0
  83. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/point.go +181 -0
  84. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/scalar.go +203 -0
  85. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twist.go +138 -0
  86. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twistPoint.go +135 -0
  87. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twistTables.go +216 -0
  88. package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twist_basemult.go +62 -0
  89. package/govendor/github.com/cloudflare/circl/go.mod +3 -0
  90. package/govendor/github.com/cloudflare/circl/internal/conv/conv.go +173 -0
  91. package/govendor/github.com/cloudflare/circl/internal/sha3/doc.go +62 -0
  92. package/govendor/github.com/cloudflare/circl/internal/sha3/hashes.go +69 -0
  93. package/govendor/github.com/cloudflare/circl/internal/sha3/keccakf.go +391 -0
  94. package/govendor/github.com/cloudflare/circl/internal/sha3/rc.go +29 -0
  95. package/govendor/github.com/cloudflare/circl/internal/sha3/sha3.go +200 -0
  96. package/govendor/github.com/cloudflare/circl/internal/sha3/sha3_s390x.s +33 -0
  97. package/govendor/github.com/cloudflare/circl/internal/sha3/shake.go +119 -0
  98. package/govendor/github.com/cloudflare/circl/internal/sha3/xor.go +15 -0
  99. package/govendor/github.com/cloudflare/circl/internal/sha3/xor_generic.go +33 -0
  100. package/govendor/github.com/cloudflare/circl/internal/sha3/xor_unaligned.go +61 -0
  101. package/govendor/github.com/cloudflare/circl/math/fp448/fp.go +164 -0
  102. package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.go +43 -0
  103. package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.h +591 -0
  104. package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.s +75 -0
  105. package/govendor/github.com/cloudflare/circl/math/fp448/fp_generic.go +339 -0
  106. package/govendor/github.com/cloudflare/circl/math/fp448/fp_noasm.go +12 -0
  107. package/govendor/github.com/cloudflare/circl/math/fp448/fuzzer.go +75 -0
  108. package/govendor/github.com/cloudflare/circl/math/integer.go +16 -0
  109. package/govendor/github.com/cloudflare/circl/math/mlsbset/mlsbset.go +145 -0
  110. package/govendor/github.com/cloudflare/circl/math/mlsbset/power.go +64 -0
  111. package/govendor/github.com/cloudflare/circl/math/primes.go +34 -0
  112. package/govendor/github.com/cloudflare/circl/math/wnaf.go +84 -0
  113. package/govendor/github.com/cloudflare/circl/sign/ed448/ed448.go +417 -0
  114. package/govendor/github.com/cloudflare/circl/sign/ed448/signapi.go +89 -0
  115. package/govendor/github.com/cloudflare/circl/sign/sign.go +119 -0
  116. package/{examples/react-compiler/vendor/picocolors → govendor/github.com/decred/dcrd/dcrec/secp256k1/v4}/LICENSE +4 -2
  117. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/README.md +72 -0
  118. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/compressedbytepoints.go +18 -0
  119. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve.go +1310 -0
  120. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve_embedded.go +14 -0
  121. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve_precompute.go +14 -0
  122. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/doc.go +59 -0
  123. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdh.go +21 -0
  124. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/README.md +52 -0
  125. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/doc.go +42 -0
  126. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/error.go +134 -0
  127. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/signature.go +1001 -0
  128. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ellipticadaptor.go +296 -0
  129. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/error.go +67 -0
  130. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/field.go +1696 -0
  131. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/go.mod +3 -0
  132. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/loadprecomputed.go +91 -0
  133. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/modnscalar.go +1105 -0
  134. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/nonce.go +263 -0
  135. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/privkey.go +111 -0
  136. package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/pubkey.go +236 -0
  137. package/govendor/github.com/ebitengine/purego/LICENSE +201 -0
  138. package/govendor/github.com/ebitengine/purego/README.md +124 -0
  139. package/govendor/github.com/ebitengine/purego/abi_amd64.h +99 -0
  140. package/govendor/github.com/ebitengine/purego/abi_arm64.h +39 -0
  141. package/govendor/github.com/ebitengine/purego/abi_loong64.h +60 -0
  142. package/govendor/github.com/ebitengine/purego/cdecl.go +11 -0
  143. package/govendor/github.com/ebitengine/purego/cgo.go +19 -0
  144. package/govendor/github.com/ebitengine/purego/dlerror.go +17 -0
  145. package/govendor/github.com/ebitengine/purego/dlfcn.go +99 -0
  146. package/govendor/github.com/ebitengine/purego/dlfcn_android.go +34 -0
  147. package/govendor/github.com/ebitengine/purego/dlfcn_darwin.go +20 -0
  148. package/govendor/github.com/ebitengine/purego/dlfcn_freebsd.go +14 -0
  149. package/govendor/github.com/ebitengine/purego/dlfcn_linux.go +16 -0
  150. package/govendor/github.com/ebitengine/purego/dlfcn_netbsd.go +15 -0
  151. package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_freebsd.go +11 -0
  152. package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_linux.go +19 -0
  153. package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_netbsd.go +11 -0
  154. package/govendor/github.com/ebitengine/purego/dlfcn_playground.go +24 -0
  155. package/govendor/github.com/ebitengine/purego/dlfcn_stubs.s +22 -0
  156. package/govendor/github.com/ebitengine/purego/func.go +780 -0
  157. package/govendor/github.com/ebitengine/purego/gen.go +6 -0
  158. package/govendor/github.com/ebitengine/purego/go.mod +3 -0
  159. package/govendor/github.com/ebitengine/purego/go_runtime.go +19 -0
  160. package/govendor/github.com/ebitengine/purego/internal/cgo/dlfcn_cgo_unix.go +56 -0
  161. package/govendor/github.com/ebitengine/purego/internal/cgo/empty.go +6 -0
  162. package/govendor/github.com/ebitengine/purego/internal/cgo/syscall_cgo_unix.go +49 -0
  163. package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_amd64.h +99 -0
  164. package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_arm64.h +39 -0
  165. package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_loong64.h +60 -0
  166. package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_ppc64x.h +195 -0
  167. package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_riscv64.h +72 -0
  168. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_386.s +29 -0
  169. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_amd64.s +39 -0
  170. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_arm.s +58 -0
  171. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_arm64.s +35 -0
  172. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_loong64.s +38 -0
  173. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_ppc64le.s +79 -0
  174. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_riscv64.s +36 -0
  175. package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_s390x.s +52 -0
  176. package/govendor/github.com/ebitengine/purego/internal/fakecgo/callbacks.go +93 -0
  177. package/govendor/github.com/ebitengine/purego/internal/fakecgo/doc.go +32 -0
  178. package/govendor/github.com/ebitengine/purego/internal/fakecgo/fakecgo.go +14 -0
  179. package/govendor/github.com/ebitengine/purego/internal/fakecgo/freebsd.go +27 -0
  180. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_darwin.go +86 -0
  181. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_freebsd.go +77 -0
  182. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_libinit.go +72 -0
  183. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_linux.go +77 -0
  184. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_netbsd.go +83 -0
  185. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_setenv.go +18 -0
  186. package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_util.go +38 -0
  187. package/govendor/github.com/ebitengine/purego/internal/fakecgo/iscgo.go +19 -0
  188. package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo.go +37 -0
  189. package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_darwin.go +37 -0
  190. package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_freebsd.go +27 -0
  191. package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_linux.go +27 -0
  192. package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_netbsd.go +36 -0
  193. package/govendor/github.com/ebitengine/purego/internal/fakecgo/linux.go +189 -0
  194. package/govendor/github.com/ebitengine/purego/internal/fakecgo/netbsd.go +23 -0
  195. package/govendor/github.com/ebitengine/purego/internal/fakecgo/setenv.go +19 -0
  196. package/govendor/github.com/ebitengine/purego/internal/fakecgo/threadentry_amd64.go +15 -0
  197. package/govendor/github.com/ebitengine/purego/internal/fakecgo/threadentry_noasm.go +24 -0
  198. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_386.s +130 -0
  199. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_amd64.s +124 -0
  200. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_arm.s +131 -0
  201. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_arm64.s +81 -0
  202. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_386.s +78 -0
  203. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_amd64.s +69 -0
  204. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_arm.s +69 -0
  205. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_arm64.s +60 -0
  206. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_loong64.s +60 -0
  207. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_ppc64le.s +69 -0
  208. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_riscv64.s +60 -0
  209. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_s390x.s +51 -0
  210. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_loong64.s +78 -0
  211. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_ppc64le.s +132 -0
  212. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_riscv64.s +76 -0
  213. package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_s390x.s +158 -0
  214. package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols.go +165 -0
  215. package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_darwin.go +59 -0
  216. package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_freebsd.go +48 -0
  217. package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_linux.go +158 -0
  218. package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_netbsd.go +59 -0
  219. package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_darwin.s +19 -0
  220. package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_freebsd.s +16 -0
  221. package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_linux.s +46 -0
  222. package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_netbsd.s +19 -0
  223. package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_stubs.s +55 -0
  224. package/govendor/github.com/ebitengine/purego/internal/strings/strings.go +40 -0
  225. package/govendor/github.com/ebitengine/purego/is_ios.go +13 -0
  226. package/govendor/github.com/ebitengine/purego/nocgo.go +25 -0
  227. package/govendor/github.com/ebitengine/purego/struct_amd64.go +518 -0
  228. package/govendor/github.com/ebitengine/purego/struct_arm.go +100 -0
  229. package/govendor/github.com/ebitengine/purego/struct_arm64.go +757 -0
  230. package/govendor/github.com/ebitengine/purego/struct_loong64.go +244 -0
  231. package/govendor/github.com/ebitengine/purego/struct_other.go +54 -0
  232. package/govendor/github.com/ebitengine/purego/struct_ppc64le.go +180 -0
  233. package/govendor/github.com/ebitengine/purego/struct_riscv64.go +158 -0
  234. package/govendor/github.com/ebitengine/purego/struct_s390x.go +158 -0
  235. package/govendor/github.com/ebitengine/purego/sys_386.s +153 -0
  236. package/govendor/github.com/ebitengine/purego/sys_amd64.s +206 -0
  237. package/govendor/github.com/ebitengine/purego/sys_arm.s +153 -0
  238. package/govendor/github.com/ebitengine/purego/sys_arm64.s +133 -0
  239. package/govendor/github.com/ebitengine/purego/sys_loong64.s +131 -0
  240. package/govendor/github.com/ebitengine/purego/sys_ppc64le.s +126 -0
  241. package/govendor/github.com/ebitengine/purego/sys_riscv64.s +135 -0
  242. package/govendor/github.com/ebitengine/purego/sys_s390x.s +148 -0
  243. package/govendor/github.com/ebitengine/purego/sys_unix_386.s +234 -0
  244. package/govendor/github.com/ebitengine/purego/sys_unix_arm.s +99 -0
  245. package/govendor/github.com/ebitengine/purego/sys_unix_arm64.s +75 -0
  246. package/govendor/github.com/ebitengine/purego/sys_unix_loong64.s +79 -0
  247. package/govendor/github.com/ebitengine/purego/sys_unix_ppc64le.s +114 -0
  248. package/govendor/github.com/ebitengine/purego/sys_unix_riscv64.s +79 -0
  249. package/govendor/github.com/ebitengine/purego/sys_unix_s390x.s +111 -0
  250. package/govendor/github.com/ebitengine/purego/syscall.go +95 -0
  251. package/govendor/github.com/ebitengine/purego/syscall_32bit.go +97 -0
  252. package/govendor/github.com/ebitengine/purego/syscall_cgo_linux.go +20 -0
  253. package/govendor/github.com/ebitengine/purego/syscall_notstackargs.go +55 -0
  254. package/govendor/github.com/ebitengine/purego/syscall_ppc64le.go +73 -0
  255. package/govendor/github.com/ebitengine/purego/syscall_stackargs_ppc64le.go +38 -0
  256. package/govendor/github.com/ebitengine/purego/syscall_stackargs_s390x.go +38 -0
  257. package/govendor/github.com/ebitengine/purego/syscall_unix.go +392 -0
  258. package/govendor/github.com/ebitengine/purego/syscall_windows.go +54 -0
  259. package/govendor/github.com/ebitengine/purego/zcallback_386.s +4014 -0
  260. package/govendor/github.com/ebitengine/purego/zcallback_amd64.s +2014 -0
  261. package/govendor/github.com/ebitengine/purego/zcallback_arm.s +4014 -0
  262. package/govendor/github.com/ebitengine/purego/zcallback_arm64.s +4014 -0
  263. package/govendor/github.com/ebitengine/purego/zcallback_loong64.s +4014 -0
  264. package/govendor/github.com/ebitengine/purego/zcallback_ppc64le.s +4014 -0
  265. package/govendor/github.com/ebitengine/purego/zcallback_riscv64.s +4051 -0
  266. package/govendor/github.com/ebitengine/purego/zcallback_s390x.s +4015 -0
  267. package/govendor/github.com/klauspost/compress/.gitattributes +3 -0
  268. package/govendor/github.com/klauspost/compress/.goreleaser.yml +132 -0
  269. package/govendor/github.com/klauspost/compress/AGENTS.md +106 -0
  270. package/govendor/github.com/klauspost/compress/LICENSE +304 -0
  271. package/govendor/github.com/klauspost/compress/README.md +766 -0
  272. package/govendor/github.com/klauspost/compress/SECURITY.md +25 -0
  273. package/govendor/github.com/klauspost/compress/compressible.go +85 -0
  274. package/govendor/github.com/klauspost/compress/fse/README.md +79 -0
  275. package/govendor/github.com/klauspost/compress/fse/bitreader.go +122 -0
  276. package/govendor/github.com/klauspost/compress/fse/bitwriter.go +167 -0
  277. package/govendor/github.com/klauspost/compress/fse/bytereader.go +47 -0
  278. package/govendor/github.com/klauspost/compress/fse/compress.go +685 -0
  279. package/govendor/github.com/klauspost/compress/fse/decompress.go +383 -0
  280. package/govendor/github.com/klauspost/compress/fse/fse.go +150 -0
  281. package/govendor/github.com/klauspost/compress/gen.sh +4 -0
  282. package/govendor/github.com/klauspost/compress/go.mod +3 -0
  283. package/govendor/github.com/klauspost/compress/huff0/README.md +89 -0
  284. package/govendor/github.com/klauspost/compress/huff0/bitreader.go +323 -0
  285. package/govendor/github.com/klauspost/compress/huff0/bitwriter.go +97 -0
  286. package/govendor/github.com/klauspost/compress/huff0/build_table.go +168 -0
  287. package/govendor/github.com/klauspost/compress/huff0/compress.go +747 -0
  288. package/govendor/github.com/klauspost/compress/huff0/decompress.go +1188 -0
  289. package/govendor/github.com/klauspost/compress/huff0/decompress_amd64.go +128 -0
  290. package/govendor/github.com/klauspost/compress/huff0/decompress_amd64.s +3476 -0
  291. package/govendor/github.com/klauspost/compress/huff0/decompress_arm64.go +72 -0
  292. package/govendor/github.com/klauspost/compress/huff0/decompress_arm64.s +1648 -0
  293. package/govendor/github.com/klauspost/compress/huff0/decompress_asm.go +252 -0
  294. package/govendor/github.com/klauspost/compress/huff0/decompress_generic.go +278 -0
  295. package/govendor/github.com/klauspost/compress/huff0/huff0.go +337 -0
  296. package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go +34 -0
  297. package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +10 -0
  298. package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s +34 -0
  299. package/govendor/github.com/klauspost/compress/internal/le/le.go +5 -0
  300. package/govendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go +42 -0
  301. package/govendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go +52 -0
  302. package/govendor/github.com/klauspost/compress/internal/regmask/regmask.go +50 -0
  303. package/govendor/github.com/klauspost/compress/internal/regmask/regmask_386.go +39 -0
  304. package/govendor/github.com/klauspost/compress/internal/snapref/LICENSE +27 -0
  305. package/govendor/github.com/klauspost/compress/internal/snapref/decode.go +264 -0
  306. package/govendor/github.com/klauspost/compress/internal/snapref/decode_other.go +113 -0
  307. package/govendor/github.com/klauspost/compress/internal/snapref/encode.go +291 -0
  308. package/govendor/github.com/klauspost/compress/internal/snapref/encode_other.go +250 -0
  309. package/govendor/github.com/klauspost/compress/internal/snapref/snappy.go +98 -0
  310. package/govendor/github.com/klauspost/compress/s2sx.mod +3 -0
  311. package/govendor/github.com/klauspost/compress/s2sx.sum +0 -0
  312. package/govendor/github.com/klauspost/compress/zstd/README.md +487 -0
  313. package/govendor/github.com/klauspost/compress/zstd/bitreader.go +176 -0
  314. package/govendor/github.com/klauspost/compress/zstd/bitwriter.go +151 -0
  315. package/govendor/github.com/klauspost/compress/zstd/blockdec.go +728 -0
  316. package/govendor/github.com/klauspost/compress/zstd/blockenc.go +923 -0
  317. package/govendor/github.com/klauspost/compress/zstd/blocktype_string.go +85 -0
  318. package/govendor/github.com/klauspost/compress/zstd/bytebuf.go +131 -0
  319. package/govendor/github.com/klauspost/compress/zstd/bytereader.go +82 -0
  320. package/govendor/github.com/klauspost/compress/zstd/decodeheader.go +261 -0
  321. package/govendor/github.com/klauspost/compress/zstd/decoder.go +995 -0
  322. package/govendor/github.com/klauspost/compress/zstd/decoder_options.go +213 -0
  323. package/govendor/github.com/klauspost/compress/zstd/dict.go +605 -0
  324. package/govendor/github.com/klauspost/compress/zstd/enc_base.go +200 -0
  325. package/govendor/github.com/klauspost/compress/zstd/enc_best.go +572 -0
  326. package/govendor/github.com/klauspost/compress/zstd/enc_better.go +1262 -0
  327. package/govendor/github.com/klauspost/compress/zstd/enc_dfast.go +1136 -0
  328. package/govendor/github.com/klauspost/compress/zstd/enc_fast.go +892 -0
  329. package/govendor/github.com/klauspost/compress/zstd/enc_jobs.go +352 -0
  330. package/govendor/github.com/klauspost/compress/zstd/encoder.go +881 -0
  331. package/govendor/github.com/klauspost/compress/zstd/encoder_options.go +422 -0
  332. package/govendor/github.com/klauspost/compress/zstd/framedec.go +404 -0
  333. package/govendor/github.com/klauspost/compress/zstd/frameenc.go +137 -0
  334. package/govendor/github.com/klauspost/compress/zstd/fse_decoder.go +333 -0
  335. package/govendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +132 -0
  336. package/govendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +148 -0
  337. package/govendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go +70 -0
  338. package/govendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +71 -0
  339. package/govendor/github.com/klauspost/compress/zstd/fse_encoder.go +701 -0
  340. package/govendor/github.com/klauspost/compress/zstd/fse_predefined.go +168 -0
  341. package/govendor/github.com/klauspost/compress/zstd/hash.go +57 -0
  342. package/govendor/github.com/klauspost/compress/zstd/history.go +119 -0
  343. package/{examples/react-compiler/vendor/jsesc/LICENSE-MIT.txt → govendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt} +3 -1
  344. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md +71 -0
  345. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +242 -0
  346. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +210 -0
  347. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +184 -0
  348. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go +11 -0
  349. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +75 -0
  350. package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go +11 -0
  351. package/govendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +15 -0
  352. package/govendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +66 -0
  353. package/govendor/github.com/klauspost/compress/zstd/matchlen_generic.go +37 -0
  354. package/govendor/github.com/klauspost/compress/zstd/seqdec.go +620 -0
  355. package/govendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +115 -0
  356. package/govendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +4762 -0
  357. package/govendor/github.com/klauspost/compress/zstd/seqdec_arm64.go +81 -0
  358. package/govendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +2919 -0
  359. package/govendor/github.com/klauspost/compress/zstd/seqdec_asm.go +327 -0
  360. package/govendor/github.com/klauspost/compress/zstd/seqdec_generic.go +258 -0
  361. package/govendor/github.com/klauspost/compress/zstd/seqenc.go +159 -0
  362. package/govendor/github.com/klauspost/compress/zstd/simple.go +54 -0
  363. package/govendor/github.com/klauspost/compress/zstd/snappy.go +447 -0
  364. package/govendor/github.com/klauspost/compress/zstd/zip.go +141 -0
  365. package/govendor/github.com/klauspost/compress/zstd/zstd.go +126 -0
  366. package/govendor/github.com/tetratelabs/wazero/.editorconfig +7 -0
  367. package/govendor/github.com/tetratelabs/wazero/.gitattributes +2 -0
  368. package/govendor/github.com/tetratelabs/wazero/.gitmodules +3 -0
  369. package/govendor/github.com/tetratelabs/wazero/CONTRIBUTING.md +75 -0
  370. package/govendor/github.com/tetratelabs/wazero/LICENSE +201 -0
  371. package/govendor/github.com/tetratelabs/wazero/Makefile +385 -0
  372. package/govendor/github.com/tetratelabs/wazero/NOTICE +2 -0
  373. package/govendor/github.com/tetratelabs/wazero/RATIONALE.md +1577 -0
  374. package/govendor/github.com/tetratelabs/wazero/README.md +135 -0
  375. package/govendor/github.com/tetratelabs/wazero/api/features.go +228 -0
  376. package/govendor/github.com/tetratelabs/wazero/api/wasm.go +766 -0
  377. package/govendor/github.com/tetratelabs/wazero/builder.go +367 -0
  378. package/govendor/github.com/tetratelabs/wazero/cache.go +123 -0
  379. package/govendor/github.com/tetratelabs/wazero/codecov.yml +9 -0
  380. package/govendor/github.com/tetratelabs/wazero/config.go +899 -0
  381. package/govendor/github.com/tetratelabs/wazero/experimental/checkpoint.go +35 -0
  382. package/govendor/github.com/tetratelabs/wazero/experimental/close.go +63 -0
  383. package/govendor/github.com/tetratelabs/wazero/experimental/compilationworkers.go +19 -0
  384. package/govendor/github.com/tetratelabs/wazero/experimental/experimental.go +41 -0
  385. package/govendor/github.com/tetratelabs/wazero/experimental/features.go +39 -0
  386. package/govendor/github.com/tetratelabs/wazero/experimental/importresolver.go +19 -0
  387. package/govendor/github.com/tetratelabs/wazero/experimental/listener.go +324 -0
  388. package/govendor/github.com/tetratelabs/wazero/experimental/memory.go +52 -0
  389. package/govendor/github.com/tetratelabs/wazero/experimental/sys/dir.go +92 -0
  390. package/govendor/github.com/tetratelabs/wazero/experimental/sys/errno.go +98 -0
  391. package/govendor/github.com/tetratelabs/wazero/experimental/sys/error.go +45 -0
  392. package/govendor/github.com/tetratelabs/wazero/experimental/sys/file.go +356 -0
  393. package/govendor/github.com/tetratelabs/wazero/experimental/sys/fs.go +292 -0
  394. package/govendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go +70 -0
  395. package/govendor/github.com/tetratelabs/wazero/experimental/sys/poll.go +48 -0
  396. package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go +106 -0
  397. package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go +13 -0
  398. package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go +7 -0
  399. package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go +37 -0
  400. package/govendor/github.com/tetratelabs/wazero/experimental/sys/time.go +10 -0
  401. package/govendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go +160 -0
  402. package/govendor/github.com/tetratelabs/wazero/fsconfig.go +213 -0
  403. package/govendor/github.com/tetratelabs/wazero/go.mod +3 -0
  404. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/args.go +97 -0
  405. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/clock.go +116 -0
  406. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/environ.go +100 -0
  407. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go +2015 -0
  408. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go +247 -0
  409. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/proc.go +44 -0
  410. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/random.go +55 -0
  411. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/sched.go +22 -0
  412. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/sock.go +188 -0
  413. package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/wasi.go +314 -0
  414. package/govendor/github.com/tetratelabs/wazero/internal/descriptor/table.go +149 -0
  415. package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go +3884 -0
  416. package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go +22 -0
  417. package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go +4963 -0
  418. package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go +2978 -0
  419. package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/signature.go +825 -0
  420. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go +170 -0
  421. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go +3 -0
  422. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go +402 -0
  423. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go +226 -0
  424. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go +33 -0
  425. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go +186 -0
  426. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go +9 -0
  427. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s +29 -0
  428. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go +248 -0
  429. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go +440 -0
  430. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go +168 -0
  431. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go +35 -0
  432. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go +2495 -0
  433. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go +1714 -0
  434. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go +71 -0
  435. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go +187 -0
  436. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go +3782 -0
  437. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go +334 -0
  438. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go +352 -0
  439. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go +992 -0
  440. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go +336 -0
  441. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go +181 -0
  442. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go +130 -0
  443. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go +376 -0
  444. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go +9 -0
  445. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s +29 -0
  446. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go +233 -0
  447. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go +430 -0
  448. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go +215 -0
  449. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go +2562 -0
  450. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go +2357 -0
  451. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go +301 -0
  452. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go +2227 -0
  453. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go +340 -0
  454. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go +451 -0
  455. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go +631 -0
  456. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go +486 -0
  457. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go +351 -0
  458. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go +128 -0
  459. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go +397 -0
  460. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go +84 -0
  461. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go +119 -0
  462. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go +124 -0
  463. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go +123 -0
  464. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go +1189 -0
  465. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go +96 -0
  466. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go +19 -0
  467. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go +884 -0
  468. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go +1008 -0
  469. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go +347 -0
  470. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go +13 -0
  471. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go +13 -0
  472. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_other.go +15 -0
  473. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go +663 -0
  474. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go +5045 -0
  475. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go +10 -0
  476. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go +13 -0
  477. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go +88 -0
  478. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go +30 -0
  479. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go +32 -0
  480. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go +29 -0
  481. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go +11 -0
  482. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go +403 -0
  483. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go +381 -0
  484. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go +32 -0
  485. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go +790 -0
  486. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go +107 -0
  487. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go +12 -0
  488. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go +3024 -0
  489. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go +393 -0
  490. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go +334 -0
  491. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go +313 -0
  492. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go +49 -0
  493. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go +14 -0
  494. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go +115 -0
  495. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go +114 -0
  496. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go +189 -0
  497. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go +150 -0
  498. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go +248 -0
  499. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go +96 -0
  500. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go +5 -0
  501. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go +5 -0
  502. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go +215 -0
  503. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go +15 -0
  504. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go +26 -0
  505. package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go +11 -0
  506. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go +10 -0
  507. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go +5 -0
  508. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/compilationworkers.go +6 -0
  509. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go +2 -0
  510. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/importresolver.go +6 -0
  511. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go +7 -0
  512. package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go +4 -0
  513. package/govendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go +42 -0
  514. package/govendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go +76 -0
  515. package/govendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go +29 -0
  516. package/govendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go +9 -0
  517. package/govendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go +285 -0
  518. package/govendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go +271 -0
  519. package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go +26 -0
  520. package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go +22 -0
  521. package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_arm64.go +13 -0
  522. package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go +5 -0
  523. package/govendor/github.com/tetratelabs/wazero/internal/platform/crypto.go +17 -0
  524. package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go +72 -0
  525. package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go +19 -0
  526. package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go +14 -0
  527. package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go +22 -0
  528. package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go +31 -0
  529. package/govendor/github.com/tetratelabs/wazero/internal/platform/path.go +6 -0
  530. package/govendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go +17 -0
  531. package/govendor/github.com/tetratelabs/wazero/internal/platform/platform.go +71 -0
  532. package/govendor/github.com/tetratelabs/wazero/internal/platform/time.go +76 -0
  533. package/govendor/github.com/tetratelabs/wazero/internal/platform/time_notwindows.go +11 -0
  534. package/govendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go +40 -0
  535. package/govendor/github.com/tetratelabs/wazero/internal/sock/sock.go +89 -0
  536. package/govendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go +11 -0
  537. package/govendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go +10 -0
  538. package/govendor/github.com/tetratelabs/wazero/internal/sys/fs.go +458 -0
  539. package/govendor/github.com/tetratelabs/wazero/internal/sys/lazy.go +135 -0
  540. package/govendor/github.com/tetratelabs/wazero/internal/sys/stdio.go +125 -0
  541. package/govendor/github.com/tetratelabs/wazero/internal/sys/sys.go +228 -0
  542. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go +105 -0
  543. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go +12 -0
  544. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go +14 -0
  545. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go +24 -0
  546. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go +137 -0
  547. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file.go +576 -0
  548. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go +39 -0
  549. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go +28 -0
  550. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go +133 -0
  551. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go +29 -0
  552. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go +42 -0
  553. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s +8 -0
  554. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go +38 -0
  555. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go +18 -0
  556. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go +42 -0
  557. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_unix.go +22 -0
  558. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_unsupported.go +17 -0
  559. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go +28 -0
  560. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go +17 -0
  561. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go +13 -0
  562. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go +23 -0
  563. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go +38 -0
  564. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_bsd.go +26 -0
  565. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go +26 -0
  566. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_supported.go +30 -0
  567. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unix.go +20 -0
  568. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go +16 -0
  569. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go +163 -0
  570. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go +318 -0
  571. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go +17 -0
  572. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go +55 -0
  573. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s +8 -0
  574. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go +57 -0
  575. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go +12 -0
  576. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go +224 -0
  577. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go +117 -0
  578. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_unix.go +16 -0
  579. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_unsupported.go +16 -0
  580. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go +55 -0
  581. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go +186 -0
  582. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go +76 -0
  583. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go +48 -0
  584. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go +79 -0
  585. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go +80 -0
  586. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go +16 -0
  587. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_supported.go +40 -0
  588. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go +40 -0
  589. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go +118 -0
  590. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go +13 -0
  591. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go +20 -0
  592. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go +13 -0
  593. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go +6 -0
  594. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_unix.go +17 -0
  595. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_unsupported.go +14 -0
  596. package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go +25 -0
  597. package/govendor/github.com/tetratelabs/wazero/internal/u32/u32.go +11 -0
  598. package/govendor/github.com/tetratelabs/wazero/internal/u64/u64.go +15 -0
  599. package/govendor/github.com/tetratelabs/wazero/internal/version/version.go +52 -0
  600. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/args.go +6 -0
  601. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/clock.go +16 -0
  602. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/environ.go +6 -0
  603. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/errno.go +314 -0
  604. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/fs.go +164 -0
  605. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/poll.go +15 -0
  606. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/proc.go +6 -0
  607. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/random.go +3 -0
  608. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/rights.go +148 -0
  609. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/sched.go +3 -0
  610. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/sock.go +71 -0
  611. package/govendor/github.com/tetratelabs/wazero/internal/wasip1/wasi.go +26 -0
  612. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go +124 -0
  613. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go +116 -0
  614. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go +22 -0
  615. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go +79 -0
  616. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go +235 -0
  617. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go +250 -0
  618. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go +11 -0
  619. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go +32 -0
  620. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go +56 -0
  621. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go +50 -0
  622. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go +9 -0
  623. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go +68 -0
  624. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go +47 -0
  625. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go +42 -0
  626. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go +151 -0
  627. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go +309 -0
  628. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go +79 -0
  629. package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go +97 -0
  630. package/govendor/github.com/tetratelabs/wazero/internal/wasm/const_expr.go +259 -0
  631. package/govendor/github.com/tetratelabs/wazero/internal/wasm/counts.go +53 -0
  632. package/govendor/github.com/tetratelabs/wazero/internal/wasm/engine.go +76 -0
  633. package/govendor/github.com/tetratelabs/wazero/internal/wasm/exception.go +9 -0
  634. package/govendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go +2804 -0
  635. package/govendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go +210 -0
  636. package/govendor/github.com/tetratelabs/wazero/internal/wasm/global.go +55 -0
  637. package/govendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go +279 -0
  638. package/govendor/github.com/tetratelabs/wazero/internal/wasm/host.go +179 -0
  639. package/govendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go +1962 -0
  640. package/govendor/github.com/tetratelabs/wazero/internal/wasm/memory.go +475 -0
  641. package/govendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go +128 -0
  642. package/govendor/github.com/tetratelabs/wazero/internal/wasm/module.go +1359 -0
  643. package/govendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go +251 -0
  644. package/govendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go +73 -0
  645. package/govendor/github.com/tetratelabs/wazero/internal/wasm/store.go +726 -0
  646. package/govendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go +95 -0
  647. package/govendor/github.com/tetratelabs/wazero/internal/wasm/table.go +321 -0
  648. package/govendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go +170 -0
  649. package/govendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go +225 -0
  650. package/govendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go +54 -0
  651. package/govendor/github.com/tetratelabs/wazero/netlify.toml +15 -0
  652. package/govendor/github.com/tetratelabs/wazero/runtime.go +399 -0
  653. package/govendor/github.com/tetratelabs/wazero/sys/clock.go +26 -0
  654. package/govendor/github.com/tetratelabs/wazero/sys/error.go +83 -0
  655. package/govendor/github.com/tetratelabs/wazero/sys/stat.go +105 -0
  656. package/govendor/github.com/tetratelabs/wazero/sys/stat_bsd.go +29 -0
  657. package/govendor/github.com/tetratelabs/wazero/sys/stat_posix.go +29 -0
  658. package/govendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go +14 -0
  659. package/govendor/github.com/tetratelabs/wazero/sys/stat_windows.go +24 -0
  660. package/govendor/golang.org/x/crypto/LICENSE +27 -0
  661. package/govendor/golang.org/x/crypto/PATENTS +22 -0
  662. package/govendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go +46 -0
  663. package/govendor/golang.org/x/crypto/cryptobyte/asn1.go +825 -0
  664. package/govendor/golang.org/x/crypto/cryptobyte/builder.go +350 -0
  665. package/govendor/golang.org/x/crypto/cryptobyte/string.go +183 -0
  666. package/govendor/golang.org/x/crypto/go.mod +3 -0
  667. package/govendor/golang.org/x/crypto/pbkdf2/pbkdf2.go +30 -0
  668. package/govendor/golang.org/x/net/LICENSE +27 -0
  669. package/govendor/golang.org/x/net/PATENTS +22 -0
  670. package/govendor/golang.org/x/net/dns/dnsmessage/message.go +2733 -0
  671. package/govendor/golang.org/x/net/dns/dnsmessage/svcb.go +333 -0
  672. package/govendor/golang.org/x/net/go.mod +3 -0
  673. package/govendor/golang.org/x/net/http/httpguts/guts.go +50 -0
  674. package/govendor/golang.org/x/net/http/httpguts/httplex.go +347 -0
  675. package/govendor/golang.org/x/net/http2/README.md +19 -0
  676. package/govendor/golang.org/x/net/http2/ascii.go +65 -0
  677. package/govendor/golang.org/x/net/http2/ciphers.go +641 -0
  678. package/govendor/golang.org/x/net/http2/client_conn_pool.go +301 -0
  679. package/govendor/golang.org/x/net/http2/client_priority_go126.go +20 -0
  680. package/govendor/golang.org/x/net/http2/client_priority_go127.go +13 -0
  681. package/govendor/golang.org/x/net/http2/clientconn.go +57 -0
  682. package/govendor/golang.org/x/net/http2/config.go +171 -0
  683. package/govendor/golang.org/x/net/http2/config_go125.go +15 -0
  684. package/govendor/golang.org/x/net/http2/config_go126.go +15 -0
  685. package/govendor/golang.org/x/net/http2/databuffer.go +146 -0
  686. package/govendor/golang.org/x/net/http2/errors.go +145 -0
  687. package/govendor/golang.org/x/net/http2/flow.go +175 -0
  688. package/govendor/golang.org/x/net/http2/frame.go +1887 -0
  689. package/govendor/golang.org/x/net/http2/gotrack.go +181 -0
  690. package/govendor/golang.org/x/net/http2/hpack/encode.go +244 -0
  691. package/govendor/golang.org/x/net/http2/hpack/hpack.go +522 -0
  692. package/govendor/golang.org/x/net/http2/hpack/huffman.go +226 -0
  693. package/govendor/golang.org/x/net/http2/hpack/static_table.go +188 -0
  694. package/govendor/golang.org/x/net/http2/hpack/tables.go +426 -0
  695. package/govendor/golang.org/x/net/http2/http2.go +415 -0
  696. package/govendor/golang.org/x/net/http2/pipe.go +184 -0
  697. package/govendor/golang.org/x/net/http2/server.go +3239 -0
  698. package/govendor/golang.org/x/net/http2/server_common.go +268 -0
  699. package/govendor/golang.org/x/net/http2/server_wrap.go +217 -0
  700. package/govendor/golang.org/x/net/http2/transport.go +3084 -0
  701. package/govendor/golang.org/x/net/http2/transport_common.go +516 -0
  702. package/govendor/golang.org/x/net/http2/transport_wrap.go +583 -0
  703. package/govendor/golang.org/x/net/http2/unencrypted.go +32 -0
  704. package/govendor/golang.org/x/net/http2/write.go +381 -0
  705. package/govendor/golang.org/x/net/http2/writesched.go +253 -0
  706. package/govendor/golang.org/x/net/http2/writesched_common.go +90 -0
  707. package/govendor/golang.org/x/net/http2/writesched_priority_rfc7540.go +420 -0
  708. package/govendor/golang.org/x/net/http2/writesched_priority_rfc9218.go +226 -0
  709. package/govendor/golang.org/x/net/http2/writesched_random.go +81 -0
  710. package/govendor/golang.org/x/net/http2/writesched_roundrobin.go +121 -0
  711. package/govendor/golang.org/x/net/idna/idna.go +884 -0
  712. package/govendor/golang.org/x/net/idna/punycode.go +220 -0
  713. package/govendor/golang.org/x/net/idna/tables15.0.0.go +5144 -0
  714. package/govendor/golang.org/x/net/idna/tables17.0.0.go +5302 -0
  715. package/govendor/golang.org/x/net/idna/trie.go +51 -0
  716. package/govendor/golang.org/x/net/idna/trieval.go +119 -0
  717. package/govendor/golang.org/x/net/internal/httpcommon/README.md +7 -0
  718. package/govendor/golang.org/x/net/internal/httpcommon/ascii.go +53 -0
  719. package/govendor/golang.org/x/net/internal/httpcommon/gzip.go +132 -0
  720. package/govendor/golang.org/x/net/internal/httpcommon/headermap.go +115 -0
  721. package/govendor/golang.org/x/net/internal/httpcommon/request.go +503 -0
  722. package/govendor/golang.org/x/net/internal/httpsfv/httpsfv.go +665 -0
  723. package/govendor/golang.org/x/sys/cpu/asm_aix_ppc64.s +17 -0
  724. package/govendor/golang.org/x/sys/cpu/asm_darwin_arm64_gc.s +12 -0
  725. package/govendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s +17 -0
  726. package/govendor/golang.org/x/sys/cpu/byteorder.go +66 -0
  727. package/govendor/golang.org/x/sys/cpu/cpu.go +357 -0
  728. package/govendor/golang.org/x/sys/cpu/cpu_aix.go +33 -0
  729. package/govendor/golang.org/x/sys/cpu/cpu_arm.go +73 -0
  730. package/govendor/golang.org/x/sys/cpu/cpu_arm64.go +191 -0
  731. package/govendor/golang.org/x/sys/cpu/cpu_arm64.s +35 -0
  732. package/govendor/golang.org/x/sys/cpu/cpu_darwin_arm64.go +67 -0
  733. package/govendor/golang.org/x/sys/cpu/cpu_darwin_arm64_other.go +31 -0
  734. package/govendor/golang.org/x/sys/cpu/cpu_darwin_x86.go +61 -0
  735. package/govendor/golang.org/x/sys/cpu/cpu_gc_arm64.go +12 -0
  736. package/govendor/golang.org/x/sys/cpu/cpu_gc_riscv64.go +11 -0
  737. package/govendor/golang.org/x/sys/cpu/cpu_gc_s390x.go +21 -0
  738. package/govendor/golang.org/x/sys/cpu/cpu_gc_x86.go +15 -0
  739. package/govendor/golang.org/x/sys/cpu/cpu_gc_x86.s +26 -0
  740. package/govendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go +12 -0
  741. package/govendor/golang.org/x/sys/cpu/cpu_gccgo_s390x.go +22 -0
  742. package/govendor/golang.org/x/sys/cpu/cpu_gccgo_x86.c +37 -0
  743. package/govendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go +25 -0
  744. package/govendor/golang.org/x/sys/cpu/cpu_linux.go +15 -0
  745. package/govendor/golang.org/x/sys/cpu/cpu_linux_arm.go +39 -0
  746. package/govendor/golang.org/x/sys/cpu/cpu_linux_arm64.go +120 -0
  747. package/govendor/golang.org/x/sys/cpu/cpu_linux_loong64.go +22 -0
  748. package/govendor/golang.org/x/sys/cpu/cpu_linux_mips64x.go +22 -0
  749. package/govendor/golang.org/x/sys/cpu/cpu_linux_noinit.go +9 -0
  750. package/govendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go +34 -0
  751. package/govendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go +173 -0
  752. package/govendor/golang.org/x/sys/cpu/cpu_linux_s390x.go +40 -0
  753. package/govendor/golang.org/x/sys/cpu/cpu_loong64.go +62 -0
  754. package/govendor/golang.org/x/sys/cpu/cpu_loong64.s +13 -0
  755. package/govendor/golang.org/x/sys/cpu/cpu_mips64x.go +15 -0
  756. package/govendor/golang.org/x/sys/cpu/cpu_mipsx.go +11 -0
  757. package/govendor/golang.org/x/sys/cpu/cpu_netbsd_amd64.go +47 -0
  758. package/govendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go +173 -0
  759. package/govendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.go +65 -0
  760. package/govendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.s +11 -0
  761. package/govendor/golang.org/x/sys/cpu/cpu_other_arm.go +9 -0
  762. package/govendor/golang.org/x/sys/cpu/cpu_other_arm64.go +11 -0
  763. package/govendor/golang.org/x/sys/cpu/cpu_other_mips64x.go +11 -0
  764. package/govendor/golang.org/x/sys/cpu/cpu_other_ppc64x.go +12 -0
  765. package/govendor/golang.org/x/sys/cpu/cpu_other_riscv64.go +11 -0
  766. package/govendor/golang.org/x/sys/cpu/cpu_other_x86.go +11 -0
  767. package/govendor/golang.org/x/sys/cpu/cpu_ppc64x.go +16 -0
  768. package/govendor/golang.org/x/sys/cpu/cpu_riscv64.go +33 -0
  769. package/govendor/golang.org/x/sys/cpu/cpu_riscv64.s +17 -0
  770. package/govendor/golang.org/x/sys/cpu/cpu_s390x.go +172 -0
  771. package/govendor/golang.org/x/sys/cpu/cpu_s390x.s +57 -0
  772. package/govendor/golang.org/x/sys/cpu/cpu_sparc64.go +12 -0
  773. package/govendor/golang.org/x/sys/cpu/cpu_wasm.go +17 -0
  774. package/govendor/golang.org/x/sys/cpu/cpu_windows.go +26 -0
  775. package/govendor/golang.org/x/sys/cpu/cpu_windows_arm64.go +38 -0
  776. package/govendor/golang.org/x/sys/cpu/cpu_x86.go +259 -0
  777. package/govendor/golang.org/x/sys/cpu/cpu_zos.go +10 -0
  778. package/govendor/golang.org/x/sys/cpu/cpu_zos_s390x.go +25 -0
  779. package/govendor/golang.org/x/sys/cpu/endian_big.go +10 -0
  780. package/govendor/golang.org/x/sys/cpu/endian_little.go +10 -0
  781. package/govendor/golang.org/x/sys/cpu/hwcap_linux.go +71 -0
  782. package/govendor/golang.org/x/sys/cpu/parse.go +55 -0
  783. package/govendor/golang.org/x/sys/cpu/proc_cpuinfo_linux.go +53 -0
  784. package/govendor/golang.org/x/sys/cpu/runtime_auxv.go +16 -0
  785. package/govendor/golang.org/x/sys/cpu/runtime_auxv_go121.go +18 -0
  786. package/govendor/golang.org/x/sys/cpu/syscall_aix_gccgo.go +26 -0
  787. package/govendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go +35 -0
  788. package/govendor/golang.org/x/sys/cpu/syscall_darwin_arm64_gc.go +54 -0
  789. package/govendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go +98 -0
  790. package/govendor/golang.org/x/sys/cpu/zcpu_windows.go +48 -0
  791. package/govendor/golang.org/x/sys/unix/asm_linux_ppc64x.s +20 -0
  792. package/govendor/golang.org/x/sys/unix/asm_linux_sparc64.s +48 -0
  793. package/govendor/golang.org/x/sys/unix/asm_zos_s390x.s +4 -186
  794. package/govendor/golang.org/x/sys/unix/bpxsvc_zos.s +9 -9
  795. package/govendor/golang.org/x/sys/unix/fcntl_darwin.go +2 -2
  796. package/govendor/golang.org/x/sys/unix/mkerrors.sh +1 -1
  797. package/govendor/golang.org/x/sys/unix/ptrace_zos.go +537 -0
  798. package/govendor/golang.org/x/sys/unix/ptrace_zos_singlestep.go +425 -0
  799. package/govendor/golang.org/x/sys/unix/syscall_darwin.go +1 -0
  800. package/govendor/golang.org/x/sys/unix/syscall_freebsd.go +9 -0
  801. package/govendor/golang.org/x/sys/unix/syscall_linux.go +20 -1
  802. package/govendor/golang.org/x/sys/unix/syscall_linux_sparc64.go +3 -3
  803. package/govendor/golang.org/x/sys/unix/syscall_netbsd.go +9 -0
  804. package/govendor/golang.org/x/sys/unix/syscall_unix_gc.go +1 -1
  805. package/govendor/golang.org/x/sys/unix/syscall_zos_s390x.go +0 -4
  806. package/govendor/golang.org/x/sys/unix/xattr_bsd.go +12 -12
  807. package/govendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go +6 -0
  808. package/govendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go +6 -0
  809. package/govendor/golang.org/x/sys/unix/zerrors_linux.go +32 -0
  810. package/govendor/golang.org/x/sys/unix/zerrors_linux_386.go +2 -0
  811. package/govendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +2 -0
  812. package/govendor/golang.org/x/sys/unix/zerrors_linux_arm.go +2 -0
  813. package/govendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +2 -0
  814. package/govendor/golang.org/x/sys/unix/zerrors_linux_loong64.go +2 -0
  815. package/govendor/golang.org/x/sys/unix/zerrors_linux_mips.go +2 -0
  816. package/govendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +2 -0
  817. package/govendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +2 -0
  818. package/govendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +2 -0
  819. package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc.go +2 -0
  820. package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +2 -0
  821. package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +2 -0
  822. package/govendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +2 -0
  823. package/govendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +2 -0
  824. package/govendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +2 -0
  825. package/govendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go +11 -0
  826. package/govendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go +11 -0
  827. package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go +159 -0
  828. package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go +159 -0
  829. package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go +159 -0
  830. package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go +159 -0
  831. package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go +159 -0
  832. package/govendor/golang.org/x/sys/unix/zsyscall_linux.go +28 -1
  833. package/govendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go +3 -3
  834. package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go +159 -0
  835. package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go +159 -0
  836. package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go +159 -0
  837. package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go +159 -0
  838. package/govendor/golang.org/x/sys/unix/zsysnum_zos_s390x.go +13 -0
  839. package/govendor/golang.org/x/sys/unix/ztypes_linux.go +65 -36
  840. package/govendor/golang.org/x/sys/unix/ztypes_linux_386.go +6 -0
  841. package/govendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +7 -0
  842. package/govendor/golang.org/x/sys/unix/ztypes_linux_arm.go +6 -0
  843. package/govendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +7 -0
  844. package/govendor/golang.org/x/sys/unix/ztypes_linux_loong64.go +7 -0
  845. package/govendor/golang.org/x/sys/unix/ztypes_linux_mips.go +6 -0
  846. package/govendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +7 -0
  847. package/govendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +7 -0
  848. package/govendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go +6 -0
  849. package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc.go +6 -0
  850. package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +7 -0
  851. package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +7 -0
  852. package/govendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go +7 -0
  853. package/govendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +7 -0
  854. package/govendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go +10 -5
  855. package/govendor/golang.org/x/sys/unix/ztypes_zos_s390x.go +78 -0
  856. package/govendor/golang.org/x/text/.gitattributes +10 -0
  857. package/govendor/golang.org/x/text/CONTRIBUTING.md +26 -0
  858. package/govendor/golang.org/x/text/LICENSE +27 -0
  859. package/govendor/golang.org/x/text/PATENTS +22 -0
  860. package/govendor/golang.org/x/text/README.md +81 -0
  861. package/govendor/golang.org/x/text/cases/cases.go +162 -0
  862. package/govendor/golang.org/x/text/cases/context.go +376 -0
  863. package/govendor/golang.org/x/text/cases/fold.go +34 -0
  864. package/govendor/golang.org/x/text/cases/icu.go +61 -0
  865. package/govendor/golang.org/x/text/cases/info.go +82 -0
  866. package/govendor/golang.org/x/text/cases/map.go +816 -0
  867. package/govendor/golang.org/x/text/cases/tables15.0.0.go +2527 -0
  868. package/govendor/golang.org/x/text/cases/tables17.0.0.go +2642 -0
  869. package/govendor/golang.org/x/text/cases/trieval.go +217 -0
  870. package/govendor/golang.org/x/text/codereview.cfg +1 -0
  871. package/govendor/golang.org/x/text/collate/collate.go +402 -0
  872. package/govendor/golang.org/x/text/collate/index.go +32 -0
  873. package/govendor/golang.org/x/text/collate/option.go +239 -0
  874. package/govendor/golang.org/x/text/collate/sort.go +81 -0
  875. package/govendor/golang.org/x/text/collate/tables.go +73789 -0
  876. package/govendor/golang.org/x/text/currency/common.go +67 -0
  877. package/govendor/golang.org/x/text/currency/currency.go +185 -0
  878. package/govendor/golang.org/x/text/currency/format.go +220 -0
  879. package/govendor/golang.org/x/text/currency/query.go +152 -0
  880. package/govendor/golang.org/x/text/currency/tables.go +2629 -0
  881. package/govendor/golang.org/x/text/doc.go +16 -0
  882. package/govendor/golang.org/x/text/feature/plural/common.go +70 -0
  883. package/govendor/golang.org/x/text/feature/plural/message.go +244 -0
  884. package/govendor/golang.org/x/text/feature/plural/plural.go +262 -0
  885. package/govendor/golang.org/x/text/feature/plural/tables.go +552 -0
  886. package/govendor/golang.org/x/text/go.mod +3 -0
  887. package/govendor/golang.org/x/text/internal/catmsg/catmsg.go +417 -0
  888. package/govendor/golang.org/x/text/internal/catmsg/codec.go +407 -0
  889. package/govendor/golang.org/x/text/internal/catmsg/varint.go +62 -0
  890. package/govendor/golang.org/x/text/internal/colltab/collelem.go +374 -0
  891. package/govendor/golang.org/x/text/internal/colltab/colltab.go +105 -0
  892. package/govendor/golang.org/x/text/internal/colltab/contract.go +145 -0
  893. package/govendor/golang.org/x/text/internal/colltab/iter.go +178 -0
  894. package/govendor/golang.org/x/text/internal/colltab/numeric.go +236 -0
  895. package/govendor/golang.org/x/text/internal/colltab/table.go +275 -0
  896. package/govendor/golang.org/x/text/internal/colltab/trie.go +159 -0
  897. package/govendor/golang.org/x/text/internal/colltab/weighter.go +31 -0
  898. package/govendor/golang.org/x/text/internal/format/format.go +41 -0
  899. package/govendor/golang.org/x/text/internal/format/parser.go +358 -0
  900. package/govendor/golang.org/x/text/internal/internal.go +49 -0
  901. package/govendor/golang.org/x/text/internal/language/common.go +16 -0
  902. package/govendor/golang.org/x/text/internal/language/compact/compact.go +61 -0
  903. package/govendor/golang.org/x/text/internal/language/compact/language.go +260 -0
  904. package/govendor/golang.org/x/text/internal/language/compact/parents.go +120 -0
  905. package/govendor/golang.org/x/text/internal/language/compact/tables.go +1015 -0
  906. package/govendor/golang.org/x/text/internal/language/compact/tags.go +91 -0
  907. package/govendor/golang.org/x/text/internal/language/compact.go +29 -0
  908. package/govendor/golang.org/x/text/internal/language/compose.go +167 -0
  909. package/govendor/golang.org/x/text/internal/language/coverage.go +28 -0
  910. package/govendor/golang.org/x/text/internal/language/language.go +627 -0
  911. package/govendor/golang.org/x/text/internal/language/lookup.go +412 -0
  912. package/govendor/golang.org/x/text/internal/language/match.go +226 -0
  913. package/govendor/golang.org/x/text/internal/language/parse.go +608 -0
  914. package/govendor/golang.org/x/text/internal/language/tables.go +3494 -0
  915. package/govendor/golang.org/x/text/internal/language/tags.go +48 -0
  916. package/govendor/golang.org/x/text/internal/match.go +67 -0
  917. package/govendor/golang.org/x/text/internal/number/common.go +55 -0
  918. package/govendor/golang.org/x/text/internal/number/decimal.go +500 -0
  919. package/govendor/golang.org/x/text/internal/number/format.go +533 -0
  920. package/govendor/golang.org/x/text/internal/number/number.go +152 -0
  921. package/govendor/golang.org/x/text/internal/number/pattern.go +485 -0
  922. package/govendor/golang.org/x/text/internal/number/roundingmode_string.go +30 -0
  923. package/govendor/golang.org/x/text/internal/number/tables.go +1219 -0
  924. package/govendor/golang.org/x/text/internal/stringset/set.go +86 -0
  925. package/govendor/golang.org/x/text/internal/tag/tag.go +100 -0
  926. package/govendor/golang.org/x/text/language/coverage.go +187 -0
  927. package/govendor/golang.org/x/text/language/display/dict.go +92 -0
  928. package/govendor/golang.org/x/text/language/display/display.go +420 -0
  929. package/govendor/golang.org/x/text/language/display/lookup.go +253 -0
  930. package/govendor/golang.org/x/text/language/display/tables.go +53114 -0
  931. package/govendor/golang.org/x/text/language/doc.go +98 -0
  932. package/govendor/golang.org/x/text/language/language.go +605 -0
  933. package/govendor/golang.org/x/text/language/match.go +735 -0
  934. package/govendor/golang.org/x/text/language/parse.go +258 -0
  935. package/govendor/golang.org/x/text/language/tables.go +298 -0
  936. package/govendor/golang.org/x/text/language/tags.go +145 -0
  937. package/govendor/golang.org/x/text/message/catalog/catalog.go +365 -0
  938. package/govendor/golang.org/x/text/message/catalog/dict.go +133 -0
  939. package/govendor/golang.org/x/text/secure/bidirule/bidirule.go +346 -0
  940. package/govendor/golang.org/x/text/transform/transform.go +709 -0
  941. package/govendor/golang.org/x/text/unicode/bidi/bidi.go +359 -0
  942. package/govendor/golang.org/x/text/unicode/bidi/bracket.go +335 -0
  943. package/govendor/golang.org/x/text/unicode/bidi/core.go +1050 -0
  944. package/govendor/golang.org/x/text/unicode/bidi/prop.go +206 -0
  945. package/govendor/golang.org/x/text/unicode/bidi/tables15.0.0.go +2042 -0
  946. package/govendor/golang.org/x/text/unicode/bidi/tables17.0.0.go +2135 -0
  947. package/govendor/golang.org/x/text/unicode/bidi/trieval.go +48 -0
  948. package/govendor/golang.org/x/text/unicode/norm/composition.go +524 -0
  949. package/govendor/golang.org/x/text/unicode/norm/forminfo.go +296 -0
  950. package/govendor/golang.org/x/text/unicode/norm/input.go +109 -0
  951. package/govendor/golang.org/x/text/unicode/norm/iter.go +454 -0
  952. package/govendor/golang.org/x/text/unicode/norm/normalize.go +613 -0
  953. package/govendor/golang.org/x/text/unicode/norm/readwriter.go +125 -0
  954. package/govendor/golang.org/x/text/unicode/norm/tables15.0.0.go +7907 -0
  955. package/govendor/golang.org/x/text/unicode/norm/tables17.0.0.go +8104 -0
  956. package/govendor/golang.org/x/text/unicode/norm/transform.go +89 -0
  957. package/govendor/golang.org/x/text/unicode/norm/trie.go +54 -0
  958. package/govendor/manifest.txt +11 -2
  959. package/govendor/software.sslmate.com/src/go-pkcs12/.gitattributes +10 -0
  960. package/govendor/software.sslmate.com/src/go-pkcs12/LICENSE +28 -0
  961. package/govendor/software.sslmate.com/src/go-pkcs12/README.md +31 -0
  962. package/govendor/software.sslmate.com/src/go-pkcs12/bmp-string.go +63 -0
  963. package/govendor/software.sslmate.com/src/go-pkcs12/crypto.go +331 -0
  964. package/govendor/software.sslmate.com/src/go-pkcs12/errors.go +23 -0
  965. package/govendor/software.sslmate.com/src/go-pkcs12/go.mod +3 -0
  966. package/govendor/software.sslmate.com/src/go-pkcs12/internal/rc2/rc2.go +271 -0
  967. package/govendor/software.sslmate.com/src/go-pkcs12/mac.go +202 -0
  968. package/govendor/software.sslmate.com/src/go-pkcs12/pbkdf.go +184 -0
  969. package/govendor/software.sslmate.com/src/go-pkcs12/pkcs12.go +1000 -0
  970. package/govendor/software.sslmate.com/src/go-pkcs12/safebags.go +107 -0
  971. package/package.json +11 -1
  972. package/types/tsconfig.json +17 -0
  973. package/types/typenative-npm.d.ts +1327 -80
  974. package/types/typenative.d.ts +1312 -25
  975. package/.gitattributes +0 -1
  976. package/.github/workflows/ci.yml +0 -68
  977. package/.github/workflows/npm-publish-github-packages.yml +0 -88
  978. package/CHANGELOG.md +0 -1358
  979. package/CONTRIBUTION.md +0 -133
  980. package/conformance.md +0 -1450
  981. package/examples/README.md +0 -113
  982. package/examples/angular-templates/README.md +0 -141
  983. package/examples/angular-templates/bench.mjs +0 -173
  984. package/examples/angular-templates/dom-type-globals.d.ts +0 -21
  985. package/examples/angular-templates/dom-type-stubs.ts +0 -21
  986. package/examples/angular-templates/main.ts +0 -135
  987. package/examples/angular-templates/parity.mjs +0 -157
  988. package/examples/angular-templates/run-node.mjs +0 -69
  989. package/examples/angular-templates/vendor/angular/compiler/LICENSE +0 -21
  990. package/examples/desktop-notes/README.md +0 -106
  991. package/examples/desktop-notes/bench.mjs +0 -125
  992. package/examples/desktop-notes/main.ts +0 -131
  993. package/examples/desktop-notes/package-lock.json +0 -726
  994. package/examples/desktop-notes/package.json +0 -9
  995. package/examples/desktop-notes/ui.ts +0 -176
  996. package/examples/dupe-finder/README.md +0 -78
  997. package/examples/dupe-finder/bench.mjs +0 -120
  998. package/examples/dupe-finder/main.ts +0 -87
  999. package/examples/fibonacci/README.md +0 -89
  1000. package/examples/fibonacci/bench.mjs +0 -87
  1001. package/examples/fibonacci/main.ts +0 -21
  1002. package/examples/hello-world/README.md +0 -62
  1003. package/examples/hello-world/bench.mjs +0 -66
  1004. package/examples/hello-world/main.ts +0 -7
  1005. package/examples/json-server/README.md +0 -131
  1006. package/examples/json-server/bench.mjs +0 -263
  1007. package/examples/json-server/main.ts +0 -67
  1008. package/examples/mandelbrot-png/README.md +0 -63
  1009. package/examples/mandelbrot-png/bench.mjs +0 -95
  1010. package/examples/mandelbrot-png/main.ts +0 -98
  1011. package/examples/mandelbrot-png/preview.png +0 -0
  1012. package/examples/marked-cli/README.md +0 -175
  1013. package/examples/marked-cli/bench.mjs +0 -178
  1014. package/examples/marked-cli/main.ts +0 -410
  1015. package/examples/marked-cli/parity.mjs +0 -185
  1016. package/examples/marked-cli/vendor/marked/LICENSE +0 -44
  1017. package/examples/marked-cli/vendor/marked/VENDOR.md +0 -26
  1018. package/examples/openai-chat/README.md +0 -148
  1019. package/examples/openai-chat/bench.mjs +0 -175
  1020. package/examples/openai-chat/main.ts +0 -488
  1021. package/examples/react-compiler/README.md +0 -185
  1022. package/examples/react-compiler/bench.mjs +0 -181
  1023. package/examples/react-compiler/main.ts +0 -324
  1024. package/examples/react-compiler/parity.mjs +0 -362
  1025. package/examples/react-compiler/run-node.mjs +0 -90
  1026. package/examples/react-compiler/vendor/babel/LICENSE +0 -22
  1027. package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-lower.json +0 -15
  1028. package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-upper.json +0 -51
  1029. package/examples/react-compiler/vendor/charcodes/package.json +0 -25
  1030. package/examples/react-compiler/vendor/invariant/LICENSE +0 -21
  1031. package/examples/react-compiler/vendor/invariant/invariant.cjs +0 -51
  1032. package/examples/react-compiler/vendor/invariant/invariant.d.cts +0 -11
  1033. package/examples/react-compiler/vendor/invariant/types/LICENSE +0 -21
  1034. package/examples/react-compiler/vendor/jridgewell/gen-mapping/LICENSE +0 -19
  1035. package/examples/react-compiler/vendor/jridgewell/resolve-uri/LICENSE +0 -19
  1036. package/examples/react-compiler/vendor/jridgewell/sourcemap-codec/LICENSE +0 -19
  1037. package/examples/react-compiler/vendor/jridgewell/trace-mapping/LICENSE +0 -19
  1038. package/examples/react-compiler/vendor/js-tokens/LICENSE +0 -21
  1039. package/examples/react-compiler/vendor/js-tokens/index.cjs +0 -23
  1040. package/examples/react-compiler/vendor/js-tokens/index.d.cts +0 -13
  1041. package/examples/react-compiler/vendor/jsesc/jsesc.cjs +0 -337
  1042. package/examples/react-compiler/vendor/jsesc/jsesc.d.cts +0 -384
  1043. package/examples/react-compiler/vendor/jsesc/types/LICENSE +0 -21
  1044. package/examples/react-compiler/vendor/picocolors/picocolors.cjs +0 -75
  1045. package/examples/react-compiler/vendor/picocolors/picocolors.d.cts +0 -5
  1046. package/examples/react-compiler/vendor/picocolors/types.d.ts +0 -51
  1047. package/examples/react-compiler/vendor/react-compiler/LICENSE +0 -21
  1048. package/examples/react-compiler/vendor/stubs/babel-core.ts +0 -31
  1049. package/examples/react-compiler/vendor/stubs/debug.ts +0 -18
  1050. package/examples/react-compiler/vendor/stubs/js-tokens.d.ts +0 -13
  1051. package/examples/react-compiler/vendor/stubs/pretty-format.ts +0 -18
  1052. package/examples/react-compiler/vendor/zod/LICENSE +0 -21
  1053. package/examples/react-compiler/vendor/zod-validation-error/LICENSE +0 -9
  1054. package/examples/react-compiler/vendor/zod-validation-error/lib/utils/NonEmptyArray.ts +0 -5
  1055. package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinPath.ts +0 -49
  1056. package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinValues.ts +0 -48
  1057. package/examples/react-compiler/vendor/zod-validation-error/lib/utils/stringify.ts +0 -50
  1058. package/examples/react-compiler/vendor/zod-validation-error/lib/utils/titleCase.ts +0 -6
  1059. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/MessageBuilder.ts +0 -126
  1060. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/ValidationError.ts +0 -39
  1061. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/custom.ts +0 -12
  1062. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/errorMap.ts +0 -78
  1063. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/index.ts +0 -2
  1064. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidElement.ts +0 -12
  1065. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidKey.ts +0 -12
  1066. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidStringFormat.ts +0 -131
  1067. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidType.ts +0 -54
  1068. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidUnion.ts +0 -12
  1069. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidValue.ts +0 -40
  1070. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/notMultipleOf.ts +0 -12
  1071. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooBig.ts +0 -76
  1072. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooSmall.ts +0 -76
  1073. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/types.ts +0 -27
  1074. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/unrecognizedKeys.ts +0 -27
  1075. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromError.ts +0 -13
  1076. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodError.ts +0 -61
  1077. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodIssue.ts +0 -38
  1078. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/index.ts +0 -20
  1079. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationError.ts +0 -5
  1080. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationErrorLike.ts +0 -8
  1081. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isZodErrorLike.ts +0 -11
  1082. package/examples/react-compiler/vendor/zod-validation-error/lib/v4/toValidationError.ts +0 -20
  1083. package/examples/react-compiler/vendor.mjs +0 -688
  1084. package/examples/todo-cli/README.md +0 -69
  1085. package/examples/todo-cli/bench.mjs +0 -89
  1086. package/examples/todo-cli/main.ts +0 -92
  1087. package/examples/yaml-cli/README.md +0 -152
  1088. package/examples/yaml-cli/bench.mjs +0 -265
  1089. package/examples/yaml-cli/main.ts +0 -251
  1090. package/examples/yaml-cli/parity.mjs +0 -156
  1091. package/examples/yaml-cli/vendor/yaml/LICENSE +0 -13
  1092. package/examples/yaml-cli/vendor/yaml/VENDOR.md +0 -25
  1093. package/govendor/golang.org/x/sys/unix/syscall_unix_gc_ppc64x.go +0 -22
  1094. package/govendor/golang.org/x/sys/windows/types_windows_arm.go +0 -35
  1095. package/performance/README.md +0 -196
  1096. package/performance/any-dynamic.ts +0 -21
  1097. package/performance/any-record-view.ts +0 -20
  1098. package/performance/arr-hof.ts +0 -21
  1099. package/performance/arr-push-index.ts +0 -15
  1100. package/performance/arr-sort.ts +0 -21
  1101. package/performance/async-await.ts +0 -24
  1102. package/performance/class-dispatch.ts +0 -52
  1103. package/performance/events-emit.ts +0 -27
  1104. package/performance/exc-throw-rare.ts +0 -25
  1105. package/performance/exc-try-catch.ts +0 -26
  1106. package/performance/fn-closures.ts +0 -29
  1107. package/performance/fn-recursion-tree.ts +0 -31
  1108. package/performance/gen-for-of.ts +0 -46
  1109. package/performance/json-roundtrip.ts +0 -27
  1110. package/performance/map-delete.ts +0 -24
  1111. package/performance/map-string-keys.ts +0 -22
  1112. package/performance/num-arith-loop.ts +0 -14
  1113. package/performance/num-bitwise.ts +0 -21
  1114. package/performance/num-mod-int.ts +0 -16
  1115. package/performance/num-to-string.ts +0 -16
  1116. package/performance/obj-alloc.ts +0 -32
  1117. package/performance/regex-exec.ts +0 -29
  1118. package/performance/regex-site.ts +0 -28
  1119. package/performance/set-ops.ts +0 -20
  1120. package/performance/str-char-scan.ts +0 -35
  1121. package/performance/str-concat-plus.ts +0 -18
  1122. package/performance/str-concat.ts +0 -24
  1123. package/performance/str-short-scan.ts +0 -26
  1124. package/performance/str-split-join.ts +0 -23
  1125. package/performance/str-template.ts +0 -16
  1126. package/performance/union-switch.ts +0 -57
  1127. package/tsparser/check.go +0 -158
  1128. package/tsparser/go.mod +0 -14
  1129. package/tsparser/go.sum +0 -22
  1130. package/tsparser/main.go +0 -363
  1131. package/tsparser/tsparser.exe +0 -4
@@ -0,0 +1,3782 @@
1
+ package amd64
2
+
3
+ import (
4
+ "context"
5
+ "encoding/binary"
6
+ "fmt"
7
+ "math"
8
+ "strings"
9
+
10
+ "github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
11
+ "github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
12
+ "github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
13
+ "github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
14
+ "github.com/tetratelabs/wazero/internal/platform"
15
+ )
16
+
17
+ // NewBackend returns a new backend for arm64.
18
+ func NewBackend() backend.Machine {
19
+ m := &machine{
20
+ cpuFeatures: platform.CpuFeatures,
21
+ regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo),
22
+ spillSlots: map[regalloc.VRegID]int64{},
23
+ amodePool: wazevoapi.NewPool[amode](nil),
24
+ labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition),
25
+ instrPool: wazevoapi.NewPool[instruction](resetInstruction),
26
+ constSwizzleMaskConstIndex: -1,
27
+ constSqmulRoundSatIndex: -1,
28
+ constI8x16SHLMaskTableIndex: -1,
29
+ constI8x16LogicalSHRMaskTableIndex: -1,
30
+ constF64x2CvtFromIMaskIndex: -1,
31
+ constTwop52Index: -1,
32
+ constI32sMaxOnF64x2Index: -1,
33
+ constI32uMaxOnF64x2Index: -1,
34
+ constAllOnesI8x16Index: -1,
35
+ constAllOnesI16x8Index: -1,
36
+ constExtAddPairwiseI16x8uMask1Index: -1,
37
+ constExtAddPairwiseI16x8uMask2Index: -1,
38
+ }
39
+ m.regAllocFn.m = m
40
+ return m
41
+ }
42
+
43
+ type (
44
+ // machine implements backend.Machine for amd64.
45
+ machine struct {
46
+ c backend.Compiler
47
+ stackBoundsCheckDisabled bool
48
+
49
+ instrPool wazevoapi.Pool[instruction]
50
+ amodePool wazevoapi.Pool[amode]
51
+
52
+ cpuFeatures platform.CpuFeatureFlags
53
+
54
+ regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn]
55
+ regAllocFn regAllocFn
56
+ regAllocStarted bool
57
+
58
+ // labelPositionPool is the pool of labelPosition. The id is the label where
59
+ // if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID.
60
+ labelPositionPool wazevoapi.IDedPool[labelPosition]
61
+ // nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID
62
+ // so that we can have an identical label for the SSA block ID, which is useful for debugging.
63
+ nextLabel label
64
+ // rootInstr is the first instruction of the function.
65
+ rootInstr *instruction
66
+ // currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition.
67
+ currentLabelPos *labelPosition
68
+ // orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock.
69
+ orderedSSABlockLabelPos []*labelPosition
70
+ // returnLabelPos is the labelPosition for the return block.
71
+ returnLabelPos labelPosition
72
+ // perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock.
73
+ perBlockHead, perBlockEnd *instruction
74
+ // pendingInstructions are the instructions which are not yet emitted into the instruction list.
75
+ pendingInstructions []*instruction
76
+ // maxSSABlockID is the maximum ssa.BasicBlockID in the current function.
77
+ maxSSABlockID label
78
+
79
+ spillSlotSize int64
80
+ spillSlots map[regalloc.VRegID]int64
81
+ currentABI *backend.FunctionABI
82
+ clobberedRegs []regalloc.VReg
83
+
84
+ maxRequiredStackSizeForCalls int64
85
+
86
+ labelResolutionPends []labelResolutionPend
87
+
88
+ // jmpTableTargets holds the labels of the jump table targets.
89
+ jmpTableTargets [][]uint32
90
+ // jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table.
91
+ jmpTableTargetsNext int
92
+ consts []_const
93
+
94
+ constSwizzleMaskConstIndex, constSqmulRoundSatIndex,
95
+ constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex,
96
+ constF64x2CvtFromIMaskIndex, constTwop52Index,
97
+ constI32sMaxOnF64x2Index, constI32uMaxOnF64x2Index,
98
+ constAllOnesI8x16Index, constAllOnesI16x8Index,
99
+ constExtAddPairwiseI16x8uMask1Index, constExtAddPairwiseI16x8uMask2Index int
100
+ }
101
+
102
+ _const struct {
103
+ lo, hi uint64
104
+ _var []byte
105
+ label label
106
+ labelPos *labelPosition
107
+ }
108
+
109
+ labelResolutionPend struct {
110
+ instr *instruction
111
+ instrOffset int64
112
+ // imm32Offset is the offset of the last 4 bytes of the instruction.
113
+ imm32Offset int64
114
+ }
115
+ )
116
+
117
+ type (
118
+ // label represents a position in the generated code which is either
119
+ // a real instruction or the constant InstructionPool (e.g. jump tables).
120
+ //
121
+ // This is exactly the same as the traditional "label" in assembly code.
122
+ label uint32
123
+
124
+ // labelPosition represents the regions of the generated code which the label represents.
125
+ // This implements regalloc.Block.
126
+ labelPosition struct {
127
+ // sb is not nil if this corresponds to a ssa.BasicBlock.
128
+ sb ssa.BasicBlock
129
+ // cur is used to walk through the instructions in the block during the register allocation.
130
+ cur,
131
+ // begin and end are the first and last instructions of the block.
132
+ begin, end *instruction
133
+ // binaryOffset is the offset in the binary where the label is located.
134
+ binaryOffset int64
135
+ }
136
+ )
137
+
138
+ // String implements backend.Machine.
139
+ func (l label) String() string {
140
+ return fmt.Sprintf("L%d", l)
141
+ }
142
+
143
+ func resetLabelPosition(l *labelPosition) {
144
+ *l = labelPosition{}
145
+ }
146
+
147
+ const labelReturn = math.MaxUint32
148
+
149
+ func ssaBlockLabel(sb ssa.BasicBlock) label {
150
+ if sb.ReturnBlock() {
151
+ return labelReturn
152
+ }
153
+ return label(sb.ID())
154
+ }
155
+
156
+ // getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block.
157
+ func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition {
158
+ if sb.ReturnBlock() {
159
+ m.returnLabelPos.sb = sb
160
+ return &m.returnLabelPos
161
+ }
162
+
163
+ l := ssaBlockLabel(sb)
164
+ pos := m.labelPositionPool.GetOrAllocate(int(l))
165
+ pos.sb = sb
166
+ return pos
167
+ }
168
+
169
+ func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) label {
170
+ index := *i
171
+ if index == -1 {
172
+ l, pos := m.allocateLabel()
173
+ index = len(m.consts)
174
+ m.consts = append(m.consts, _const{
175
+ _var: _var,
176
+ label: l,
177
+ labelPos: pos,
178
+ })
179
+ *i = index
180
+ }
181
+ return m.consts[index].label
182
+ }
183
+
184
+ // Reset implements backend.Machine.
185
+ func (m *machine) Reset() {
186
+ m.consts = m.consts[:0]
187
+ m.clobberedRegs = m.clobberedRegs[:0]
188
+ for key := range m.spillSlots {
189
+ m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key))
190
+ }
191
+ for _, key := range m.clobberedRegs {
192
+ delete(m.spillSlots, regalloc.VRegID(key))
193
+ }
194
+
195
+ m.stackBoundsCheckDisabled = false
196
+ m.regAlloc.Reset()
197
+ m.labelPositionPool.Reset()
198
+ m.instrPool.Reset()
199
+ m.regAllocStarted = false
200
+ m.clobberedRegs = m.clobberedRegs[:0]
201
+
202
+ m.spillSlotSize = 0
203
+ m.maxRequiredStackSizeForCalls = 0
204
+ m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil
205
+ m.pendingInstructions = m.pendingInstructions[:0]
206
+ m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0]
207
+
208
+ m.amodePool.Reset()
209
+ m.jmpTableTargetsNext = 0
210
+ m.constSwizzleMaskConstIndex = -1
211
+ m.constSqmulRoundSatIndex = -1
212
+ m.constI8x16SHLMaskTableIndex = -1
213
+ m.constI8x16LogicalSHRMaskTableIndex = -1
214
+ m.constF64x2CvtFromIMaskIndex = -1
215
+ m.constTwop52Index = -1
216
+ m.constI32sMaxOnF64x2Index = -1
217
+ m.constI32uMaxOnF64x2Index = -1
218
+ m.constAllOnesI8x16Index = -1
219
+ m.constAllOnesI16x8Index = -1
220
+ m.constExtAddPairwiseI16x8uMask1Index = -1
221
+ m.constExtAddPairwiseI16x8uMask2Index = -1
222
+ }
223
+
224
+ // StartLoweringFunction implements backend.Machine StartLoweringFunction.
225
+ func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) {
226
+ m.maxSSABlockID = label(maxBlockID)
227
+ m.nextLabel = label(maxBlockID) + 1
228
+ }
229
+
230
+ // LinkAdjacentBlocks implements backend.Machine.
231
+ func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) {
232
+ prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next)
233
+ prevPos.end.next = nextPos.begin
234
+ }
235
+
236
+ // StartBlock implements backend.Machine.
237
+ func (m *machine) StartBlock(blk ssa.BasicBlock) {
238
+ m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk)
239
+ labelPos := m.currentLabelPos
240
+ end := m.allocateNop()
241
+ m.perBlockHead, m.perBlockEnd = end, end
242
+ labelPos.begin, labelPos.end = end, end
243
+ m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos)
244
+ }
245
+
246
+ // EndBlock implements ExecutableContext.
247
+ func (m *machine) EndBlock() {
248
+ // Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions.
249
+ m.insertAtPerBlockHead(m.allocateNop())
250
+
251
+ m.currentLabelPos.begin = m.perBlockHead
252
+
253
+ if m.currentLabelPos.sb.EntryBlock() {
254
+ m.rootInstr = m.perBlockHead
255
+ }
256
+ }
257
+
258
+ func (m *machine) insertAtPerBlockHead(i *instruction) {
259
+ if m.perBlockHead == nil {
260
+ m.perBlockHead = i
261
+ m.perBlockEnd = i
262
+ return
263
+ }
264
+
265
+ i.next = m.perBlockHead
266
+ m.perBlockHead.prev = i
267
+ m.perBlockHead = i
268
+ }
269
+
270
+ // FlushPendingInstructions implements backend.Machine.
271
+ func (m *machine) FlushPendingInstructions() {
272
+ l := len(m.pendingInstructions)
273
+ if l == 0 {
274
+ return
275
+ }
276
+ for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order.
277
+ m.insertAtPerBlockHead(m.pendingInstructions[i])
278
+ }
279
+ m.pendingInstructions = m.pendingInstructions[:0]
280
+ }
281
+
282
+ // DisableStackCheck implements backend.Machine.
283
+ func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true }
284
+
285
+ // SetCompiler implements backend.Machine.
286
+ func (m *machine) SetCompiler(c backend.Compiler) {
287
+ m.c = c
288
+ m.regAllocFn.ssaB = c.SSABuilder()
289
+ }
290
+
291
+ // SetCurrentABI implements backend.Machine.
292
+ func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { m.currentABI = abi }
293
+
294
+ // RegAlloc implements backend.Machine.
295
+ func (m *machine) RegAlloc() {
296
+ rf := m.regAllocFn
297
+ m.regAllocStarted = true
298
+ m.regAlloc.DoAllocation(&rf)
299
+ // Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes.
300
+ m.spillSlotSize = (m.spillSlotSize + 15) &^ 15
301
+ }
302
+
303
+ // InsertReturn implements backend.Machine.
304
+ func (m *machine) InsertReturn() {
305
+ i := m.allocateInstr().asRet()
306
+ m.insert(i)
307
+ }
308
+
309
+ // LowerSingleBranch implements backend.Machine.
310
+ func (m *machine) LowerSingleBranch(b *ssa.Instruction) {
311
+ switch b.Opcode() {
312
+ case ssa.OpcodeJump:
313
+ _, _, targetBlkID := b.BranchData()
314
+ if b.IsFallthroughJump() {
315
+ return
316
+ }
317
+ jmp := m.allocateInstr()
318
+ target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
319
+ if target == labelReturn {
320
+ jmp.asRet()
321
+ } else {
322
+ jmp.asJmp(newOperandLabel(target))
323
+ }
324
+ m.insert(jmp)
325
+ case ssa.OpcodeBrTable:
326
+ index, targetBlkIDs := b.BrTableData()
327
+ m.lowerBrTable(index, targetBlkIDs)
328
+ default:
329
+ panic("BUG: unexpected branch opcode" + b.Opcode().String())
330
+ }
331
+ }
332
+
333
+ func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) {
334
+ if m.jmpTableTargetsNext == len(m.jmpTableTargets) {
335
+ m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View())))
336
+ }
337
+
338
+ index = m.jmpTableTargetsNext
339
+ m.jmpTableTargetsNext++
340
+ m.jmpTableTargets[index] = m.jmpTableTargets[index][:0]
341
+ for _, targetBlockID := range targets.View() {
342
+ target := m.c.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID))
343
+ m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(ssaBlockLabel(target)))
344
+ }
345
+ return
346
+ }
347
+
348
+ var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp}
349
+
350
+ func (m *machine) lowerBrTable(index ssa.Value, targets ssa.Values) {
351
+ _v := m.getOperand_Reg(m.c.ValueDefinition(index))
352
+ v := m.copyToTmp(_v.reg())
353
+
354
+ targetCount := len(targets.View())
355
+
356
+ // First, we need to do the bounds check.
357
+ maxIndex := m.c.AllocateVReg(ssa.TypeI32)
358
+ m.lowerIconst(maxIndex, uint64(targetCount-1), false)
359
+ cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false)
360
+ m.insert(cmp)
361
+
362
+ // Then do the conditional move maxIndex to v if v > maxIndex.
363
+ cmov := m.allocateInstr().asCmove(condNB, newOperandReg(maxIndex), v, false)
364
+ m.insert(cmov)
365
+
366
+ // Now that v has the correct index. Load the address of the jump table into the addr.
367
+ addr := m.c.AllocateVReg(ssa.TypeI64)
368
+ leaJmpTableAddr := m.allocateInstr()
369
+ m.insert(leaJmpTableAddr)
370
+
371
+ // Then add the target's offset into jmpTableAddr.
372
+ loadTargetOffsetFromJmpTable := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd,
373
+ // Shift by 3 because each entry is 8 bytes.
374
+ newOperandMem(m.newAmodeRegRegShift(0, addr, v, 3)), addr, true)
375
+ m.insert(loadTargetOffsetFromJmpTable)
376
+
377
+ // Now ready to jump.
378
+ jmp := m.allocateInstr().asJmp(newOperandReg(addr))
379
+ m.insert(jmp)
380
+
381
+ jmpTableBegin, jmpTableBeginLabel := m.allocateBrTarget()
382
+ m.insert(jmpTableBegin)
383
+ leaJmpTableAddr.asLEA(newOperandLabel(jmpTableBeginLabel), addr)
384
+
385
+ jmpTable := m.allocateInstr()
386
+ targetSliceIndex := m.addJmpTableTarget(targets)
387
+ jmpTable.asJmpTableSequence(targetSliceIndex, targetCount)
388
+ m.insert(jmpTable)
389
+ }
390
+
391
+ // LowerConditionalBranch implements backend.Machine.
392
+ func (m *machine) LowerConditionalBranch(b *ssa.Instruction) {
393
+ cval, args, targetBlkID := b.BranchData()
394
+ if len(args) > 0 {
395
+ panic(fmt.Sprintf(
396
+ "conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s",
397
+ m.currentLabelPos.sb,
398
+ targetBlkID,
399
+ ))
400
+ }
401
+
402
+ target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
403
+ cvalDef := m.c.ValueDefinition(cval)
404
+
405
+ switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
406
+ case ssa.OpcodeIcmp:
407
+ cvalInstr := cvalDef.Instr
408
+ x, y, c := cvalInstr.IcmpData()
409
+
410
+ cc := condFromSSAIntCmpCond(c)
411
+ if b.Opcode() == ssa.OpcodeBrz {
412
+ cc = cc.invert()
413
+ }
414
+
415
+ // First, perform the comparison and set the flag.
416
+ xd, yd := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
417
+ if !m.tryLowerBandToFlag(xd, yd) {
418
+ m.lowerIcmpToFlag(xd, yd, x.Type() == ssa.TypeI64)
419
+ }
420
+
421
+ // Then perform the conditional branch.
422
+ m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
423
+ cvalDef.Instr.MarkLowered()
424
+ case ssa.OpcodeFcmp:
425
+ cvalInstr := cvalDef.Instr
426
+
427
+ f1, f2, and := m.lowerFcmpToFlags(cvalInstr)
428
+ isBrz := b.Opcode() == ssa.OpcodeBrz
429
+ if isBrz {
430
+ f1 = f1.invert()
431
+ }
432
+ if f2 == condInvalid {
433
+ m.insert(m.allocateInstr().asJmpIf(f1, newOperandLabel(target)))
434
+ } else {
435
+ if isBrz {
436
+ f2 = f2.invert()
437
+ and = !and
438
+ }
439
+ jmp1, jmp2 := m.allocateInstr(), m.allocateInstr()
440
+ m.insert(jmp1)
441
+ m.insert(jmp2)
442
+ notTaken, notTakenLabel := m.allocateBrTarget()
443
+ m.insert(notTaken)
444
+ if and {
445
+ jmp1.asJmpIf(f1.invert(), newOperandLabel(notTakenLabel))
446
+ jmp2.asJmpIf(f2, newOperandLabel(target))
447
+ } else {
448
+ jmp1.asJmpIf(f1, newOperandLabel(target))
449
+ jmp2.asJmpIf(f2, newOperandLabel(target))
450
+ }
451
+ }
452
+
453
+ cvalDef.Instr.MarkLowered()
454
+ default:
455
+ v := m.getOperand_Reg(cvalDef)
456
+
457
+ var cc cond
458
+ if b.Opcode() == ssa.OpcodeBrz {
459
+ cc = condZ
460
+ } else {
461
+ cc = condNZ
462
+ }
463
+
464
+ // Perform test %v, %v to set the flag.
465
+ cmp := m.allocateInstr().asCmpRmiR(false, v, v.reg(), false)
466
+ m.insert(cmp)
467
+ m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
468
+ }
469
+ }
470
+
471
+ // LowerInstr implements backend.Machine.
472
+ func (m *machine) LowerInstr(instr *ssa.Instruction) {
473
+ if l := instr.SourceOffset(); l.Valid() {
474
+ info := m.allocateInstr().asEmitSourceOffsetInfo(l)
475
+ m.insert(info)
476
+ }
477
+
478
+ switch op := instr.Opcode(); op {
479
+ case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable:
480
+ panic("BUG: branching instructions are handled by LowerBranches")
481
+ case ssa.OpcodeReturn:
482
+ panic("BUG: return must be handled by backend.Compiler")
483
+ case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined.
484
+ case ssa.OpcodeCall, ssa.OpcodeCallIndirect:
485
+ m.lowerCall(instr)
486
+ case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32:
487
+ m.lowerStore(instr)
488
+ case ssa.OpcodeIadd:
489
+ m.lowerAluRmiROp(instr, aluRmiROpcodeAdd)
490
+ case ssa.OpcodeIsub:
491
+ m.lowerAluRmiROp(instr, aluRmiROpcodeSub)
492
+ case ssa.OpcodeImul:
493
+ m.lowerAluRmiROp(instr, aluRmiROpcodeMul)
494
+ case ssa.OpcodeSdiv, ssa.OpcodeUdiv, ssa.OpcodeSrem, ssa.OpcodeUrem:
495
+ isDiv := op == ssa.OpcodeSdiv || op == ssa.OpcodeUdiv
496
+ isSigned := op == ssa.OpcodeSdiv || op == ssa.OpcodeSrem
497
+ m.lowerIDivRem(instr, isDiv, isSigned)
498
+ case ssa.OpcodeBand:
499
+ m.lowerAluRmiROp(instr, aluRmiROpcodeAnd)
500
+ case ssa.OpcodeBor:
501
+ m.lowerAluRmiROp(instr, aluRmiROpcodeOr)
502
+ case ssa.OpcodeBxor:
503
+ m.lowerAluRmiROp(instr, aluRmiROpcodeXor)
504
+ case ssa.OpcodeIshl:
505
+ m.lowerShiftR(instr, shiftROpShiftLeft)
506
+ case ssa.OpcodeSshr:
507
+ m.lowerShiftR(instr, shiftROpShiftRightArithmetic)
508
+ case ssa.OpcodeUshr:
509
+ m.lowerShiftR(instr, shiftROpShiftRightLogical)
510
+ case ssa.OpcodeRotl:
511
+ m.lowerShiftR(instr, shiftROpRotateLeft)
512
+ case ssa.OpcodeRotr:
513
+ m.lowerShiftR(instr, shiftROpRotateRight)
514
+ case ssa.OpcodeClz:
515
+ m.lowerClz(instr)
516
+ case ssa.OpcodeCtz:
517
+ m.lowerCtz(instr)
518
+ case ssa.OpcodePopcnt:
519
+ m.lowerUnaryRmR(instr, unaryRmROpcodePopcnt)
520
+ case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv:
521
+ m.lowerXmmRmR(instr)
522
+ case ssa.OpcodeFabs:
523
+ m.lowerFabsFneg(instr)
524
+ case ssa.OpcodeFneg:
525
+ m.lowerFabsFneg(instr)
526
+ case ssa.OpcodeCeil:
527
+ m.lowerRound(instr, roundingModeUp)
528
+ case ssa.OpcodeFloor:
529
+ m.lowerRound(instr, roundingModeDown)
530
+ case ssa.OpcodeTrunc:
531
+ m.lowerRound(instr, roundingModeZero)
532
+ case ssa.OpcodeNearest:
533
+ m.lowerRound(instr, roundingModeNearest)
534
+ case ssa.OpcodeFmin, ssa.OpcodeFmax:
535
+ m.lowerFminFmax(instr)
536
+ case ssa.OpcodeFcopysign:
537
+ m.lowerFcopysign(instr)
538
+ case ssa.OpcodeBitcast:
539
+ m.lowerBitcast(instr)
540
+ case ssa.OpcodeSqrt:
541
+ m.lowerSqrt(instr)
542
+ case ssa.OpcodeFpromote:
543
+ v := instr.Arg()
544
+ rn := m.getOperand_Reg(m.c.ValueDefinition(v))
545
+ rd := m.c.VRegOf(instr.Return())
546
+ cnt := m.allocateInstr()
547
+ cnt.asXmmUnaryRmR(sseOpcodeCvtss2sd, rn, rd)
548
+ m.insert(cnt)
549
+ case ssa.OpcodeFdemote:
550
+ v := instr.Arg()
551
+ rn := m.getOperand_Reg(m.c.ValueDefinition(v))
552
+ rd := m.c.VRegOf(instr.Return())
553
+ cnt := m.allocateInstr()
554
+ cnt.asXmmUnaryRmR(sseOpcodeCvtsd2ss, rn, rd)
555
+ m.insert(cnt)
556
+ case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat:
557
+ x, ctx := instr.Arg2()
558
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
559
+ rd := m.c.VRegOf(instr.Return())
560
+ ctxVReg := m.c.VRegOf(ctx)
561
+ m.lowerFcvtToSint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
562
+ instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat)
563
+ case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat:
564
+ x, ctx := instr.Arg2()
565
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
566
+ rd := m.c.VRegOf(instr.Return())
567
+ ctxVReg := m.c.VRegOf(ctx)
568
+ m.lowerFcvtToUint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
569
+ instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat)
570
+ case ssa.OpcodeFcvtFromSint:
571
+ x := instr.Arg()
572
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
573
+ rd := newOperandReg(m.c.VRegOf(instr.Return()))
574
+ m.lowerFcvtFromSint(rn, rd,
575
+ x.Type() == ssa.TypeI64, instr.Return().Type().Bits() == 64)
576
+ case ssa.OpcodeFcvtFromUint:
577
+ x := instr.Arg()
578
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
579
+ rd := newOperandReg(m.c.VRegOf(instr.Return()))
580
+ m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64,
581
+ instr.Return().Type().Bits() == 64)
582
+ case ssa.OpcodeVanyTrue:
583
+ m.lowerVanyTrue(instr)
584
+ case ssa.OpcodeVallTrue:
585
+ m.lowerVallTrue(instr)
586
+ case ssa.OpcodeVhighBits:
587
+ m.lowerVhighBits(instr)
588
+ case ssa.OpcodeVbnot:
589
+ m.lowerVbnot(instr)
590
+ case ssa.OpcodeVband:
591
+ x, y := instr.Arg2()
592
+ m.lowerVbBinOp(sseOpcodePand, x, y, instr.Return())
593
+ case ssa.OpcodeVbor:
594
+ x, y := instr.Arg2()
595
+ m.lowerVbBinOp(sseOpcodePor, x, y, instr.Return())
596
+ case ssa.OpcodeVbxor:
597
+ x, y := instr.Arg2()
598
+ m.lowerVbBinOp(sseOpcodePxor, x, y, instr.Return())
599
+ case ssa.OpcodeVbandnot:
600
+ m.lowerVbandnot(instr, sseOpcodePandn)
601
+ case ssa.OpcodeVbitselect:
602
+ m.lowerVbitselect(instr)
603
+ case ssa.OpcodeVIadd:
604
+ x, y, lane := instr.Arg2WithLane()
605
+ var vecOp sseOpcode
606
+ switch lane {
607
+ case ssa.VecLaneI8x16:
608
+ vecOp = sseOpcodePaddb
609
+ case ssa.VecLaneI16x8:
610
+ vecOp = sseOpcodePaddw
611
+ case ssa.VecLaneI32x4:
612
+ vecOp = sseOpcodePaddd
613
+ case ssa.VecLaneI64x2:
614
+ vecOp = sseOpcodePaddq
615
+ }
616
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
617
+
618
+ case ssa.OpcodeVSaddSat:
619
+ x, y, lane := instr.Arg2WithLane()
620
+ var vecOp sseOpcode
621
+ switch lane {
622
+ case ssa.VecLaneI8x16:
623
+ vecOp = sseOpcodePaddsb
624
+ case ssa.VecLaneI16x8:
625
+ vecOp = sseOpcodePaddsw
626
+ }
627
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
628
+
629
+ case ssa.OpcodeVUaddSat:
630
+ x, y, lane := instr.Arg2WithLane()
631
+ var vecOp sseOpcode
632
+ switch lane {
633
+ case ssa.VecLaneI8x16:
634
+ vecOp = sseOpcodePaddusb
635
+ case ssa.VecLaneI16x8:
636
+ vecOp = sseOpcodePaddusw
637
+ }
638
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
639
+
640
+ case ssa.OpcodeVIsub:
641
+ x, y, lane := instr.Arg2WithLane()
642
+ var vecOp sseOpcode
643
+ switch lane {
644
+ case ssa.VecLaneI8x16:
645
+ vecOp = sseOpcodePsubb
646
+ case ssa.VecLaneI16x8:
647
+ vecOp = sseOpcodePsubw
648
+ case ssa.VecLaneI32x4:
649
+ vecOp = sseOpcodePsubd
650
+ case ssa.VecLaneI64x2:
651
+ vecOp = sseOpcodePsubq
652
+ }
653
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
654
+
655
+ case ssa.OpcodeVSsubSat:
656
+ x, y, lane := instr.Arg2WithLane()
657
+ var vecOp sseOpcode
658
+ switch lane {
659
+ case ssa.VecLaneI8x16:
660
+ vecOp = sseOpcodePsubsb
661
+ case ssa.VecLaneI16x8:
662
+ vecOp = sseOpcodePsubsw
663
+ }
664
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
665
+
666
+ case ssa.OpcodeVUsubSat:
667
+ x, y, lane := instr.Arg2WithLane()
668
+ var vecOp sseOpcode
669
+ switch lane {
670
+ case ssa.VecLaneI8x16:
671
+ vecOp = sseOpcodePsubusb
672
+ case ssa.VecLaneI16x8:
673
+ vecOp = sseOpcodePsubusw
674
+ }
675
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
676
+
677
+ case ssa.OpcodeVImul:
678
+ m.lowerVImul(instr)
679
+ case ssa.OpcodeVIneg:
680
+ x, lane := instr.ArgWithLane()
681
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
682
+ rd := m.c.VRegOf(instr.Return())
683
+ var vecOp sseOpcode
684
+ switch lane {
685
+ case ssa.VecLaneI8x16:
686
+ vecOp = sseOpcodePsubb
687
+ case ssa.VecLaneI16x8:
688
+ vecOp = sseOpcodePsubw
689
+ case ssa.VecLaneI32x4:
690
+ vecOp = sseOpcodePsubd
691
+ case ssa.VecLaneI64x2:
692
+ vecOp = sseOpcodePsubq
693
+ default:
694
+ panic("BUG")
695
+ }
696
+
697
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
698
+ m.insert(m.allocateInstr().asZeros(tmp))
699
+
700
+ i := m.allocateInstr()
701
+ i.asXmmRmR(vecOp, rn, tmp)
702
+ m.insert(i)
703
+
704
+ m.copyTo(tmp, rd)
705
+ case ssa.OpcodeVFadd:
706
+ x, y, lane := instr.Arg2WithLane()
707
+ var vecOp sseOpcode
708
+ switch lane {
709
+ case ssa.VecLaneF32x4:
710
+ vecOp = sseOpcodeAddps
711
+ case ssa.VecLaneF64x2:
712
+ vecOp = sseOpcodeAddpd
713
+ }
714
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
715
+
716
+ case ssa.OpcodeVFsub:
717
+ x, y, lane := instr.Arg2WithLane()
718
+ var vecOp sseOpcode
719
+ switch lane {
720
+ case ssa.VecLaneF32x4:
721
+ vecOp = sseOpcodeSubps
722
+ case ssa.VecLaneF64x2:
723
+ vecOp = sseOpcodeSubpd
724
+ }
725
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
726
+
727
+ case ssa.OpcodeVFdiv:
728
+ x, y, lane := instr.Arg2WithLane()
729
+ var vecOp sseOpcode
730
+ switch lane {
731
+ case ssa.VecLaneF32x4:
732
+ vecOp = sseOpcodeDivps
733
+ case ssa.VecLaneF64x2:
734
+ vecOp = sseOpcodeDivpd
735
+ }
736
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
737
+
738
+ case ssa.OpcodeVFmul:
739
+ x, y, lane := instr.Arg2WithLane()
740
+ var vecOp sseOpcode
741
+ switch lane {
742
+ case ssa.VecLaneF32x4:
743
+ vecOp = sseOpcodeMulps
744
+ case ssa.VecLaneF64x2:
745
+ vecOp = sseOpcodeMulpd
746
+ }
747
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
748
+
749
+ case ssa.OpcodeVFneg:
750
+ x, lane := instr.ArgWithLane()
751
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
752
+ rd := m.c.VRegOf(instr.Return())
753
+
754
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
755
+
756
+ var shiftOp, xorOp sseOpcode
757
+ var shiftAmt uint32
758
+ switch lane {
759
+ case ssa.VecLaneF32x4:
760
+ shiftOp, shiftAmt, xorOp = sseOpcodePslld, 31, sseOpcodeXorps
761
+ case ssa.VecLaneF64x2:
762
+ shiftOp, shiftAmt, xorOp = sseOpcodePsllq, 63, sseOpcodeXorpd
763
+ }
764
+
765
+ zero := m.allocateInstr()
766
+ zero.asZeros(tmp)
767
+ m.insert(zero)
768
+
769
+ // Set all bits on tmp by CMPPD with arg=0 (== pseudo CMPEQPD instruction).
770
+ // See https://www.felixcloutier.com/x86/cmpps
771
+ //
772
+ // Note: if we do not clear all the bits ^ with XORPS, this might end up not setting ones on some lane
773
+ // if the lane is NaN.
774
+ cmp := m.allocateInstr()
775
+ cmp.asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_UQ), newOperandReg(tmp), tmp)
776
+ m.insert(cmp)
777
+
778
+ // Do the left shift on each lane to set only the most significant bit in each.
779
+ i := m.allocateInstr()
780
+ i.asXmmRmiReg(shiftOp, newOperandImm32(shiftAmt), tmp)
781
+ m.insert(i)
782
+
783
+ // Get the negated result by XOR on each lane with tmp.
784
+ i = m.allocateInstr()
785
+ i.asXmmRmR(xorOp, rn, tmp)
786
+ m.insert(i)
787
+
788
+ m.copyTo(tmp, rd)
789
+
790
+ case ssa.OpcodeVSqrt:
791
+ x, lane := instr.ArgWithLane()
792
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
793
+ rd := m.c.VRegOf(instr.Return())
794
+
795
+ var vecOp sseOpcode
796
+ switch lane {
797
+ case ssa.VecLaneF32x4:
798
+ vecOp = sseOpcodeSqrtps
799
+ case ssa.VecLaneF64x2:
800
+ vecOp = sseOpcodeSqrtpd
801
+ }
802
+ i := m.allocateInstr()
803
+ i.asXmmUnaryRmR(vecOp, rn, rd)
804
+ m.insert(i)
805
+
806
+ case ssa.OpcodeVImin:
807
+ x, y, lane := instr.Arg2WithLane()
808
+ var vecOp sseOpcode
809
+ switch lane {
810
+ case ssa.VecLaneI8x16:
811
+ vecOp = sseOpcodePminsb
812
+ case ssa.VecLaneI16x8:
813
+ vecOp = sseOpcodePminsw
814
+ case ssa.VecLaneI32x4:
815
+ vecOp = sseOpcodePminsd
816
+ }
817
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
818
+
819
+ case ssa.OpcodeVUmin:
820
+ x, y, lane := instr.Arg2WithLane()
821
+ var vecOp sseOpcode
822
+ switch lane {
823
+ case ssa.VecLaneI8x16:
824
+ vecOp = sseOpcodePminub
825
+ case ssa.VecLaneI16x8:
826
+ vecOp = sseOpcodePminuw
827
+ case ssa.VecLaneI32x4:
828
+ vecOp = sseOpcodePminud
829
+ }
830
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
831
+
832
+ case ssa.OpcodeVImax:
833
+ x, y, lane := instr.Arg2WithLane()
834
+ var vecOp sseOpcode
835
+ switch lane {
836
+ case ssa.VecLaneI8x16:
837
+ vecOp = sseOpcodePmaxsb
838
+ case ssa.VecLaneI16x8:
839
+ vecOp = sseOpcodePmaxsw
840
+ case ssa.VecLaneI32x4:
841
+ vecOp = sseOpcodePmaxsd
842
+ }
843
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
844
+
845
+ case ssa.OpcodeVUmax:
846
+ x, y, lane := instr.Arg2WithLane()
847
+ var vecOp sseOpcode
848
+ switch lane {
849
+ case ssa.VecLaneI8x16:
850
+ vecOp = sseOpcodePmaxub
851
+ case ssa.VecLaneI16x8:
852
+ vecOp = sseOpcodePmaxuw
853
+ case ssa.VecLaneI32x4:
854
+ vecOp = sseOpcodePmaxud
855
+ }
856
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
857
+
858
+ case ssa.OpcodeVAvgRound:
859
+ x, y, lane := instr.Arg2WithLane()
860
+ var vecOp sseOpcode
861
+ switch lane {
862
+ case ssa.VecLaneI8x16:
863
+ vecOp = sseOpcodePavgb
864
+ case ssa.VecLaneI16x8:
865
+ vecOp = sseOpcodePavgw
866
+ }
867
+ m.lowerVbBinOp(vecOp, x, y, instr.Return())
868
+
869
+ case ssa.OpcodeVIcmp:
870
+ x, y, c, lane := instr.VIcmpData()
871
+ m.lowerVIcmp(x, y, c, instr.Return(), lane)
872
+
873
+ case ssa.OpcodeVFcmp:
874
+ x, y, c, lane := instr.VFcmpData()
875
+ m.lowerVFcmp(x, y, c, instr.Return(), lane)
876
+
877
+ case ssa.OpcodeExtractlane:
878
+ x, index, signed, lane := instr.ExtractlaneData()
879
+ m.lowerExtractLane(x, index, signed, instr.Return(), lane)
880
+
881
+ case ssa.OpcodeInsertlane:
882
+ x, y, index, lane := instr.InsertlaneData()
883
+ m.lowerInsertLane(x, y, index, instr.Return(), lane)
884
+
885
+ case ssa.OpcodeSwizzle:
886
+ x, y, _ := instr.Arg2WithLane()
887
+ m.lowerSwizzle(x, y, instr.Return())
888
+
889
+ case ssa.OpcodeShuffle:
890
+ x, y, lo, hi := instr.ShuffleData()
891
+ m.lowerShuffle(x, y, lo, hi, instr.Return())
892
+
893
+ case ssa.OpcodeSplat:
894
+ x, lane := instr.ArgWithLane()
895
+ m.lowerSplat(x, instr.Return(), lane)
896
+
897
+ case ssa.OpcodeSqmulRoundSat:
898
+ x, y := instr.Arg2()
899
+ m.lowerSqmulRoundSat(x, y, instr.Return())
900
+
901
+ case ssa.OpcodeVZeroExtLoad:
902
+ ptr, offset, typ := instr.VZeroExtLoadData()
903
+ var sseOp sseOpcode
904
+ // Both movss and movsd clears the higher bits of the destination register upt 128 bits.
905
+ // https://www.felixcloutier.com/x86/movss
906
+ // https://www.felixcloutier.com/x86/movsd
907
+ if typ == ssa.TypeF32 {
908
+ sseOp = sseOpcodeMovss
909
+ } else {
910
+ sseOp = sseOpcodeMovsd
911
+ }
912
+ mem := m.lowerToAddressMode(ptr, offset)
913
+ dst := m.c.VRegOf(instr.Return())
914
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandMem(mem), dst))
915
+
916
+ case ssa.OpcodeVMinPseudo:
917
+ x, y, lane := instr.Arg2WithLane()
918
+ var vecOp sseOpcode
919
+ switch lane {
920
+ case ssa.VecLaneF32x4:
921
+ vecOp = sseOpcodeMinps
922
+ case ssa.VecLaneF64x2:
923
+ vecOp = sseOpcodeMinpd
924
+ default:
925
+ panic("BUG: unexpected lane type")
926
+ }
927
+ m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
928
+
929
+ case ssa.OpcodeVMaxPseudo:
930
+ x, y, lane := instr.Arg2WithLane()
931
+ var vecOp sseOpcode
932
+ switch lane {
933
+ case ssa.VecLaneF32x4:
934
+ vecOp = sseOpcodeMaxps
935
+ case ssa.VecLaneF64x2:
936
+ vecOp = sseOpcodeMaxpd
937
+ default:
938
+ panic("BUG: unexpected lane type")
939
+ }
940
+ m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
941
+
942
+ case ssa.OpcodeVIshl:
943
+ x, y, lane := instr.Arg2WithLane()
944
+ m.lowerVIshl(x, y, instr.Return(), lane)
945
+
946
+ case ssa.OpcodeVSshr:
947
+ x, y, lane := instr.Arg2WithLane()
948
+ m.lowerVSshr(x, y, instr.Return(), lane)
949
+
950
+ case ssa.OpcodeVUshr:
951
+ x, y, lane := instr.Arg2WithLane()
952
+ m.lowerVUshr(x, y, instr.Return(), lane)
953
+
954
+ case ssa.OpcodeVCeil:
955
+ x, lane := instr.ArgWithLane()
956
+ m.lowerVRound(x, instr.Return(), 0x2, lane == ssa.VecLaneF64x2)
957
+
958
+ case ssa.OpcodeVFloor:
959
+ x, lane := instr.ArgWithLane()
960
+ m.lowerVRound(x, instr.Return(), 0x1, lane == ssa.VecLaneF64x2)
961
+
962
+ case ssa.OpcodeVTrunc:
963
+ x, lane := instr.ArgWithLane()
964
+ m.lowerVRound(x, instr.Return(), 0x3, lane == ssa.VecLaneF64x2)
965
+
966
+ case ssa.OpcodeVNearest:
967
+ x, lane := instr.ArgWithLane()
968
+ m.lowerVRound(x, instr.Return(), 0x0, lane == ssa.VecLaneF64x2)
969
+
970
+ case ssa.OpcodeExtIaddPairwise:
971
+ x, lane, signed := instr.ExtIaddPairwiseData()
972
+ m.lowerExtIaddPairwise(x, instr.Return(), lane, signed)
973
+
974
+ case ssa.OpcodeUwidenLow, ssa.OpcodeSwidenLow:
975
+ x, lane := instr.ArgWithLane()
976
+ m.lowerWidenLow(x, instr.Return(), lane, op == ssa.OpcodeSwidenLow)
977
+
978
+ case ssa.OpcodeUwidenHigh, ssa.OpcodeSwidenHigh:
979
+ x, lane := instr.ArgWithLane()
980
+ m.lowerWidenHigh(x, instr.Return(), lane, op == ssa.OpcodeSwidenHigh)
981
+
982
+ case ssa.OpcodeLoadSplat:
983
+ ptr, offset, lane := instr.LoadSplatData()
984
+ m.lowerLoadSplat(ptr, offset, instr.Return(), lane)
985
+
986
+ case ssa.OpcodeVFcvtFromUint, ssa.OpcodeVFcvtFromSint:
987
+ x, lane := instr.ArgWithLane()
988
+ m.lowerVFcvtFromInt(x, instr.Return(), lane, op == ssa.OpcodeVFcvtFromSint)
989
+
990
+ case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat:
991
+ x, lane := instr.ArgWithLane()
992
+ m.lowerVFcvtToIntSat(x, instr.Return(), lane, op == ssa.OpcodeVFcvtToSintSat)
993
+
994
+ case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow:
995
+ x, y, lane := instr.Arg2WithLane()
996
+ m.lowerNarrow(x, y, instr.Return(), lane, op == ssa.OpcodeSnarrow)
997
+
998
+ case ssa.OpcodeFvpromoteLow:
999
+ x := instr.Arg()
1000
+ src := m.getOperand_Reg(m.c.ValueDefinition(x))
1001
+ dst := m.c.VRegOf(instr.Return())
1002
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtps2pd, src, dst))
1003
+
1004
+ case ssa.OpcodeFvdemote:
1005
+ x := instr.Arg()
1006
+ src := m.getOperand_Reg(m.c.ValueDefinition(x))
1007
+ dst := m.c.VRegOf(instr.Return())
1008
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst))
1009
+
1010
+ case ssa.OpcodeWideningPairwiseDotProductS:
1011
+ x, y := instr.Arg2()
1012
+ m.lowerWideningPairwiseDotProductS(x, y, instr.Return())
1013
+
1014
+ case ssa.OpcodeVIabs:
1015
+ m.lowerVIabs(instr)
1016
+ case ssa.OpcodeVIpopcnt:
1017
+ m.lowerVIpopcnt(instr)
1018
+ case ssa.OpcodeVFmin:
1019
+ m.lowerVFmin(instr)
1020
+ case ssa.OpcodeVFmax:
1021
+ m.lowerVFmax(instr)
1022
+ case ssa.OpcodeVFabs:
1023
+ m.lowerVFabs(instr)
1024
+ case ssa.OpcodeUndefined:
1025
+ m.insert(m.allocateInstr().asUD2())
1026
+ case ssa.OpcodeExitWithCode:
1027
+ execCtx, code := instr.ExitWithCodeData()
1028
+ m.lowerExitWithCode(m.c.VRegOf(execCtx), code)
1029
+ case ssa.OpcodeExitIfTrueWithCode:
1030
+ execCtx, c, code := instr.ExitIfTrueWithCodeData()
1031
+ m.lowerExitIfTrueWithCode(m.c.VRegOf(execCtx), c, code)
1032
+ case ssa.OpcodeLoad:
1033
+ ptr, offset, typ := instr.LoadData()
1034
+ dst := m.c.VRegOf(instr.Return())
1035
+ m.lowerLoad(ptr, offset, typ, dst)
1036
+ case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32:
1037
+ ptr, offset, _ := instr.LoadData()
1038
+ ret := m.c.VRegOf(instr.Return())
1039
+ m.lowerExtLoad(op, ptr, offset, ret)
1040
+ case ssa.OpcodeVconst:
1041
+ result := m.c.VRegOf(instr.Return())
1042
+ lo, hi := instr.VconstData()
1043
+ m.lowerVconst(result, lo, hi)
1044
+ case ssa.OpcodeSExtend, ssa.OpcodeUExtend:
1045
+ from, to, signed := instr.ExtendData()
1046
+ m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed)
1047
+ case ssa.OpcodeIcmp:
1048
+ m.lowerIcmp(instr)
1049
+ case ssa.OpcodeFcmp:
1050
+ m.lowerFcmp(instr)
1051
+ case ssa.OpcodeSelect:
1052
+ cval, x, y := instr.SelectData()
1053
+ m.lowerSelect(x, y, cval, instr.Return())
1054
+ case ssa.OpcodeIreduce:
1055
+ rn := m.getOperand_Mem_Reg(m.c.ValueDefinition(instr.Arg()))
1056
+ retVal := instr.Return()
1057
+ rd := m.c.VRegOf(retVal)
1058
+
1059
+ if retVal.Type() != ssa.TypeI32 {
1060
+ panic("TODO?: Ireduce to non-i32")
1061
+ }
1062
+ m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, rd))
1063
+
1064
+ case ssa.OpcodeAtomicLoad:
1065
+ ptr := instr.Arg()
1066
+ size := instr.AtomicTargetSize()
1067
+ dst := m.c.VRegOf(instr.Return())
1068
+
1069
+ // At this point, the ptr is ensured to be aligned, so using a normal load is atomic.
1070
+ // https://github.com/golang/go/blob/go1.24.0/src/internal/runtime/atomic/atomic_amd64.go#L29
1071
+ mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
1072
+ load := m.allocateInstr()
1073
+ switch size {
1074
+ case 8:
1075
+ load.asMov64MR(mem, dst)
1076
+ case 4:
1077
+ load.asMovzxRmR(extModeLQ, mem, dst)
1078
+ case 2:
1079
+ load.asMovzxRmR(extModeWQ, mem, dst)
1080
+ case 1:
1081
+ load.asMovzxRmR(extModeBQ, mem, dst)
1082
+ default:
1083
+ panic("BUG")
1084
+ }
1085
+ m.insert(load)
1086
+
1087
+ case ssa.OpcodeFence:
1088
+ m.insert(m.allocateInstr().asMFence())
1089
+
1090
+ case ssa.OpcodeAtomicStore:
1091
+ ptr, _val := instr.Arg2()
1092
+ size := instr.AtomicTargetSize()
1093
+
1094
+ val := m.getOperand_Reg(m.c.ValueDefinition(_val))
1095
+ // The content on the val register will be overwritten by xchg, so we need to copy it to a temporary register.
1096
+ copied := m.copyToTmp(val.reg())
1097
+
1098
+ mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
1099
+ store := m.allocateInstr().asXCHG(copied, mem, byte(size))
1100
+ m.insert(store)
1101
+
1102
+ case ssa.OpcodeAtomicCas:
1103
+ addr, exp, repl := instr.Arg3()
1104
+ size := instr.AtomicTargetSize()
1105
+ m.lowerAtomicCas(addr, exp, repl, size, instr.Return())
1106
+
1107
+ case ssa.OpcodeAtomicRmw:
1108
+ addr, val := instr.Arg2()
1109
+ atomicOp, size := instr.AtomicRmwData()
1110
+ m.lowerAtomicRmw(atomicOp, addr, val, size, instr.Return())
1111
+
1112
+ case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect:
1113
+ m.lowerTailCall(instr)
1114
+
1115
+ default:
1116
+ panic("TODO: lowering " + op.String())
1117
+ }
1118
+ }
1119
+
1120
+ func (m *machine) lowerAtomicRmw(op ssa.AtomicRmwOp, addr, val ssa.Value, size uint64, ret ssa.Value) {
1121
+ mem := m.lowerToAddressMode(addr, 0)
1122
+ _val := m.getOperand_Reg(m.c.ValueDefinition(val))
1123
+
1124
+ switch op {
1125
+ case ssa.AtomicRmwOpAdd, ssa.AtomicRmwOpSub:
1126
+ valCopied := m.copyToTmp(_val.reg())
1127
+ if op == ssa.AtomicRmwOpSub {
1128
+ // Negate the value.
1129
+ m.insert(m.allocateInstr().asNeg(newOperandReg(valCopied), true))
1130
+ }
1131
+ m.insert(m.allocateInstr().asLockXAdd(valCopied, mem, byte(size)))
1132
+ m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1133
+ m.copyTo(valCopied, m.c.VRegOf(ret))
1134
+
1135
+ case ssa.AtomicRmwOpAnd, ssa.AtomicRmwOpOr, ssa.AtomicRmwOpXor:
1136
+ accumulator := raxVReg
1137
+ // Reserve rax for the accumulator to make regalloc happy.
1138
+ // Note: do this initialization before defining valCopied, because it might be the same register and
1139
+ // if that happens, the unnecessary load/store will be performed inside the loop.
1140
+ // This can be mitigated in any way once the register allocator is clever enough.
1141
+ m.insert(m.allocateInstr().asDefineUninitializedReg(accumulator))
1142
+
1143
+ // Copy the value to a temporary register.
1144
+ valCopied := m.copyToTmp(_val.reg())
1145
+ m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1146
+
1147
+ memOp := newOperandMem(mem)
1148
+ tmp := m.c.AllocateVReg(ssa.TypeI64)
1149
+ beginLoop, beginLoopLabel := m.allocateBrTarget()
1150
+ {
1151
+ m.insert(beginLoop)
1152
+ // Reset the value on tmp by the original value.
1153
+ m.copyTo(valCopied, tmp)
1154
+ // Load the current value at the memory location into accumulator.
1155
+ switch size {
1156
+ case 1:
1157
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, memOp, accumulator))
1158
+ case 2:
1159
+ m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, memOp, accumulator))
1160
+ case 4:
1161
+ m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, memOp, accumulator))
1162
+ case 8:
1163
+ m.insert(m.allocateInstr().asMov64MR(memOp, accumulator))
1164
+ default:
1165
+ panic("BUG")
1166
+ }
1167
+ // Then perform the logical operation on the accumulator and the value on tmp.
1168
+ switch op {
1169
+ case ssa.AtomicRmwOpAnd:
1170
+ m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, newOperandReg(accumulator), tmp, true))
1171
+ case ssa.AtomicRmwOpOr:
1172
+ m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeOr, newOperandReg(accumulator), tmp, true))
1173
+ case ssa.AtomicRmwOpXor:
1174
+ m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(accumulator), tmp, true))
1175
+ default:
1176
+ panic("BUG")
1177
+ }
1178
+ // Finally, try compare-exchange the value at the memory location with the tmp.
1179
+ m.insert(m.allocateInstr().asLockCmpXCHG(tmp, memOp.addressMode(), byte(size)))
1180
+ // If it succeeds, ZF will be set, and we can break the loop.
1181
+ m.insert(m.allocateInstr().asJmpIf(condNZ, newOperandLabel(beginLoopLabel)))
1182
+ }
1183
+
1184
+ // valCopied must be alive at the end of the loop.
1185
+ m.insert(m.allocateInstr().asNopUseReg(valCopied))
1186
+
1187
+ // At this point, accumulator contains the result.
1188
+ m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
1189
+ m.copyTo(accumulator, m.c.VRegOf(ret))
1190
+
1191
+ case ssa.AtomicRmwOpXchg:
1192
+ valCopied := m.copyToTmp(_val.reg())
1193
+
1194
+ m.insert(m.allocateInstr().asXCHG(valCopied, newOperandMem(mem), byte(size)))
1195
+ m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
1196
+ m.copyTo(valCopied, m.c.VRegOf(ret))
1197
+
1198
+ default:
1199
+ panic("BUG")
1200
+ }
1201
+ }
1202
+
1203
+ func (m *machine) lowerAtomicCas(addr, exp, repl ssa.Value, size uint64, ret ssa.Value) {
1204
+ mem := m.lowerToAddressMode(addr, 0)
1205
+ expOp := m.getOperand_Reg(m.c.ValueDefinition(exp))
1206
+ replOp := m.getOperand_Reg(m.c.ValueDefinition(repl))
1207
+
1208
+ accumulator := raxVReg
1209
+ m.copyTo(expOp.reg(), accumulator)
1210
+ m.insert(m.allocateInstr().asLockCmpXCHG(replOp.reg(), mem, byte(size)))
1211
+ m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
1212
+ m.copyTo(accumulator, m.c.VRegOf(ret))
1213
+ }
1214
+
1215
+ func (m *machine) clearHigherBitsForAtomic(r regalloc.VReg, valSize uint64, resultType ssa.Type) {
1216
+ switch resultType {
1217
+ case ssa.TypeI32:
1218
+ switch valSize {
1219
+ case 1:
1220
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(r), r))
1221
+ case 2:
1222
+ m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(r), r))
1223
+ }
1224
+ case ssa.TypeI64:
1225
+ switch valSize {
1226
+ case 1:
1227
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(r), r))
1228
+ case 2:
1229
+ m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, newOperandReg(r), r))
1230
+ case 4:
1231
+ m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, newOperandReg(r), r))
1232
+ }
1233
+ }
1234
+ }
1235
+
1236
+ func (m *machine) lowerFcmp(instr *ssa.Instruction) {
1237
+ f1, f2, and := m.lowerFcmpToFlags(instr)
1238
+ rd := m.c.VRegOf(instr.Return())
1239
+ if f2 == condInvalid {
1240
+ tmp := m.c.AllocateVReg(ssa.TypeI32)
1241
+ m.insert(m.allocateInstr().asSetcc(f1, tmp))
1242
+ // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
1243
+ // the semantics of Icmp that sets either 0 or 1.
1244
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
1245
+ } else {
1246
+ tmp1, tmp2 := m.c.AllocateVReg(ssa.TypeI32), m.c.AllocateVReg(ssa.TypeI32)
1247
+ m.insert(m.allocateInstr().asSetcc(f1, tmp1))
1248
+ m.insert(m.allocateInstr().asSetcc(f2, tmp2))
1249
+ var op aluRmiROpcode
1250
+ if and {
1251
+ op = aluRmiROpcodeAnd
1252
+ } else {
1253
+ op = aluRmiROpcodeOr
1254
+ }
1255
+ m.insert(m.allocateInstr().asAluRmiR(op, newOperandReg(tmp1), tmp2, false))
1256
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp2), rd))
1257
+ }
1258
+ }
1259
+
1260
+ func (m *machine) lowerIcmp(instr *ssa.Instruction) {
1261
+ x, y, c := instr.IcmpData()
1262
+ m.lowerIcmpToFlag(m.c.ValueDefinition(x), m.c.ValueDefinition(y), x.Type() == ssa.TypeI64)
1263
+ rd := m.c.VRegOf(instr.Return())
1264
+ tmp := m.c.AllocateVReg(ssa.TypeI32)
1265
+ m.insert(m.allocateInstr().asSetcc(condFromSSAIntCmpCond(c), tmp))
1266
+ // On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
1267
+ // the semantics of Icmp that sets either 0 or 1.
1268
+ m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
1269
+ }
1270
+
1271
+ func (m *machine) lowerSelect(x, y, cval, ret ssa.Value) {
1272
+ xo, yo := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
1273
+ rd := m.c.VRegOf(ret)
1274
+
1275
+ var cond cond
1276
+ cvalDef := m.c.ValueDefinition(cval)
1277
+ switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
1278
+ case ssa.OpcodeIcmp:
1279
+ icmp := cvalDef.Instr
1280
+ xc, yc, cc := icmp.IcmpData()
1281
+ m.lowerIcmpToFlag(m.c.ValueDefinition(xc), m.c.ValueDefinition(yc), xc.Type() == ssa.TypeI64)
1282
+ cond = condFromSSAIntCmpCond(cc)
1283
+ icmp.Lowered()
1284
+ default: // TODO: match ssa.OpcodeFcmp for optimization, but seems a bit complex.
1285
+ cv := m.getOperand_Reg(cvalDef)
1286
+ test := m.allocateInstr().asCmpRmiR(false, cv, cv.reg(), false)
1287
+ m.insert(test)
1288
+ cond = condNZ
1289
+ }
1290
+
1291
+ if typ := x.Type(); typ.IsInt() {
1292
+ _64 := typ.Bits() == 64
1293
+ mov := m.allocateInstr()
1294
+ tmp := m.c.AllocateVReg(typ)
1295
+ switch yo.kind {
1296
+ case operandKindReg:
1297
+ mov.asMovRR(yo.reg(), tmp, _64)
1298
+ case operandKindMem:
1299
+ if _64 {
1300
+ mov.asMov64MR(yo, tmp)
1301
+ } else {
1302
+ mov.asMovzxRmR(extModeLQ, yo, tmp)
1303
+ }
1304
+ default:
1305
+ panic("BUG")
1306
+ }
1307
+ m.insert(mov)
1308
+ cmov := m.allocateInstr().asCmove(cond, xo, tmp, _64)
1309
+ m.insert(cmov)
1310
+ m.insert(m.allocateInstr().asMovRR(tmp, rd, _64))
1311
+ } else {
1312
+ mov := m.allocateInstr()
1313
+ tmp := m.c.AllocateVReg(typ)
1314
+ switch typ {
1315
+ case ssa.TypeF32:
1316
+ mov.asXmmUnaryRmR(sseOpcodeMovss, yo, tmp)
1317
+ case ssa.TypeF64:
1318
+ mov.asXmmUnaryRmR(sseOpcodeMovsd, yo, tmp)
1319
+ case ssa.TypeV128:
1320
+ mov.asXmmUnaryRmR(sseOpcodeMovdqu, yo, tmp)
1321
+ default:
1322
+ panic("BUG")
1323
+ }
1324
+ m.insert(mov)
1325
+
1326
+ cmov := m.allocateInstr().asXmmCMov(cond, xo, tmp, typ.Size())
1327
+ m.insert(cmov)
1328
+
1329
+ m.copyTo(tmp, rd)
1330
+ }
1331
+ }
1332
+
1333
+ func (m *machine) lowerXmmCmovAfterRegAlloc(i *instruction) {
1334
+ x := i.op1
1335
+ rd := i.op2.reg()
1336
+ cond := cond(i.u1)
1337
+
1338
+ jcc := m.allocateInstr()
1339
+ m.insert(jcc)
1340
+
1341
+ mov := m.allocateInstr()
1342
+ switch i.u2 {
1343
+ case 4:
1344
+ mov.asXmmUnaryRmR(sseOpcodeMovss, x, rd)
1345
+ case 8:
1346
+ mov.asXmmUnaryRmR(sseOpcodeMovsd, x, rd)
1347
+ case 16:
1348
+ mov.asXmmUnaryRmR(sseOpcodeMovdqu, x, rd)
1349
+ default:
1350
+ panic("BUG")
1351
+ }
1352
+ m.insert(mov)
1353
+
1354
+ nop, end := m.allocateBrTarget()
1355
+ m.insert(nop)
1356
+ jcc.asJmpIf(cond.invert(), newOperandLabel(end))
1357
+ }
1358
+
1359
+ func (m *machine) lowerExtend(_arg, ret ssa.Value, from, to byte, signed bool) {
1360
+ rd0 := m.c.VRegOf(ret)
1361
+ arg := m.getOperand_Mem_Reg(m.c.ValueDefinition(_arg))
1362
+
1363
+ rd := m.c.AllocateVReg(ret.Type())
1364
+
1365
+ ext := m.allocateInstr()
1366
+ switch {
1367
+ case from == 8 && to == 16 && signed:
1368
+ ext.asMovsxRmR(extModeBQ, arg, rd)
1369
+ case from == 8 && to == 16 && !signed:
1370
+ ext.asMovzxRmR(extModeBL, arg, rd)
1371
+ case from == 8 && to == 32 && signed:
1372
+ ext.asMovsxRmR(extModeBL, arg, rd)
1373
+ case from == 8 && to == 32 && !signed:
1374
+ ext.asMovzxRmR(extModeBQ, arg, rd)
1375
+ case from == 8 && to == 64 && signed:
1376
+ ext.asMovsxRmR(extModeBQ, arg, rd)
1377
+ case from == 8 && to == 64 && !signed:
1378
+ ext.asMovzxRmR(extModeBQ, arg, rd)
1379
+ case from == 16 && to == 32 && signed:
1380
+ ext.asMovsxRmR(extModeWL, arg, rd)
1381
+ case from == 16 && to == 32 && !signed:
1382
+ ext.asMovzxRmR(extModeWL, arg, rd)
1383
+ case from == 16 && to == 64 && signed:
1384
+ ext.asMovsxRmR(extModeWQ, arg, rd)
1385
+ case from == 16 && to == 64 && !signed:
1386
+ ext.asMovzxRmR(extModeWQ, arg, rd)
1387
+ case from == 32 && to == 64 && signed:
1388
+ ext.asMovsxRmR(extModeLQ, arg, rd)
1389
+ case from == 32 && to == 64 && !signed:
1390
+ ext.asMovzxRmR(extModeLQ, arg, rd)
1391
+ default:
1392
+ panic(fmt.Sprintf("BUG: unhandled extend: from=%d, to=%d, signed=%t", from, to, signed))
1393
+ }
1394
+ m.insert(ext)
1395
+
1396
+ m.copyTo(rd, rd0)
1397
+ }
1398
+
1399
+ func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) {
1400
+ if lo == 0 && hi == 0 {
1401
+ m.insert(m.allocateInstr().asZeros(dst))
1402
+ return
1403
+ }
1404
+
1405
+ load := m.allocateInstr()
1406
+ l, pos := m.allocateLabel()
1407
+ m.consts = append(m.consts, _const{label: l, labelPos: pos, lo: lo, hi: hi})
1408
+ load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(l)), dst)
1409
+ m.insert(load)
1410
+ }
1411
+
1412
+ func (m *machine) lowerCtz(instr *ssa.Instruction) {
1413
+ if m.cpuFeatures.Has(platform.CpuFeatureAmd64BMI1) {
1414
+ m.lowerUnaryRmR(instr, unaryRmROpcodeTzcnt)
1415
+ } else {
1416
+ // On processors that do not support TZCNT, the BSF instruction is
1417
+ // executed instead. The key difference between TZCNT and BSF
1418
+ // instruction is that if source operand is zero, the content of
1419
+ // destination operand is undefined.
1420
+ // https://www.felixcloutier.com/x86/tzcnt.html
1421
+
1422
+ x := instr.Arg()
1423
+ if !x.Type().IsInt() {
1424
+ panic("BUG?")
1425
+ }
1426
+ _64 := x.Type().Bits() == 64
1427
+
1428
+ xDef := m.c.ValueDefinition(x)
1429
+ tmp := m.c.AllocateVReg(x.Type())
1430
+ rm := m.getOperand_Reg(xDef)
1431
+
1432
+ // First, we have to check if the target is non-zero.
1433
+ test := m.allocateInstr()
1434
+ test.asCmpRmiR(false, rm, rm.reg(), _64)
1435
+ m.insert(test)
1436
+
1437
+ jmpNz := m.allocateInstr()
1438
+ m.insert(jmpNz)
1439
+
1440
+ // If the value is zero, we just push the const value.
1441
+ m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
1442
+
1443
+ // Now jump right after the non-zero case.
1444
+ jmpAtEnd := m.allocateInstr()
1445
+ m.insert(jmpAtEnd)
1446
+
1447
+ // jmpNz target label is set here.
1448
+ nop, nz := m.allocateBrTarget()
1449
+ jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
1450
+ m.insert(nop)
1451
+
1452
+ // Emit the non-zero case.
1453
+ bsr := m.allocateInstr()
1454
+ bsr.asUnaryRmR(unaryRmROpcodeBsf, rm, tmp, _64)
1455
+ m.insert(bsr)
1456
+
1457
+ // jmpAtEnd target label is set here.
1458
+ nopEnd, end := m.allocateBrTarget()
1459
+ jmpAtEnd.asJmp(newOperandLabel(end))
1460
+ m.insert(nopEnd)
1461
+
1462
+ m.copyTo(tmp, m.c.VRegOf(instr.Return()))
1463
+ }
1464
+ }
1465
+
1466
+ func (m *machine) lowerClz(instr *ssa.Instruction) {
1467
+ if m.cpuFeatures.Has(platform.CpuFeatureAmd64ABM) {
1468
+ m.lowerUnaryRmR(instr, unaryRmROpcodeLzcnt)
1469
+ } else {
1470
+ // On processors that do not support LZCNT, we combine BSR (calculating
1471
+ // most significant set bit) with XOR. This logic is described in
1472
+ // "Replace Raw Assembly Code with Builtin Intrinsics" section in:
1473
+ // https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code.
1474
+
1475
+ x := instr.Arg()
1476
+ if !x.Type().IsInt() {
1477
+ panic("BUG?")
1478
+ }
1479
+ _64 := x.Type().Bits() == 64
1480
+
1481
+ xDef := m.c.ValueDefinition(x)
1482
+ rm := m.getOperand_Reg(xDef)
1483
+ tmp := m.c.AllocateVReg(x.Type())
1484
+
1485
+ // First, we have to check if the rm is non-zero as BSR is undefined
1486
+ // on zero. See https://www.felixcloutier.com/x86/bsr.
1487
+ test := m.allocateInstr()
1488
+ test.asCmpRmiR(false, rm, rm.reg(), _64)
1489
+ m.insert(test)
1490
+
1491
+ jmpNz := m.allocateInstr()
1492
+ m.insert(jmpNz)
1493
+
1494
+ // If the value is zero, we just push the const value.
1495
+ m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
1496
+
1497
+ // Now jump right after the non-zero case.
1498
+ jmpAtEnd := m.allocateInstr()
1499
+ m.insert(jmpAtEnd)
1500
+
1501
+ // jmpNz target label is set here.
1502
+ nop, nz := m.allocateBrTarget()
1503
+ jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
1504
+ m.insert(nop)
1505
+
1506
+ // Emit the non-zero case.
1507
+ bsr := m.allocateInstr()
1508
+ bsr.asUnaryRmR(unaryRmROpcodeBsr, rm, tmp, _64)
1509
+ m.insert(bsr)
1510
+
1511
+ // Now we XOR the value with the bit length minus one.
1512
+ xor := m.allocateInstr()
1513
+ xor.asAluRmiR(aluRmiROpcodeXor, newOperandImm32(uint32(x.Type().Bits()-1)), tmp, _64)
1514
+ m.insert(xor)
1515
+
1516
+ // jmpAtEnd target label is set here.
1517
+ nopEnd, end := m.allocateBrTarget()
1518
+ jmpAtEnd.asJmp(newOperandLabel(end))
1519
+ m.insert(nopEnd)
1520
+
1521
+ m.copyTo(tmp, m.c.VRegOf(instr.Return()))
1522
+ }
1523
+ }
1524
+
1525
+ func (m *machine) lowerUnaryRmR(si *ssa.Instruction, op unaryRmROpcode) {
1526
+ x := si.Arg()
1527
+ if !x.Type().IsInt() {
1528
+ panic("BUG?")
1529
+ }
1530
+ _64 := x.Type().Bits() == 64
1531
+
1532
+ xDef := m.c.ValueDefinition(x)
1533
+ rm := m.getOperand_Mem_Reg(xDef)
1534
+ rd := m.c.VRegOf(si.Return())
1535
+
1536
+ instr := m.allocateInstr()
1537
+ instr.asUnaryRmR(op, rm, rd, _64)
1538
+ m.insert(instr)
1539
+ }
1540
+
1541
+ func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, dst regalloc.VReg) {
1542
+ mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1543
+ load := m.allocateInstr()
1544
+ switch typ {
1545
+ case ssa.TypeI32:
1546
+ load.asMovzxRmR(extModeLQ, mem, dst)
1547
+ case ssa.TypeI64:
1548
+ load.asMov64MR(mem, dst)
1549
+ case ssa.TypeF32:
1550
+ load.asXmmUnaryRmR(sseOpcodeMovss, mem, dst)
1551
+ case ssa.TypeF64:
1552
+ load.asXmmUnaryRmR(sseOpcodeMovsd, mem, dst)
1553
+ case ssa.TypeV128:
1554
+ load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, dst)
1555
+ default:
1556
+ panic("BUG")
1557
+ }
1558
+ m.insert(load)
1559
+ }
1560
+
1561
+ func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, dst regalloc.VReg) {
1562
+ mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1563
+ load := m.allocateInstr()
1564
+ switch op {
1565
+ case ssa.OpcodeUload8:
1566
+ load.asMovzxRmR(extModeBQ, mem, dst)
1567
+ case ssa.OpcodeUload16:
1568
+ load.asMovzxRmR(extModeWQ, mem, dst)
1569
+ case ssa.OpcodeUload32:
1570
+ load.asMovzxRmR(extModeLQ, mem, dst)
1571
+ case ssa.OpcodeSload8:
1572
+ load.asMovsxRmR(extModeBQ, mem, dst)
1573
+ case ssa.OpcodeSload16:
1574
+ load.asMovsxRmR(extModeWQ, mem, dst)
1575
+ case ssa.OpcodeSload32:
1576
+ load.asMovsxRmR(extModeLQ, mem, dst)
1577
+ default:
1578
+ panic("BUG")
1579
+ }
1580
+ m.insert(load)
1581
+ }
1582
+
1583
+ func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) {
1584
+ condDef := m.c.ValueDefinition(cond)
1585
+ if !m.c.MatchInstr(condDef, ssa.OpcodeIcmp) {
1586
+ panic("TODO: ExitIfTrue must come after Icmp at the moment: " + condDef.Instr.Opcode().String())
1587
+ }
1588
+ cvalInstr := condDef.Instr
1589
+ cvalInstr.MarkLowered()
1590
+
1591
+ // We need to copy the execution context to a temp register, because if it's spilled,
1592
+ // it might end up being reloaded inside the exiting branch.
1593
+ execCtxTmp := m.copyToTmp(execCtx)
1594
+
1595
+ x, y, c := cvalInstr.IcmpData()
1596
+ xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1597
+ if !m.tryLowerBandToFlag(xx, yy) {
1598
+ m.lowerIcmpToFlag(xx, yy, x.Type() == ssa.TypeI64)
1599
+ }
1600
+
1601
+ jmpIf := m.allocateInstr()
1602
+ m.insert(jmpIf)
1603
+ l := m.lowerExitWithCode(execCtxTmp, code)
1604
+ jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l))
1605
+ }
1606
+
1607
+ func (m *machine) tryLowerBandToFlag(x, y backend.SSAValueDefinition) (ok bool) {
1608
+ var target backend.SSAValueDefinition
1609
+ var got bool
1610
+ if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 {
1611
+ if m.c.MatchInstr(y, ssa.OpcodeBand) {
1612
+ target = y
1613
+ got = true
1614
+ }
1615
+ }
1616
+
1617
+ if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 {
1618
+ if m.c.MatchInstr(x, ssa.OpcodeBand) {
1619
+ target = x
1620
+ got = true
1621
+ }
1622
+ }
1623
+
1624
+ if !got {
1625
+ return false
1626
+ }
1627
+
1628
+ bandInstr := target.Instr
1629
+ bandX, bandY := bandInstr.Arg2()
1630
+
1631
+ xx := m.getOperand_Reg(m.c.ValueDefinition(bandX))
1632
+ yy := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(bandY))
1633
+ test := m.allocateInstr().asCmpRmiR(false, yy, xx.reg(), bandX.Type() == ssa.TypeI64)
1634
+ m.insert(test)
1635
+ bandInstr.MarkLowered()
1636
+ return true
1637
+ }
1638
+
1639
+ func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) (saveRsp, saveRbp, setExitCode *instruction) {
1640
+ saveRsp = m.allocateInstr().asMovRM(
1641
+ rspVReg,
1642
+ newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.U32(), execCtx)),
1643
+ 8,
1644
+ )
1645
+
1646
+ saveRbp = m.allocateInstr().asMovRM(
1647
+ rbpVReg,
1648
+ newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetFramePointerBeforeGoCall.U32(), execCtx)),
1649
+ 8,
1650
+ )
1651
+ setExitCode = m.allocateInstr().asMovRM(
1652
+ exitCodeReg,
1653
+ newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetExitCodeOffset.U32(), execCtx)),
1654
+ 4,
1655
+ )
1656
+ return
1657
+ }
1658
+
1659
+ func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel label) {
1660
+ exitCodeReg := rbpVReg
1661
+ saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg)
1662
+
1663
+ // Set save RSP, RBP, and write exit code.
1664
+ m.insert(saveRsp)
1665
+ m.insert(saveRbp)
1666
+ m.lowerIconst(exitCodeReg, uint64(code), false)
1667
+ m.insert(setExitCode)
1668
+
1669
+ ripReg := rbpVReg
1670
+
1671
+ // Next is to save the current address for stack unwinding.
1672
+ nop, currentAddrLabel := m.allocateBrTarget()
1673
+ m.insert(nop)
1674
+ readRip := m.allocateInstr().asLEA(newOperandLabel(currentAddrLabel), ripReg)
1675
+ m.insert(readRip)
1676
+ saveRip := m.allocateInstr().asMovRM(
1677
+ ripReg,
1678
+ newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)),
1679
+ 8,
1680
+ )
1681
+ m.insert(saveRip)
1682
+
1683
+ // Finally exit.
1684
+ exitSq := m.allocateExitSeq(execCtx)
1685
+ m.insert(exitSq)
1686
+
1687
+ // Return the label for continuation.
1688
+ continuation, afterLabel := m.allocateBrTarget()
1689
+ m.insert(continuation)
1690
+ return afterLabel
1691
+ }
1692
+
1693
+ func (m *machine) lowerAluRmiROp(si *ssa.Instruction, op aluRmiROpcode) {
1694
+ x, y := si.Arg2()
1695
+ if !x.Type().IsInt() {
1696
+ panic("BUG?")
1697
+ }
1698
+
1699
+ _64 := x.Type().Bits() == 64
1700
+
1701
+ xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1702
+
1703
+ // TODO: commutative args can be swapped if one of them is an immediate.
1704
+ rn := m.getOperand_Reg(xDef)
1705
+ rm := m.getOperand_Mem_Imm32_Reg(yDef)
1706
+ rd := m.c.VRegOf(si.Return())
1707
+
1708
+ // rn is being overwritten, so we first copy its value to a temp register,
1709
+ // in case it is referenced again later.
1710
+ tmp := m.copyToTmp(rn.reg())
1711
+
1712
+ alu := m.allocateInstr()
1713
+ alu.asAluRmiR(op, rm, tmp, _64)
1714
+ m.insert(alu)
1715
+
1716
+ // tmp now contains the result, we copy it to the dest register.
1717
+ m.copyTo(tmp, rd)
1718
+ }
1719
+
1720
+ func (m *machine) lowerShiftR(si *ssa.Instruction, op shiftROp) {
1721
+ x, amt := si.Arg2()
1722
+ if !x.Type().IsInt() {
1723
+ panic("BUG?")
1724
+ }
1725
+ _64 := x.Type().Bits() == 64
1726
+
1727
+ xDef, amtDef := m.c.ValueDefinition(x), m.c.ValueDefinition(amt)
1728
+
1729
+ opAmt := m.getOperand_Imm32_Reg(amtDef)
1730
+ rx := m.getOperand_Reg(xDef)
1731
+ rd := m.c.VRegOf(si.Return())
1732
+
1733
+ // rx is being overwritten, so we first copy its value to a temp register,
1734
+ // in case it is referenced again later.
1735
+ tmpDst := m.copyToTmp(rx.reg())
1736
+
1737
+ if opAmt.kind == operandKindReg {
1738
+ // If opAmt is a register we must copy its value to rcx,
1739
+ // because shiftR encoding mandates that the shift amount is in rcx.
1740
+ m.copyTo(opAmt.reg(), rcxVReg)
1741
+
1742
+ alu := m.allocateInstr()
1743
+ alu.asShiftR(op, newOperandReg(rcxVReg), tmpDst, _64)
1744
+ m.insert(alu)
1745
+
1746
+ } else {
1747
+ alu := m.allocateInstr()
1748
+ alu.asShiftR(op, opAmt, tmpDst, _64)
1749
+ m.insert(alu)
1750
+ }
1751
+
1752
+ // tmp now contains the result, we copy it to the dest register.
1753
+ m.copyTo(tmpDst, rd)
1754
+ }
1755
+
1756
+ func (m *machine) lowerXmmRmR(instr *ssa.Instruction) {
1757
+ x, y := instr.Arg2()
1758
+ if !x.Type().IsFloat() {
1759
+ panic("BUG?")
1760
+ }
1761
+ _64 := x.Type().Bits() == 64
1762
+
1763
+ var op sseOpcode
1764
+ if _64 {
1765
+ switch instr.Opcode() {
1766
+ case ssa.OpcodeFadd:
1767
+ op = sseOpcodeAddsd
1768
+ case ssa.OpcodeFsub:
1769
+ op = sseOpcodeSubsd
1770
+ case ssa.OpcodeFmul:
1771
+ op = sseOpcodeMulsd
1772
+ case ssa.OpcodeFdiv:
1773
+ op = sseOpcodeDivsd
1774
+ default:
1775
+ panic("BUG")
1776
+ }
1777
+ } else {
1778
+ switch instr.Opcode() {
1779
+ case ssa.OpcodeFadd:
1780
+ op = sseOpcodeAddss
1781
+ case ssa.OpcodeFsub:
1782
+ op = sseOpcodeSubss
1783
+ case ssa.OpcodeFmul:
1784
+ op = sseOpcodeMulss
1785
+ case ssa.OpcodeFdiv:
1786
+ op = sseOpcodeDivss
1787
+ default:
1788
+ panic("BUG")
1789
+ }
1790
+ }
1791
+
1792
+ xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
1793
+ rn := m.getOperand_Reg(yDef)
1794
+ rm := m.getOperand_Reg(xDef)
1795
+ rd := m.c.VRegOf(instr.Return())
1796
+
1797
+ // rm is being overwritten, so we first copy its value to a temp register,
1798
+ // in case it is referenced again later.
1799
+ tmp := m.copyToTmp(rm.reg())
1800
+
1801
+ xmm := m.allocateInstr().asXmmRmR(op, rn, tmp)
1802
+ m.insert(xmm)
1803
+
1804
+ m.copyTo(tmp, rd)
1805
+ }
1806
+
1807
+ func (m *machine) lowerSqrt(instr *ssa.Instruction) {
1808
+ x := instr.Arg()
1809
+ if !x.Type().IsFloat() {
1810
+ panic("BUG")
1811
+ }
1812
+ _64 := x.Type().Bits() == 64
1813
+ var op sseOpcode
1814
+ if _64 {
1815
+ op = sseOpcodeSqrtsd
1816
+ } else {
1817
+ op = sseOpcodeSqrtss
1818
+ }
1819
+
1820
+ xDef := m.c.ValueDefinition(x)
1821
+ rm := m.getOperand_Mem_Reg(xDef)
1822
+ rd := m.c.VRegOf(instr.Return())
1823
+
1824
+ xmm := m.allocateInstr().asXmmUnaryRmR(op, rm, rd)
1825
+ m.insert(xmm)
1826
+ }
1827
+
1828
+ func (m *machine) lowerFabsFneg(instr *ssa.Instruction) {
1829
+ x := instr.Arg()
1830
+ if !x.Type().IsFloat() {
1831
+ panic("BUG")
1832
+ }
1833
+ _64 := x.Type().Bits() == 64
1834
+ var op sseOpcode
1835
+ var mask uint64
1836
+ if _64 {
1837
+ switch instr.Opcode() {
1838
+ case ssa.OpcodeFabs:
1839
+ mask, op = 0x7fffffffffffffff, sseOpcodeAndpd
1840
+ case ssa.OpcodeFneg:
1841
+ mask, op = 0x8000000000000000, sseOpcodeXorpd
1842
+ }
1843
+ } else {
1844
+ switch instr.Opcode() {
1845
+ case ssa.OpcodeFabs:
1846
+ mask, op = 0x7fffffff, sseOpcodeAndps
1847
+ case ssa.OpcodeFneg:
1848
+ mask, op = 0x80000000, sseOpcodeXorps
1849
+ }
1850
+ }
1851
+
1852
+ tmp := m.c.AllocateVReg(x.Type())
1853
+
1854
+ xDef := m.c.ValueDefinition(x)
1855
+ rm := m.getOperand_Reg(xDef)
1856
+ rd := m.c.VRegOf(instr.Return())
1857
+
1858
+ m.lowerFconst(tmp, mask, _64)
1859
+
1860
+ xmm := m.allocateInstr().asXmmRmR(op, rm, tmp)
1861
+ m.insert(xmm)
1862
+
1863
+ m.copyTo(tmp, rd)
1864
+ }
1865
+
1866
+ func (m *machine) lowerStore(si *ssa.Instruction) {
1867
+ value, ptr, offset, storeSizeInBits := si.StoreData()
1868
+ rm := m.getOperand_Reg(m.c.ValueDefinition(value))
1869
+ mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
1870
+
1871
+ store := m.allocateInstr()
1872
+ switch value.Type() {
1873
+ case ssa.TypeI32:
1874
+ store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
1875
+ case ssa.TypeI64:
1876
+ store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
1877
+ case ssa.TypeF32:
1878
+ store.asXmmMovRM(sseOpcodeMovss, rm.reg(), mem)
1879
+ case ssa.TypeF64:
1880
+ store.asXmmMovRM(sseOpcodeMovsd, rm.reg(), mem)
1881
+ case ssa.TypeV128:
1882
+ store.asXmmMovRM(sseOpcodeMovdqu, rm.reg(), mem)
1883
+ default:
1884
+ panic("BUG")
1885
+ }
1886
+ m.insert(store)
1887
+ }
1888
+
1889
+ func (m *machine) lowerCall(si *ssa.Instruction) {
1890
+ isDirectCall := si.Opcode() == ssa.OpcodeCall
1891
+ indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
1892
+
1893
+ if isMemmove {
1894
+ // Go's memmove *might* use all xmm0-xmm15, so we need to release them.
1895
+ // https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#architecture-specifics
1896
+ // https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L286-L301
1897
+ for i := regalloc.RealReg(0); i < 16; i++ {
1898
+ m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[xmm0+i]))
1899
+ }
1900
+ // Since Go 1.24 it may also use DX, which is not reserved for the function call's 3 args.
1901
+ // https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L123
1902
+ m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[rdx]))
1903
+ }
1904
+
1905
+ if isDirectCall {
1906
+ call := m.allocateInstr().asCall(directCallee, calleeABI)
1907
+ m.insert(call)
1908
+ } else {
1909
+ ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
1910
+ callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
1911
+ m.insert(callInd)
1912
+ }
1913
+
1914
+ if isMemmove {
1915
+ for i := regalloc.RealReg(0); i < 16; i++ {
1916
+ m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[xmm0+i]))
1917
+ }
1918
+ m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[rdx]))
1919
+ }
1920
+
1921
+ m.insertReturns(si, calleeABI, stackSlotSize)
1922
+ }
1923
+
1924
+ func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, bool, *backend.FunctionABI, int64) {
1925
+ var indirectCalleePtr ssa.Value
1926
+ var directCallee ssa.FuncRef
1927
+ var sigID ssa.SignatureID
1928
+ var args []ssa.Value
1929
+ var isMemmove bool
1930
+ if isDirectCall {
1931
+ directCallee, sigID, args = si.CallData()
1932
+ } else {
1933
+ indirectCalleePtr, sigID, args, isMemmove = si.CallIndirectData()
1934
+ }
1935
+ calleeABI := m.c.GetFunctionABI(m.c.SSABuilder().ResolveSignature(sigID))
1936
+
1937
+ stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize())
1938
+ if m.maxRequiredStackSizeForCalls < stackSlotSize+16 {
1939
+ m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // 16 == return address + RBP.
1940
+ }
1941
+
1942
+ // Note: See machine.SetupPrologue for the stack layout.
1943
+ // The stack pointer decrease/increase will be inserted later in the compilation.
1944
+
1945
+ for i, arg := range args {
1946
+ reg := m.c.VRegOf(arg)
1947
+ def := m.c.ValueDefinition(arg)
1948
+ m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize)
1949
+ }
1950
+ return indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize
1951
+ }
1952
+
1953
+ func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) {
1954
+ var index int
1955
+ r1, rs := si.Returns()
1956
+ if r1.Valid() {
1957
+ m.callerGenFunctionReturnVReg(calleeABI, 0, m.c.VRegOf(r1), stackSlotSize)
1958
+ index++
1959
+ }
1960
+
1961
+ for _, r := range rs {
1962
+ m.callerGenFunctionReturnVReg(calleeABI, index, m.c.VRegOf(r), stackSlotSize)
1963
+ index++
1964
+ }
1965
+ }
1966
+
1967
+ func (m *machine) lowerTailCall(si *ssa.Instruction) {
1968
+ isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall
1969
+ indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
1970
+ if isMemmove {
1971
+ panic("memmove not supported in tail calls")
1972
+ }
1973
+
1974
+ isAllRegs := stackSlotSize == 0
1975
+
1976
+ switch {
1977
+ case isDirectCall && isAllRegs:
1978
+ call := m.allocateInstr().asTailCallReturnCall(directCallee, calleeABI)
1979
+ m.insert(call)
1980
+ case !isDirectCall && isAllRegs:
1981
+ // In a tail call we insert the epilogue before the jump instruction,
1982
+ // so an arbitrary register might be overwritten while restoring the stack.
1983
+ // So, as compared to a regular indirect call, we ensure the pointer is stored
1984
+ // in a caller-saved register (r11).
1985
+ // For details, see internal/engine/RATIONALE.md
1986
+ ptrOp := m.getOperand_Reg(m.c.ValueDefinition(indirectCalleePtr))
1987
+ tmpJmp := r11VReg
1988
+ m.InsertMove(tmpJmp, ptrOp.reg(), ssa.TypeI64)
1989
+ callInd := m.allocateInstr().asTailCallReturnCallIndirect(newOperandReg(tmpJmp), calleeABI)
1990
+ m.insert(callInd)
1991
+ case isDirectCall && !isAllRegs:
1992
+ call := m.allocateInstr().asCall(directCallee, calleeABI)
1993
+ m.insert(call)
1994
+ case !isDirectCall && !isAllRegs:
1995
+ ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
1996
+ callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
1997
+ m.insert(callInd)
1998
+ }
1999
+
2000
+ // If this is a proper tail call, returns will be cleared in the postRegAlloc phase.
2001
+ m.insertReturns(si, calleeABI, stackSlotSize)
2002
+ }
2003
+
2004
+ // callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the
2005
+ // caller side of the function call.
2006
+ func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, stackSlotSize int64) {
2007
+ arg := &a.Args[argIndex]
2008
+ if def.IsFromInstr() {
2009
+ // Constant instructions are inlined.
2010
+ if inst := def.Instr; inst.Constant() {
2011
+ m.insertLoadConstant(inst, reg)
2012
+ }
2013
+ }
2014
+ if arg.Kind == backend.ABIArgKindReg {
2015
+ m.InsertMove(arg.Reg, reg, arg.Type)
2016
+ } else {
2017
+ store := m.allocateInstr()
2018
+ mem := newOperandMem(m.newAmodeImmReg(
2019
+ // -stackSlotSize because the stack pointer is not yet decreased.
2020
+ uint32(arg.Offset-stackSlotSize), rspVReg))
2021
+ switch arg.Type {
2022
+ case ssa.TypeI32:
2023
+ store.asMovRM(reg, mem, 4)
2024
+ case ssa.TypeI64:
2025
+ store.asMovRM(reg, mem, 8)
2026
+ case ssa.TypeF32:
2027
+ store.asXmmMovRM(sseOpcodeMovss, reg, mem)
2028
+ case ssa.TypeF64:
2029
+ store.asXmmMovRM(sseOpcodeMovsd, reg, mem)
2030
+ case ssa.TypeV128:
2031
+ store.asXmmMovRM(sseOpcodeMovdqu, reg, mem)
2032
+ default:
2033
+ panic("BUG")
2034
+ }
2035
+ m.insert(store)
2036
+ }
2037
+ }
2038
+
2039
+ func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, stackSlotSize int64) {
2040
+ r := &a.Rets[retIndex]
2041
+ if r.Kind == backend.ABIArgKindReg {
2042
+ m.InsertMove(reg, r.Reg, r.Type)
2043
+ } else {
2044
+ load := m.allocateInstr()
2045
+ mem := newOperandMem(m.newAmodeImmReg(
2046
+ // -stackSlotSize because the stack pointer is not yet decreased.
2047
+ uint32(a.ArgStackSize+r.Offset-stackSlotSize), rspVReg))
2048
+ switch r.Type {
2049
+ case ssa.TypeI32:
2050
+ load.asMovzxRmR(extModeLQ, mem, reg)
2051
+ case ssa.TypeI64:
2052
+ load.asMov64MR(mem, reg)
2053
+ case ssa.TypeF32:
2054
+ load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg)
2055
+ case ssa.TypeF64:
2056
+ load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg)
2057
+ case ssa.TypeV128:
2058
+ load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg)
2059
+ default:
2060
+ panic("BUG")
2061
+ }
2062
+ m.insert(load)
2063
+ }
2064
+ }
2065
+
2066
+ // InsertMove implements backend.Machine.
2067
+ func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) {
2068
+ switch typ {
2069
+ case ssa.TypeI32, ssa.TypeI64:
2070
+ i := m.allocateInstr().asMovRR(src, dst, typ.Bits() == 64)
2071
+ m.insert(i)
2072
+ case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
2073
+ var op sseOpcode
2074
+ switch typ {
2075
+ case ssa.TypeF32:
2076
+ op = sseOpcodeMovss
2077
+ case ssa.TypeF64:
2078
+ op = sseOpcodeMovsd
2079
+ case ssa.TypeV128:
2080
+ op = sseOpcodeMovdqa
2081
+ }
2082
+ i := m.allocateInstr().asXmmUnaryRmR(op, newOperandReg(src), dst)
2083
+ m.insert(i)
2084
+ default:
2085
+ panic("BUG")
2086
+ }
2087
+ }
2088
+
2089
+ // Format implements backend.Machine.
2090
+ func (m *machine) Format() string {
2091
+ begins := map[*instruction]label{}
2092
+ for l := label(0); l < m.nextLabel; l++ {
2093
+ pos := m.labelPositionPool.Get(int(l))
2094
+ if pos != nil {
2095
+ begins[pos.begin] = l
2096
+ }
2097
+ }
2098
+
2099
+ var lines []string
2100
+ for cur := m.rootInstr; cur != nil; cur = cur.next {
2101
+ if l, ok := begins[cur]; ok {
2102
+ var labelStr string
2103
+ if l <= m.maxSSABlockID {
2104
+ labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, l)
2105
+ } else {
2106
+ labelStr = fmt.Sprintf("%s:", l)
2107
+ }
2108
+ lines = append(lines, labelStr)
2109
+ }
2110
+ if cur.kind == nop0 {
2111
+ continue
2112
+ }
2113
+ lines = append(lines, "\t"+cur.String())
2114
+ }
2115
+ for _, vc := range m.consts {
2116
+ if vc._var == nil {
2117
+ lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label, vc.lo, vc.hi))
2118
+ } else {
2119
+ lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label, vc._var))
2120
+ }
2121
+ }
2122
+ return "\n" + strings.Join(lines, "\n") + "\n"
2123
+ }
2124
+
2125
+ func (m *machine) encodeWithoutSSA(root *instruction) {
2126
+ m.labelResolutionPends = m.labelResolutionPends[:0]
2127
+ bufPtr := m.c.BufPtr()
2128
+ for cur := root; cur != nil; cur = cur.next {
2129
+ offset := int64(len(*bufPtr))
2130
+ if cur.kind == nop0 {
2131
+ l := cur.nop0Label()
2132
+ pos := m.labelPositionPool.Get(int(l))
2133
+ if pos != nil {
2134
+ pos.binaryOffset = offset
2135
+ }
2136
+ }
2137
+
2138
+ needLabelResolution := cur.encode(m.c)
2139
+ if needLabelResolution {
2140
+ m.labelResolutionPends = append(m.labelResolutionPends,
2141
+ labelResolutionPend{instr: cur, imm32Offset: int64(len(*bufPtr)) - 4},
2142
+ )
2143
+ }
2144
+ }
2145
+
2146
+ for i := range m.labelResolutionPends {
2147
+ p := &m.labelResolutionPends[i]
2148
+ switch p.instr.kind {
2149
+ case jmp, jmpIf, lea:
2150
+ target := p.instr.jmpLabel()
2151
+ targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
2152
+ imm32Offset := p.imm32Offset
2153
+ jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
2154
+ binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset))
2155
+ default:
2156
+ panic("BUG")
2157
+ }
2158
+ }
2159
+ }
2160
+
2161
+ // Encode implements backend.Machine Encode.
2162
+ func (m *machine) Encode(ctx context.Context) (err error) {
2163
+ bufPtr := m.c.BufPtr()
2164
+
2165
+ var fn string
2166
+ var fnIndex int
2167
+ var labelPosToLabel map[*labelPosition]label
2168
+ if wazevoapi.PerfMapEnabled {
2169
+ fn = wazevoapi.GetCurrentFunctionName(ctx)
2170
+ labelPosToLabel = make(map[*labelPosition]label)
2171
+ for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ {
2172
+ pos := m.labelPositionPool.Get(i)
2173
+ labelPosToLabel[pos] = label(i)
2174
+ }
2175
+ fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx)
2176
+ }
2177
+
2178
+ m.labelResolutionPends = m.labelResolutionPends[:0]
2179
+ for _, pos := range m.orderedSSABlockLabelPos {
2180
+ offset := int64(len(*bufPtr))
2181
+ pos.binaryOffset = offset
2182
+ for cur := pos.begin; cur != pos.end.next; cur = cur.next {
2183
+ offset := int64(len(*bufPtr))
2184
+
2185
+ switch cur.kind {
2186
+ case nop0:
2187
+ l := cur.nop0Label()
2188
+ if pos := m.labelPositionPool.Get(int(l)); pos != nil {
2189
+ pos.binaryOffset = offset
2190
+ }
2191
+ case sourceOffsetInfo:
2192
+ m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo())
2193
+ }
2194
+
2195
+ needLabelResolution := cur.encode(m.c)
2196
+ if needLabelResolution {
2197
+ m.labelResolutionPends = append(m.labelResolutionPends,
2198
+ labelResolutionPend{instr: cur, instrOffset: offset, imm32Offset: int64(len(*bufPtr)) - 4},
2199
+ )
2200
+ }
2201
+ }
2202
+
2203
+ if wazevoapi.PerfMapEnabled {
2204
+ l := labelPosToLabel[pos]
2205
+ size := int64(len(*bufPtr)) - offset
2206
+ wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, l))
2207
+ }
2208
+ }
2209
+
2210
+ for i := range m.consts {
2211
+ offset := int64(len(*bufPtr))
2212
+ vc := &m.consts[i]
2213
+ vc.labelPos.binaryOffset = offset
2214
+ if vc._var == nil {
2215
+ lo, hi := vc.lo, vc.hi
2216
+ m.c.Emit8Bytes(lo)
2217
+ m.c.Emit8Bytes(hi)
2218
+ } else {
2219
+ for _, b := range vc._var {
2220
+ m.c.EmitByte(b)
2221
+ }
2222
+ }
2223
+ }
2224
+
2225
+ buf := *bufPtr
2226
+ for i := range m.labelResolutionPends {
2227
+ p := &m.labelResolutionPends[i]
2228
+ switch p.instr.kind {
2229
+ case jmp, jmpIf, lea, xmmUnaryRmR:
2230
+ target := p.instr.jmpLabel()
2231
+ targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
2232
+ imm32Offset := p.imm32Offset
2233
+ jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
2234
+ binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset))
2235
+ case jmpTableIsland:
2236
+ tableBegin := p.instrOffset
2237
+ // Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes.
2238
+ targets := m.jmpTableTargets[p.instr.u1]
2239
+ for i, l := range targets {
2240
+ targetOffset := m.labelPositionPool.Get(int(l)).binaryOffset
2241
+ jmpOffset := targetOffset - tableBegin
2242
+ binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset))
2243
+ }
2244
+ default:
2245
+ panic("BUG")
2246
+ }
2247
+ }
2248
+ return
2249
+ }
2250
+
2251
+ // ResolveRelocations implements backend.Machine.
2252
+ func (m *machine) ResolveRelocations(refToBinaryOffset []int, _ int, binary []byte, relocations []backend.RelocationInfo, _ []int) {
2253
+ for _, r := range relocations {
2254
+ offset := r.Offset
2255
+ calleeFnOffset := refToBinaryOffset[r.FuncRef]
2256
+ // offset is the offset of the last 4 bytes of the call instruction.
2257
+ callInstrOffsetBytes := binary[offset : offset+4]
2258
+ diff := int64(calleeFnOffset) - (offset + 4) // +4 because we want the offset of the next instruction (In x64, RIP always points to the next instruction).
2259
+ callInstrOffsetBytes[0] = byte(diff)
2260
+ callInstrOffsetBytes[1] = byte(diff >> 8)
2261
+ callInstrOffsetBytes[2] = byte(diff >> 16)
2262
+ callInstrOffsetBytes[3] = byte(diff >> 24)
2263
+ }
2264
+ }
2265
+
2266
+ // CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo.
2267
+ func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return }
2268
+
2269
+ func (m *machine) lowerIcmpToFlag(xd, yd backend.SSAValueDefinition, _64 bool) {
2270
+ x := m.getOperand_Reg(xd)
2271
+ y := m.getOperand_Mem_Imm32_Reg(yd)
2272
+ cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64)
2273
+ m.insert(cmp)
2274
+ }
2275
+
2276
+ func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and bool) {
2277
+ x, y, c := instr.FcmpData()
2278
+ switch c {
2279
+ case ssa.FloatCmpCondEqual:
2280
+ f1, f2 = condNP, condZ
2281
+ and = true
2282
+ case ssa.FloatCmpCondNotEqual:
2283
+ f1, f2 = condP, condNZ
2284
+ case ssa.FloatCmpCondLessThan:
2285
+ f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThan)
2286
+ f2 = condInvalid
2287
+ x, y = y, x
2288
+ case ssa.FloatCmpCondLessThanOrEqual:
2289
+ f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThanOrEqual)
2290
+ f2 = condInvalid
2291
+ x, y = y, x
2292
+ default:
2293
+ f1 = condFromSSAFloatCmpCond(c)
2294
+ f2 = condInvalid
2295
+ }
2296
+
2297
+ var opc sseOpcode
2298
+ if x.Type() == ssa.TypeF32 {
2299
+ opc = sseOpcodeUcomiss
2300
+ } else {
2301
+ opc = sseOpcodeUcomisd
2302
+ }
2303
+
2304
+ xr := m.getOperand_Reg(m.c.ValueDefinition(x))
2305
+ yr := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
2306
+ m.insert(m.allocateInstr().asXmmCmpRmR(opc, yr, xr.reg()))
2307
+ return
2308
+ }
2309
+
2310
+ // allocateInstr allocates an instruction.
2311
+ func (m *machine) allocateInstr() *instruction {
2312
+ instr := m.instrPool.Allocate()
2313
+ if !m.regAllocStarted {
2314
+ instr.addedBeforeRegAlloc = true
2315
+ }
2316
+ return instr
2317
+ }
2318
+
2319
+ func (m *machine) allocateNop() *instruction {
2320
+ instr := m.allocateInstr()
2321
+ instr.kind = nop0
2322
+ return instr
2323
+ }
2324
+
2325
+ func (m *machine) insert(i *instruction) {
2326
+ m.pendingInstructions = append(m.pendingInstructions, i)
2327
+ }
2328
+
2329
+ func (m *machine) allocateBrTarget() (nop *instruction, l label) { //nolint
2330
+ l, pos := m.allocateLabel()
2331
+ nop = m.allocateInstr()
2332
+ nop.asNop0WithLabel(l)
2333
+ pos.begin, pos.end = nop, nop
2334
+ return
2335
+ }
2336
+
2337
+ func (m *machine) allocateLabel() (label, *labelPosition) {
2338
+ l := m.nextLabel
2339
+ pos := m.labelPositionPool.GetOrAllocate(int(l))
2340
+ m.nextLabel++
2341
+ return l, pos
2342
+ }
2343
+
2344
+ func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 {
2345
+ offset, ok := m.spillSlots[id]
2346
+ if !ok {
2347
+ offset = m.spillSlotSize
2348
+ m.spillSlots[id] = offset
2349
+ m.spillSlotSize += int64(size)
2350
+ }
2351
+ return offset
2352
+ }
2353
+
2354
+ func (m *machine) copyTo(src regalloc.VReg, dst regalloc.VReg) {
2355
+ mov := m.allocateInstr()
2356
+ if src.RegType() == regalloc.RegTypeInt {
2357
+ mov.asMovRR(src, dst, true)
2358
+ } else {
2359
+ mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst)
2360
+ }
2361
+ m.insert(mov)
2362
+ }
2363
+
2364
+ func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg {
2365
+ typ := m.c.TypeOf(v)
2366
+ tmp := m.c.AllocateVReg(typ)
2367
+ m.copyTo(v, tmp)
2368
+ return tmp
2369
+ }
2370
+
2371
+ func (m *machine) requiredStackSize() int64 {
2372
+ return m.maxRequiredStackSizeForCalls +
2373
+ m.frameSize() +
2374
+ 16 + // Need for stack checking.
2375
+ 16 // return address and the caller RBP.
2376
+ }
2377
+
2378
+ func (m *machine) frameSize() int64 {
2379
+ s := m.clobberedRegSlotSize() + m.spillSlotSize
2380
+ if s&0xf != 0 {
2381
+ panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s))
2382
+ }
2383
+ return s
2384
+ }
2385
+
2386
+ func (m *machine) clobberedRegSlotSize() int64 {
2387
+ return int64(len(m.clobberedRegs) * 16)
2388
+ }
2389
+
2390
+ func (m *machine) lowerIDivRem(si *ssa.Instruction, isDiv bool, signed bool) {
2391
+ x, y, execCtx := si.Arg3()
2392
+
2393
+ dividend := m.getOperand_Reg(m.c.ValueDefinition(x))
2394
+ divisor := m.getOperand_Reg(m.c.ValueDefinition(y))
2395
+ ctxVReg := m.c.VRegOf(execCtx)
2396
+ tmpGp := m.c.AllocateVReg(si.Return().Type())
2397
+
2398
+ m.copyTo(dividend.reg(), raxVReg)
2399
+ m.insert(m.allocateInstr().asDefineUninitializedReg(rdxVReg))
2400
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2401
+ seq := m.allocateInstr().asIdivRemSequence(ctxVReg, divisor.reg(), tmpGp, isDiv, signed, x.Type().Bits() == 64)
2402
+ m.insert(seq)
2403
+ rd := m.c.VRegOf(si.Return())
2404
+ if isDiv {
2405
+ m.copyTo(raxVReg, rd)
2406
+ } else {
2407
+ m.copyTo(rdxVReg, rd)
2408
+ }
2409
+ }
2410
+
2411
+ func (m *machine) lowerIDivRemSequenceAfterRegAlloc(i *instruction) {
2412
+ execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData()
2413
+
2414
+ dividend := raxVReg
2415
+
2416
+ // Ensure yr is not zero.
2417
+ test := m.allocateInstr()
2418
+ test.asCmpRmiR(false, newOperandReg(divisor), divisor, _64)
2419
+ m.insert(test)
2420
+
2421
+ jnz := m.allocateInstr()
2422
+ m.insert(jnz)
2423
+
2424
+ nz := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerDivisionByZero)
2425
+
2426
+ // If not zero, we can proceed with the division.
2427
+ jnz.asJmpIf(condNZ, newOperandLabel(nz))
2428
+
2429
+ var ifRemNeg1 *instruction
2430
+ if signed {
2431
+ var neg1 uint64
2432
+ if _64 {
2433
+ neg1 = 0xffffffffffffffff
2434
+ } else {
2435
+ neg1 = 0xffffffff
2436
+ }
2437
+ m.lowerIconst(tmpGp, neg1, _64)
2438
+
2439
+ if isDiv {
2440
+ // For signed division, we have to have branches for "math.MinInt{32,64} / -1"
2441
+ // case which results in the floating point exception via division error as
2442
+ // the resulting value exceeds the maximum of signed int.
2443
+
2444
+ // First, we check if the divisor is -1.
2445
+ cmp := m.allocateInstr()
2446
+ cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
2447
+ m.insert(cmp)
2448
+
2449
+ ifNotNeg1 := m.allocateInstr()
2450
+ m.insert(ifNotNeg1)
2451
+
2452
+ var minInt uint64
2453
+ if _64 {
2454
+ minInt = 0x8000000000000000
2455
+ } else {
2456
+ minInt = 0x80000000
2457
+ }
2458
+ m.lowerIconst(tmpGp, minInt, _64)
2459
+
2460
+ // Next we check if the quotient is the most negative value for the signed integer, i.e.
2461
+ // if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1) respectively.
2462
+ cmp2 := m.allocateInstr()
2463
+ cmp2.asCmpRmiR(true, newOperandReg(tmpGp), dividend, _64)
2464
+ m.insert(cmp2)
2465
+
2466
+ ifNotMinInt := m.allocateInstr()
2467
+ m.insert(ifNotMinInt)
2468
+
2469
+ // Trap if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1),
2470
+ // as that is the overflow in division as the result becomes 2^31 which is larger than
2471
+ // the maximum of signed 32-bit int (2^31-1).
2472
+ end := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2473
+ ifNotNeg1.asJmpIf(condNZ, newOperandLabel(end))
2474
+ ifNotMinInt.asJmpIf(condNZ, newOperandLabel(end))
2475
+ } else {
2476
+ // If it is remainder, zeros DX register and compare the divisor to -1.
2477
+ xor := m.allocateInstr().asZeros(rdxVReg)
2478
+ m.insert(xor)
2479
+
2480
+ // We check if the divisor is -1.
2481
+ cmp := m.allocateInstr()
2482
+ cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
2483
+ m.insert(cmp)
2484
+
2485
+ ifRemNeg1 = m.allocateInstr()
2486
+ m.insert(ifRemNeg1)
2487
+ }
2488
+
2489
+ // Sign-extend DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
2490
+ sed := m.allocateInstr()
2491
+ sed.asSignExtendData(_64)
2492
+ m.insert(sed)
2493
+ } else {
2494
+ // Zeros DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
2495
+ zeros := m.allocateInstr().asZeros(rdxVReg)
2496
+ m.insert(zeros)
2497
+ }
2498
+
2499
+ div := m.allocateInstr()
2500
+ div.asDiv(newOperandReg(divisor), signed, _64)
2501
+ m.insert(div)
2502
+
2503
+ nop, end := m.allocateBrTarget()
2504
+ m.insert(nop)
2505
+ // If we are compiling a Rem instruction, when the divisor is -1 we land at the end of the function.
2506
+ if ifRemNeg1 != nil {
2507
+ ifRemNeg1.asJmpIf(condZ, newOperandLabel(end))
2508
+ }
2509
+ }
2510
+
2511
+ func (m *machine) lowerRound(instr *ssa.Instruction, imm roundingMode) {
2512
+ x := instr.Arg()
2513
+ if !x.Type().IsFloat() {
2514
+ panic("BUG?")
2515
+ }
2516
+ var op sseOpcode
2517
+ if x.Type().Bits() == 64 {
2518
+ op = sseOpcodeRoundsd
2519
+ } else {
2520
+ op = sseOpcodeRoundss
2521
+ }
2522
+
2523
+ xDef := m.c.ValueDefinition(x)
2524
+ rm := m.getOperand_Mem_Reg(xDef)
2525
+ rd := m.c.VRegOf(instr.Return())
2526
+
2527
+ xmm := m.allocateInstr().asXmmUnaryRmRImm(op, uint8(imm), rm, rd)
2528
+ m.insert(xmm)
2529
+ }
2530
+
2531
+ func (m *machine) lowerFminFmax(instr *ssa.Instruction) {
2532
+ x, y := instr.Arg2()
2533
+ if !x.Type().IsFloat() {
2534
+ panic("BUG?")
2535
+ }
2536
+
2537
+ _64 := x.Type().Bits() == 64
2538
+ isMin := instr.Opcode() == ssa.OpcodeFmin
2539
+ var minMaxOp sseOpcode
2540
+
2541
+ switch {
2542
+ case _64 && isMin:
2543
+ minMaxOp = sseOpcodeMinpd
2544
+ case _64 && !isMin:
2545
+ minMaxOp = sseOpcodeMaxpd
2546
+ case !_64 && isMin:
2547
+ minMaxOp = sseOpcodeMinps
2548
+ case !_64 && !isMin:
2549
+ minMaxOp = sseOpcodeMaxps
2550
+ }
2551
+
2552
+ xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
2553
+ rm := m.getOperand_Reg(xDef)
2554
+ // We cannot ensure that y is aligned to 16 bytes, so we have to use it on reg.
2555
+ rn := m.getOperand_Reg(yDef)
2556
+ rd := m.c.VRegOf(instr.Return())
2557
+
2558
+ tmp := m.copyToTmp(rm.reg())
2559
+
2560
+ // Check if this is (either x1 or x2 is NaN) or (x1 equals x2) case.
2561
+ cmp := m.allocateInstr()
2562
+ if _64 {
2563
+ cmp.asXmmCmpRmR(sseOpcodeUcomisd, rn, tmp)
2564
+ } else {
2565
+ cmp.asXmmCmpRmR(sseOpcodeUcomiss, rn, tmp)
2566
+ }
2567
+ m.insert(cmp)
2568
+
2569
+ // At this point, we have the three cases of conditional flags below
2570
+ // (See https://www.felixcloutier.com/x86/ucomiss#operation for detail.)
2571
+ //
2572
+ // 1) Two values are NaN-free and different: All flags are cleared.
2573
+ // 2) Two values are NaN-free and equal: Only ZF flags is set.
2574
+ // 3) One of Two values is NaN: ZF, PF and CF flags are set.
2575
+
2576
+ // Jump instruction to handle 1) case by checking the ZF flag
2577
+ // as ZF is only set for 2) and 3) cases.
2578
+ nanFreeOrDiffJump := m.allocateInstr()
2579
+ m.insert(nanFreeOrDiffJump)
2580
+
2581
+ // Start handling 2) and 3).
2582
+
2583
+ // Jump if one of two values is NaN by checking the parity flag (PF).
2584
+ ifIsNan := m.allocateInstr()
2585
+ m.insert(ifIsNan)
2586
+
2587
+ // Start handling 2) NaN-free and equal.
2588
+
2589
+ // Before we exit this case, we have to ensure that positive zero (or negative zero for min instruction) is
2590
+ // returned if two values are positive and negative zeros.
2591
+ var op sseOpcode
2592
+ switch {
2593
+ case !_64 && isMin:
2594
+ op = sseOpcodeOrps
2595
+ case _64 && isMin:
2596
+ op = sseOpcodeOrpd
2597
+ case !_64 && !isMin:
2598
+ op = sseOpcodeAndps
2599
+ case _64 && !isMin:
2600
+ op = sseOpcodeAndpd
2601
+ }
2602
+ orAnd := m.allocateInstr()
2603
+ orAnd.asXmmRmR(op, rn, tmp)
2604
+ m.insert(orAnd)
2605
+
2606
+ // Done, jump to end.
2607
+ sameExitJump := m.allocateInstr()
2608
+ m.insert(sameExitJump)
2609
+
2610
+ // Start handling 3) either is NaN.
2611
+ isNanTarget, isNan := m.allocateBrTarget()
2612
+ m.insert(isNanTarget)
2613
+ ifIsNan.asJmpIf(condP, newOperandLabel(isNan))
2614
+
2615
+ // We emit the ADD instruction to produce the NaN in tmp.
2616
+ add := m.allocateInstr()
2617
+ if _64 {
2618
+ add.asXmmRmR(sseOpcodeAddsd, rn, tmp)
2619
+ } else {
2620
+ add.asXmmRmR(sseOpcodeAddss, rn, tmp)
2621
+ }
2622
+ m.insert(add)
2623
+
2624
+ // Exit from the NaN case branch.
2625
+ nanExitJmp := m.allocateInstr()
2626
+ m.insert(nanExitJmp)
2627
+
2628
+ // Start handling 1).
2629
+ doMinMaxTarget, doMinMax := m.allocateBrTarget()
2630
+ m.insert(doMinMaxTarget)
2631
+ nanFreeOrDiffJump.asJmpIf(condNZ, newOperandLabel(doMinMax))
2632
+
2633
+ // Now handle the NaN-free and different values case.
2634
+ minMax := m.allocateInstr()
2635
+ minMax.asXmmRmR(minMaxOp, rn, tmp)
2636
+ m.insert(minMax)
2637
+
2638
+ endNop, end := m.allocateBrTarget()
2639
+ m.insert(endNop)
2640
+ nanExitJmp.asJmp(newOperandLabel(end))
2641
+ sameExitJump.asJmp(newOperandLabel(end))
2642
+
2643
+ m.copyTo(tmp, rd)
2644
+ }
2645
+
2646
+ func (m *machine) lowerFcopysign(instr *ssa.Instruction) {
2647
+ x, y := instr.Arg2()
2648
+ if !x.Type().IsFloat() {
2649
+ panic("BUG")
2650
+ }
2651
+
2652
+ _64 := x.Type().Bits() == 64
2653
+
2654
+ xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
2655
+ rm := m.getOperand_Reg(xDef)
2656
+ rn := m.getOperand_Reg(yDef)
2657
+ rd := m.c.VRegOf(instr.Return())
2658
+
2659
+ // Clear the non-sign bits of src via AND with the mask.
2660
+ var opAnd, opOr sseOpcode
2661
+ var signMask uint64
2662
+ if _64 {
2663
+ signMask, opAnd, opOr = 0x8000000000000000, sseOpcodeAndpd, sseOpcodeOrpd
2664
+ } else {
2665
+ signMask, opAnd, opOr = 0x80000000, sseOpcodeAndps, sseOpcodeOrps
2666
+ }
2667
+
2668
+ signBitReg := m.c.AllocateVReg(x.Type())
2669
+ m.lowerFconst(signBitReg, signMask, _64)
2670
+ nonSignBitReg := m.c.AllocateVReg(x.Type())
2671
+ m.lowerFconst(nonSignBitReg, ^signMask, _64)
2672
+
2673
+ // Extract the sign bits of rn.
2674
+ and := m.allocateInstr().asXmmRmR(opAnd, rn, signBitReg)
2675
+ m.insert(and)
2676
+
2677
+ // Clear the sign bit of dst via AND with the non-sign bit mask.
2678
+ xor := m.allocateInstr().asXmmRmR(opAnd, rm, nonSignBitReg)
2679
+ m.insert(xor)
2680
+
2681
+ // Copy the sign bits of src to dst via OR.
2682
+ or := m.allocateInstr().asXmmRmR(opOr, newOperandReg(signBitReg), nonSignBitReg)
2683
+ m.insert(or)
2684
+
2685
+ m.copyTo(nonSignBitReg, rd)
2686
+ }
2687
+
2688
+ func (m *machine) lowerBitcast(instr *ssa.Instruction) {
2689
+ x, dstTyp := instr.BitcastData()
2690
+ srcTyp := x.Type()
2691
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
2692
+ rd := m.c.VRegOf(instr.Return())
2693
+ switch {
2694
+ case srcTyp == ssa.TypeF32 && dstTyp == ssa.TypeI32:
2695
+ cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovd, rn.reg(), rd, false)
2696
+ m.insert(cvt)
2697
+ case srcTyp == ssa.TypeI32 && dstTyp == ssa.TypeF32:
2698
+ cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovd, rn, rd, false)
2699
+ m.insert(cvt)
2700
+ case srcTyp == ssa.TypeF64 && dstTyp == ssa.TypeI64:
2701
+ cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovq, rn.reg(), rd, true)
2702
+ m.insert(cvt)
2703
+ case srcTyp == ssa.TypeI64 && dstTyp == ssa.TypeF64:
2704
+ cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovq, rn, rd, true)
2705
+ m.insert(cvt)
2706
+ default:
2707
+ panic(fmt.Sprintf("invalid bitcast from %s to %s", srcTyp, dstTyp))
2708
+ }
2709
+ }
2710
+
2711
+ func (m *machine) lowerFcvtToSint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
2712
+ var tmpXmm regalloc.VReg
2713
+ if dst64 {
2714
+ tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
2715
+ } else {
2716
+ tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
2717
+ }
2718
+
2719
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
2720
+ tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
2721
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2722
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
2723
+
2724
+ m.insert(m.allocateFcvtToSintSequence(ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat))
2725
+ m.copyTo(tmpGp, rd)
2726
+ }
2727
+
2728
+ func (m *machine) lowerFcvtToSintSequenceAfterRegalloc(i *instruction) {
2729
+ execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData()
2730
+ var cmpOp, truncOp sseOpcode
2731
+ if src64 {
2732
+ cmpOp, truncOp = sseOpcodeUcomisd, sseOpcodeCvttsd2si
2733
+ } else {
2734
+ cmpOp, truncOp = sseOpcodeUcomiss, sseOpcodeCvttss2si
2735
+ }
2736
+
2737
+ trunc := m.allocateInstr()
2738
+ trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
2739
+ m.insert(trunc)
2740
+
2741
+ // Check if the dst operand was INT_MIN, by checking it against 1.
2742
+ cmp1 := m.allocateInstr()
2743
+ cmp1.asCmpRmiR(true, newOperandImm32(1), tmpGp, dst64)
2744
+ m.insert(cmp1)
2745
+
2746
+ // If no overflow, then we are done.
2747
+ doneTarget, done := m.allocateBrTarget()
2748
+ ifNoOverflow := m.allocateInstr()
2749
+ ifNoOverflow.asJmpIf(condNO, newOperandLabel(done))
2750
+ m.insert(ifNoOverflow)
2751
+
2752
+ // Now, check for NaN.
2753
+ cmpNan := m.allocateInstr()
2754
+ cmpNan.asXmmCmpRmR(cmpOp, newOperandReg(src), src)
2755
+ m.insert(cmpNan)
2756
+
2757
+ // We allocate the "non-nan target" here, but we will insert it later.
2758
+ notNanTarget, notNaN := m.allocateBrTarget()
2759
+ ifNotNan := m.allocateInstr()
2760
+ ifNotNan.asJmpIf(condNP, newOperandLabel(notNaN))
2761
+ m.insert(ifNotNan)
2762
+
2763
+ if sat {
2764
+ // If NaN and saturating, return 0.
2765
+ zeroDst := m.allocateInstr().asZeros(tmpGp)
2766
+ m.insert(zeroDst)
2767
+
2768
+ jmpEnd := m.allocateInstr()
2769
+ jmpEnd.asJmp(newOperandLabel(done))
2770
+ m.insert(jmpEnd)
2771
+
2772
+ // Otherwise:
2773
+ m.insert(notNanTarget)
2774
+
2775
+ // Zero-out the tmp register.
2776
+ zero := m.allocateInstr().asZeros(tmpXmm)
2777
+ m.insert(zero)
2778
+
2779
+ cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2780
+ m.insert(cmpXmm)
2781
+
2782
+ // if >= jump to end.
2783
+ jmpEnd2 := m.allocateInstr()
2784
+ jmpEnd2.asJmpIf(condB, newOperandLabel(done))
2785
+ m.insert(jmpEnd2)
2786
+
2787
+ // Otherwise, saturate to INT_MAX.
2788
+ if dst64 {
2789
+ m.lowerIconst(tmpGp, math.MaxInt64, dst64)
2790
+ } else {
2791
+ m.lowerIconst(tmpGp, math.MaxInt32, dst64)
2792
+ }
2793
+
2794
+ } else {
2795
+
2796
+ // If non-sat, NaN, trap.
2797
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
2798
+
2799
+ // Otherwise, we will jump here.
2800
+ m.insert(notNanTarget)
2801
+
2802
+ // jump over trap if src larger than threshold
2803
+ condAboveThreshold := condNB
2804
+
2805
+ // The magic constants are various combination of minInt for int[32|64] represented as float[32|64].
2806
+ var minInt uint64
2807
+ switch {
2808
+ case src64 && dst64:
2809
+ minInt = 0xc3e0000000000000
2810
+ case src64 && !dst64:
2811
+ condAboveThreshold = condNBE
2812
+ minInt = 0xC1E0_0000_0020_0000
2813
+ case !src64 && dst64:
2814
+ minInt = 0xDF00_0000
2815
+ case !src64 && !dst64:
2816
+ minInt = 0xCF00_0000
2817
+ }
2818
+
2819
+ loadToGP := m.allocateInstr().asImm(tmpGp2, minInt, src64)
2820
+ m.insert(loadToGP)
2821
+
2822
+ movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp2), tmpXmm, src64)
2823
+ m.insert(movToXmm)
2824
+
2825
+ cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2826
+ m.insert(cmpXmm)
2827
+
2828
+ jmpIfLarger := m.allocateInstr()
2829
+ checkPositiveTarget, checkPositive := m.allocateBrTarget()
2830
+ jmpIfLarger.asJmpIf(condAboveThreshold, newOperandLabel(checkPositive))
2831
+ m.insert(jmpIfLarger)
2832
+
2833
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2834
+
2835
+ // If positive, it was a real overflow.
2836
+ m.insert(checkPositiveTarget)
2837
+
2838
+ // Zero out the temp register.
2839
+ xorpd := m.allocateInstr()
2840
+ xorpd.asXmmRmR(sseOpcodeXorpd, newOperandReg(tmpXmm), tmpXmm)
2841
+ m.insert(xorpd)
2842
+
2843
+ pos := m.allocateInstr()
2844
+ pos.asXmmCmpRmR(cmpOp, newOperandReg(src), tmpXmm)
2845
+ m.insert(pos)
2846
+
2847
+ // If >= jump to end.
2848
+ jmp := m.allocateInstr().asJmpIf(condNB, newOperandLabel(done))
2849
+ m.insert(jmp)
2850
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2851
+ }
2852
+
2853
+ m.insert(doneTarget)
2854
+ }
2855
+
2856
+ func (m *machine) lowerFcvtToUint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
2857
+ tmpXmm, tmpXmm2 := m.c.AllocateVReg(ssa.TypeF64), m.c.AllocateVReg(ssa.TypeF64)
2858
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
2859
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm2))
2860
+ tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
2861
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
2862
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
2863
+
2864
+ m.insert(m.allocateFcvtToUintSequence(
2865
+ ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat,
2866
+ ))
2867
+ m.copyTo(tmpGp, rd)
2868
+ }
2869
+
2870
+ func (m *machine) lowerFcvtToUintSequenceAfterRegalloc(i *instruction) {
2871
+ execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData()
2872
+
2873
+ var subOp, cmpOp, truncOp sseOpcode
2874
+ if src64 {
2875
+ subOp, cmpOp, truncOp = sseOpcodeSubsd, sseOpcodeUcomisd, sseOpcodeCvttsd2si
2876
+ } else {
2877
+ subOp, cmpOp, truncOp = sseOpcodeSubss, sseOpcodeUcomiss, sseOpcodeCvttss2si
2878
+ }
2879
+
2880
+ doneTarget, done := m.allocateBrTarget()
2881
+
2882
+ switch {
2883
+ case src64 && dst64:
2884
+ loadToGP := m.allocateInstr().asImm(tmpGp, 0x43e0000000000000, true)
2885
+ m.insert(loadToGP)
2886
+ movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
2887
+ m.insert(movToXmm)
2888
+ case src64 && !dst64:
2889
+ loadToGP := m.allocateInstr().asImm(tmpGp, 0x41e0000000000000, true)
2890
+ m.insert(loadToGP)
2891
+ movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
2892
+ m.insert(movToXmm)
2893
+ case !src64 && dst64:
2894
+ loadToGP := m.allocateInstr().asImm(tmpGp, 0x5f000000, false)
2895
+ m.insert(loadToGP)
2896
+ movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
2897
+ m.insert(movToXmm)
2898
+ case !src64 && !dst64:
2899
+ loadToGP := m.allocateInstr().asImm(tmpGp, 0x4f000000, false)
2900
+ m.insert(loadToGP)
2901
+ movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
2902
+ m.insert(movToXmm)
2903
+ }
2904
+
2905
+ cmp := m.allocateInstr()
2906
+ cmp.asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
2907
+ m.insert(cmp)
2908
+
2909
+ // If above `tmp` ("large threshold"), jump to `ifAboveThreshold`
2910
+ ifAboveThresholdTarget, ifAboveThreshold := m.allocateBrTarget()
2911
+ jmpIfAboveThreshold := m.allocateInstr()
2912
+ jmpIfAboveThreshold.asJmpIf(condNB, newOperandLabel(ifAboveThreshold))
2913
+ m.insert(jmpIfAboveThreshold)
2914
+
2915
+ ifNotNaNTarget, ifNotNaN := m.allocateBrTarget()
2916
+ jmpIfNotNaN := m.allocateInstr()
2917
+ jmpIfNotNaN.asJmpIf(condNP, newOperandLabel(ifNotNaN))
2918
+ m.insert(jmpIfNotNaN)
2919
+
2920
+ // If NaN, handle the error condition.
2921
+ if sat {
2922
+ // On NaN, saturating, we just return 0.
2923
+ zeros := m.allocateInstr().asZeros(tmpGp)
2924
+ m.insert(zeros)
2925
+
2926
+ jmpEnd := m.allocateInstr()
2927
+ jmpEnd.asJmp(newOperandLabel(done))
2928
+ m.insert(jmpEnd)
2929
+ } else {
2930
+ // On NaN, non-saturating, we trap.
2931
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
2932
+ }
2933
+
2934
+ // If not NaN, land here.
2935
+ m.insert(ifNotNaNTarget)
2936
+
2937
+ // Truncation happens here.
2938
+
2939
+ trunc := m.allocateInstr()
2940
+ trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
2941
+ m.insert(trunc)
2942
+
2943
+ // Check if the result is negative.
2944
+ cmpNeg := m.allocateInstr()
2945
+ cmpNeg.asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
2946
+ m.insert(cmpNeg)
2947
+
2948
+ // If non-neg, jump to end.
2949
+ jmpIfNonNeg := m.allocateInstr()
2950
+ jmpIfNonNeg.asJmpIf(condNL, newOperandLabel(done))
2951
+ m.insert(jmpIfNonNeg)
2952
+
2953
+ if sat {
2954
+ // If the input was "small" (< 2**(width -1)), the only way to get an integer
2955
+ // overflow is because the input was too small: saturate to the min value, i.e. 0.
2956
+ zeros := m.allocateInstr().asZeros(tmpGp)
2957
+ m.insert(zeros)
2958
+
2959
+ jmpEnd := m.allocateInstr()
2960
+ jmpEnd.asJmp(newOperandLabel(done))
2961
+ m.insert(jmpEnd)
2962
+ } else {
2963
+ // If not saturating, trap.
2964
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
2965
+ }
2966
+
2967
+ // If above the threshold, land here.
2968
+ m.insert(ifAboveThresholdTarget)
2969
+
2970
+ // tmpDiff := threshold - rn.
2971
+ copySrc := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), tmpXmm2)
2972
+ m.insert(copySrc)
2973
+
2974
+ sub := m.allocateInstr()
2975
+ sub.asXmmRmR(subOp, newOperandReg(tmpXmm), tmpXmm2) // must be -0x8000000000000000
2976
+ m.insert(sub)
2977
+
2978
+ trunc2 := m.allocateInstr()
2979
+ trunc2.asXmmToGpr(truncOp, tmpXmm2, tmpGp, dst64)
2980
+ m.insert(trunc2)
2981
+
2982
+ // Check if the result is negative.
2983
+ cmpNeg2 := m.allocateInstr().asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
2984
+ m.insert(cmpNeg2)
2985
+
2986
+ ifNextLargeTarget, ifNextLarge := m.allocateBrTarget()
2987
+ jmpIfNextLarge := m.allocateInstr()
2988
+ jmpIfNextLarge.asJmpIf(condNL, newOperandLabel(ifNextLarge))
2989
+ m.insert(jmpIfNextLarge)
2990
+
2991
+ if sat {
2992
+ // The input was "large" (>= maxInt), so the only way to get an integer
2993
+ // overflow is because the input was too large: saturate to the max value.
2994
+ var maxInt uint64
2995
+ if dst64 {
2996
+ maxInt = math.MaxUint64
2997
+ } else {
2998
+ maxInt = math.MaxUint32
2999
+ }
3000
+ m.lowerIconst(tmpGp, maxInt, dst64)
3001
+
3002
+ jmpToEnd := m.allocateInstr()
3003
+ jmpToEnd.asJmp(newOperandLabel(done))
3004
+ m.insert(jmpToEnd)
3005
+ } else {
3006
+ // If not saturating, trap.
3007
+ m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
3008
+ }
3009
+
3010
+ m.insert(ifNextLargeTarget)
3011
+
3012
+ var op operand
3013
+ if dst64 {
3014
+ m.lowerIconst(tmpGp2, 0x8000000000000000, true)
3015
+ op = newOperandReg(tmpGp2)
3016
+ } else {
3017
+ op = newOperandImm32(0x80000000)
3018
+ }
3019
+
3020
+ add := m.allocateInstr()
3021
+ add.asAluRmiR(aluRmiROpcodeAdd, op, tmpGp, dst64)
3022
+ m.insert(add)
3023
+
3024
+ m.insert(doneTarget)
3025
+ }
3026
+
3027
+ func (m *machine) lowerFcvtFromSint(rn, rd operand, src64, dst64 bool) {
3028
+ var op sseOpcode
3029
+ if dst64 {
3030
+ op = sseOpcodeCvtsi2sd
3031
+ } else {
3032
+ op = sseOpcodeCvtsi2ss
3033
+ }
3034
+
3035
+ trunc := m.allocateInstr()
3036
+ trunc.asGprToXmm(op, rn, rd.reg(), src64)
3037
+ m.insert(trunc)
3038
+ }
3039
+
3040
+ func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) {
3041
+ var op sseOpcode
3042
+ if dst64 {
3043
+ op = sseOpcodeCvtsi2sd
3044
+ } else {
3045
+ op = sseOpcodeCvtsi2ss
3046
+ }
3047
+
3048
+ // Src is 32 bit, then we just perform the conversion with 64 bit width.
3049
+ //
3050
+ // See the following link for why we use 64bit conversion for unsigned 32bit integer sources:
3051
+ // https://stackoverflow.com/questions/41495498/fpu-operations-generated-by-gcc-during-casting-integer-to-float.
3052
+ //
3053
+ // Here's the summary:
3054
+ // >> CVTSI2SS is indeed designed for converting a signed integer to a scalar single-precision float,
3055
+ // >> not an unsigned integer like you have here. So what gives? Well, a 64-bit processor has 64-bit wide
3056
+ // >> registers available, so the unsigned 32-bit input values can be stored as signed 64-bit intermediate values,
3057
+ // >> which allows CVTSI2SS to be used after all.
3058
+ //
3059
+ if !src64 {
3060
+ // Before we convert, we have to clear the higher 32-bits of the 64-bit register
3061
+ // to get the correct result.
3062
+ tmp := m.c.AllocateVReg(ssa.TypeI32)
3063
+ m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, tmp))
3064
+ m.insert(m.allocateInstr().asGprToXmm(op, newOperandReg(tmp), rd.reg(), true))
3065
+ return
3066
+ }
3067
+
3068
+ // If uint64, we have to do a bit more work.
3069
+ endTarget, end := m.allocateBrTarget()
3070
+
3071
+ var tmpXmm regalloc.VReg
3072
+ if dst64 {
3073
+ tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
3074
+ } else {
3075
+ tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
3076
+ }
3077
+
3078
+ // Check if the most significant bit (sign bit) is set.
3079
+ test := m.allocateInstr()
3080
+ test.asCmpRmiR(false, rn, rn.reg(), src64)
3081
+ m.insert(test)
3082
+
3083
+ // Jump if the sign bit is set.
3084
+ ifSignTarget, ifSign := m.allocateBrTarget()
3085
+ jmpIfNeg := m.allocateInstr()
3086
+ jmpIfNeg.asJmpIf(condS, newOperandLabel(ifSign))
3087
+ m.insert(jmpIfNeg)
3088
+
3089
+ // If the sign bit is not set, we could fit the unsigned int into float32/float64.
3090
+ // So, we convert it to float and emit jump instruction to exit from this branch.
3091
+ cvt := m.allocateInstr()
3092
+ cvt.asGprToXmm(op, rn, tmpXmm, src64)
3093
+ m.insert(cvt)
3094
+
3095
+ // We are done, jump to end.
3096
+ jmpEnd := m.allocateInstr()
3097
+ jmpEnd.asJmp(newOperandLabel(end))
3098
+ m.insert(jmpEnd)
3099
+
3100
+ // Now handling the case where sign-bit is set.
3101
+ // We emit the following sequences:
3102
+ // mov %rn, %tmp
3103
+ // shr 1, %tmp
3104
+ // mov %rn, %tmp2
3105
+ // and 1, %tmp2
3106
+ // or %tmp2, %tmp
3107
+ // cvtsi2ss %tmp, %xmm0
3108
+ // addsd %xmm0, %xmm0
3109
+ m.insert(ifSignTarget)
3110
+
3111
+ tmp := m.copyToTmp(rn.reg())
3112
+ shr := m.allocateInstr()
3113
+ shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(1), tmp, src64)
3114
+ m.insert(shr)
3115
+
3116
+ tmp2 := m.copyToTmp(rn.reg())
3117
+ and := m.allocateInstr()
3118
+ and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, src64)
3119
+ m.insert(and)
3120
+
3121
+ or := m.allocateInstr()
3122
+ or.asAluRmiR(aluRmiROpcodeOr, newOperandReg(tmp2), tmp, src64)
3123
+ m.insert(or)
3124
+
3125
+ cvt2 := m.allocateInstr()
3126
+ cvt2.asGprToXmm(op, newOperandReg(tmp), tmpXmm, src64)
3127
+ m.insert(cvt2)
3128
+
3129
+ addsd := m.allocateInstr()
3130
+ if dst64 {
3131
+ addsd.asXmmRmR(sseOpcodeAddsd, newOperandReg(tmpXmm), tmpXmm)
3132
+ } else {
3133
+ addsd.asXmmRmR(sseOpcodeAddss, newOperandReg(tmpXmm), tmpXmm)
3134
+ }
3135
+ m.insert(addsd)
3136
+
3137
+ m.insert(endTarget)
3138
+ m.copyTo(tmpXmm, rd.reg())
3139
+ }
3140
+
3141
+ func (m *machine) lowerVanyTrue(instr *ssa.Instruction) {
3142
+ x := instr.Arg()
3143
+ rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3144
+ rd := m.c.VRegOf(instr.Return())
3145
+
3146
+ tmp := m.c.AllocateVReg(ssa.TypeI32)
3147
+
3148
+ cmp := m.allocateInstr()
3149
+ cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.reg())
3150
+ m.insert(cmp)
3151
+
3152
+ setcc := m.allocateInstr()
3153
+ setcc.asSetcc(condNZ, tmp)
3154
+ m.insert(setcc)
3155
+
3156
+ // Clear the irrelevant bits.
3157
+ and := m.allocateInstr()
3158
+ and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false)
3159
+ m.insert(and)
3160
+
3161
+ m.copyTo(tmp, rd)
3162
+ }
3163
+
3164
+ func (m *machine) lowerVallTrue(instr *ssa.Instruction) {
3165
+ x, lane := instr.ArgWithLane()
3166
+ var op sseOpcode
3167
+ switch lane {
3168
+ case ssa.VecLaneI8x16:
3169
+ op = sseOpcodePcmpeqb
3170
+ case ssa.VecLaneI16x8:
3171
+ op = sseOpcodePcmpeqw
3172
+ case ssa.VecLaneI32x4:
3173
+ op = sseOpcodePcmpeqd
3174
+ case ssa.VecLaneI64x2:
3175
+ op = sseOpcodePcmpeqq
3176
+ }
3177
+ rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3178
+ rd := m.c.VRegOf(instr.Return())
3179
+
3180
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3181
+
3182
+ zeros := m.allocateInstr()
3183
+ zeros.asZeros(tmp)
3184
+ m.insert(zeros)
3185
+
3186
+ pcmp := m.allocateInstr()
3187
+ pcmp.asXmmRmR(op, rm, tmp)
3188
+ m.insert(pcmp)
3189
+
3190
+ test := m.allocateInstr()
3191
+ test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp)
3192
+ m.insert(test)
3193
+
3194
+ tmp2 := m.c.AllocateVReg(ssa.TypeI32)
3195
+
3196
+ setcc := m.allocateInstr()
3197
+ setcc.asSetcc(condZ, tmp2)
3198
+ m.insert(setcc)
3199
+
3200
+ // Clear the irrelevant bits.
3201
+ and := m.allocateInstr()
3202
+ and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false)
3203
+ m.insert(and)
3204
+
3205
+ m.copyTo(tmp2, rd)
3206
+ }
3207
+
3208
+ func (m *machine) lowerVhighBits(instr *ssa.Instruction) {
3209
+ x, lane := instr.ArgWithLane()
3210
+ rm := m.getOperand_Reg(m.c.ValueDefinition(x))
3211
+ rd := m.c.VRegOf(instr.Return())
3212
+ switch lane {
3213
+ case ssa.VecLaneI8x16:
3214
+ mov := m.allocateInstr()
3215
+ mov.asXmmToGpr(sseOpcodePmovmskb, rm.reg(), rd, false)
3216
+ m.insert(mov)
3217
+
3218
+ case ssa.VecLaneI16x8:
3219
+ // When we have:
3220
+ // R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)]
3221
+ // R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)]
3222
+ // where RX(wn) is n-th signed word (16-bit) of RX register,
3223
+ //
3224
+ // "PACKSSWB R1, R2" produces
3225
+ // R1 = [
3226
+ // byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)),
3227
+ // byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)),
3228
+ // byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)),
3229
+ // byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)),
3230
+ // ]
3231
+ // where R1 is the destination register, and
3232
+ // byte_sat(w) = int8(w) if w fits as signed 8-bit,
3233
+ // 0x80 if w is less than 0x80
3234
+ // 0x7F if w is greater than 0x7f
3235
+ //
3236
+ // See https://www.felixcloutier.com/x86/packsswb:packssdw for detail.
3237
+ //
3238
+ // Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8).
3239
+ tmp := m.copyToTmp(rm.reg())
3240
+ res := m.c.AllocateVReg(ssa.TypeI32)
3241
+
3242
+ pak := m.allocateInstr()
3243
+ pak.asXmmRmR(sseOpcodePacksswb, rm, tmp)
3244
+ m.insert(pak)
3245
+
3246
+ mov := m.allocateInstr()
3247
+ mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false)
3248
+ m.insert(mov)
3249
+
3250
+ // Clear the higher bits than 8.
3251
+ shr := m.allocateInstr()
3252
+ shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false)
3253
+ m.insert(shr)
3254
+
3255
+ m.copyTo(res, rd)
3256
+
3257
+ case ssa.VecLaneI32x4:
3258
+ mov := m.allocateInstr()
3259
+ mov.asXmmToGpr(sseOpcodeMovmskps, rm.reg(), rd, true)
3260
+ m.insert(mov)
3261
+
3262
+ case ssa.VecLaneI64x2:
3263
+ mov := m.allocateInstr()
3264
+ mov.asXmmToGpr(sseOpcodeMovmskpd, rm.reg(), rd, true)
3265
+ m.insert(mov)
3266
+ }
3267
+ }
3268
+
3269
+ func (m *machine) lowerVbnot(instr *ssa.Instruction) {
3270
+ x := instr.Arg()
3271
+ xDef := m.c.ValueDefinition(x)
3272
+ rm := m.getOperand_Reg(xDef)
3273
+ rd := m.c.VRegOf(instr.Return())
3274
+
3275
+ tmp := m.copyToTmp(rm.reg())
3276
+ tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3277
+
3278
+ // Ensure tmp2 is considered defined by regalloc.
3279
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3280
+
3281
+ // Set all bits on tmp register.
3282
+ pak := m.allocateInstr()
3283
+ pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2)
3284
+ m.insert(pak)
3285
+
3286
+ // Then XOR with tmp to reverse all bits on v.register.
3287
+ xor := m.allocateInstr()
3288
+ xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)
3289
+ m.insert(xor)
3290
+
3291
+ m.copyTo(tmp, rd)
3292
+ }
3293
+
3294
+ func (m *machine) lowerSplat(x, ret ssa.Value, lane ssa.VecLane) {
3295
+ tmpDst := m.c.AllocateVReg(ssa.TypeV128)
3296
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst))
3297
+
3298
+ switch lane {
3299
+ case ssa.VecLaneI8x16:
3300
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3301
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmp))
3302
+ xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3303
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, xx, tmpDst))
3304
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp))
3305
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpDst))
3306
+ case ssa.VecLaneI16x8:
3307
+ xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3308
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, xx, tmpDst))
3309
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, xx, tmpDst))
3310
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3311
+ case ssa.VecLaneI32x4:
3312
+ xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3313
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, xx, tmpDst))
3314
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3315
+ case ssa.VecLaneI64x2:
3316
+ xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3317
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, xx, tmpDst))
3318
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, xx, tmpDst))
3319
+ case ssa.VecLaneF32x4:
3320
+ xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3321
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, 0, xx, tmpDst))
3322
+ m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
3323
+ case ssa.VecLaneF64x2:
3324
+ xx := m.getOperand_Reg(m.c.ValueDefinition(x))
3325
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst))
3326
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, xx, tmpDst))
3327
+ default:
3328
+ panic(fmt.Sprintf("invalid lane type: %s", lane))
3329
+ }
3330
+
3331
+ m.copyTo(tmpDst, m.c.VRegOf(ret))
3332
+ }
3333
+
3334
+ func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) {
3335
+ var xMask, yMask [2]uint64
3336
+ for i := 0; i < 8; i++ {
3337
+ loLane := byte(lo >> (i * 8))
3338
+ if loLane < 16 {
3339
+ xMask[0] |= uint64(loLane) << (i * 8)
3340
+ yMask[0] |= uint64(0x80) << (i * 8)
3341
+ } else {
3342
+ xMask[0] |= uint64(0x80) << (i * 8)
3343
+ yMask[0] |= uint64(loLane-16) << (i * 8)
3344
+ }
3345
+ hiLane := byte(hi >> (i * 8))
3346
+ if hiLane < 16 {
3347
+ xMask[1] |= uint64(hiLane) << (i * 8)
3348
+ yMask[1] |= uint64(0x80) << (i * 8)
3349
+ } else {
3350
+ xMask[1] |= uint64(0x80) << (i * 8)
3351
+ yMask[1] |= uint64(hiLane-16) << (i * 8)
3352
+ }
3353
+ }
3354
+
3355
+ xl, xmaskPos := m.allocateLabel()
3356
+ m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xl, labelPos: xmaskPos})
3357
+ yl, ymaskPos := m.allocateLabel()
3358
+ m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: yl, labelPos: ymaskPos})
3359
+
3360
+ xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
3361
+ tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg())
3362
+
3363
+ // Apply mask to X.
3364
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3365
+ loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xl)), tmp)
3366
+ m.insert(loadMaskLo)
3367
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX))
3368
+
3369
+ // Apply mask to Y.
3370
+ loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(yl)), tmp)
3371
+ m.insert(loadMaskHi)
3372
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY))
3373
+
3374
+ // Combine the results.
3375
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodeOrps, newOperandReg(tmpX), tmpY))
3376
+
3377
+ m.copyTo(tmpY, m.c.VRegOf(ret))
3378
+ }
3379
+
3380
+ func (m *machine) lowerVbBinOpUnaligned(op sseOpcode, x, y, ret ssa.Value) {
3381
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3382
+ rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3383
+ rd := m.c.VRegOf(ret)
3384
+
3385
+ tmp := m.copyToTmp(rn.reg())
3386
+
3387
+ binOp := m.allocateInstr()
3388
+ binOp.asXmmRmR(op, rm, tmp)
3389
+ m.insert(binOp)
3390
+
3391
+ m.copyTo(tmp, rd)
3392
+ }
3393
+
3394
+ func (m *machine) lowerVbBinOp(op sseOpcode, x, y, ret ssa.Value) {
3395
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3396
+ rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3397
+ rd := m.c.VRegOf(ret)
3398
+
3399
+ tmp := m.copyToTmp(rn.reg())
3400
+
3401
+ binOp := m.allocateInstr()
3402
+ binOp.asXmmRmR(op, rm, tmp)
3403
+ m.insert(binOp)
3404
+
3405
+ m.copyTo(tmp, rd)
3406
+ }
3407
+
3408
+ func (m *machine) lowerVFcmp(x, y ssa.Value, c ssa.FloatCmpCond, ret ssa.Value, lane ssa.VecLane) {
3409
+ var cmpOp sseOpcode
3410
+ switch lane {
3411
+ case ssa.VecLaneF32x4:
3412
+ cmpOp = sseOpcodeCmpps
3413
+ case ssa.VecLaneF64x2:
3414
+ cmpOp = sseOpcodeCmppd
3415
+ default:
3416
+ panic(fmt.Sprintf("invalid lane type: %s", lane))
3417
+ }
3418
+
3419
+ xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
3420
+ var cmpImm cmpPred
3421
+ switch c {
3422
+ case ssa.FloatCmpCondGreaterThan:
3423
+ yy, xx = xx, yy
3424
+ cmpImm = cmpPredLT_OS
3425
+ case ssa.FloatCmpCondGreaterThanOrEqual:
3426
+ yy, xx = xx, yy
3427
+ cmpImm = cmpPredLE_OS
3428
+ case ssa.FloatCmpCondEqual:
3429
+ cmpImm = cmpPredEQ_OQ
3430
+ case ssa.FloatCmpCondNotEqual:
3431
+ cmpImm = cmpPredNEQ_UQ
3432
+ case ssa.FloatCmpCondLessThan:
3433
+ cmpImm = cmpPredLT_OS
3434
+ case ssa.FloatCmpCondLessThanOrEqual:
3435
+ cmpImm = cmpPredLE_OS
3436
+ default:
3437
+ panic(fmt.Sprintf("invalid float comparison condition: %s", c))
3438
+ }
3439
+
3440
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3441
+ xxx := m.getOperand_Mem_Reg(xx)
3442
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, xxx, tmp))
3443
+
3444
+ rm := m.getOperand_Mem_Reg(yy)
3445
+ m.insert(m.allocateInstr().asXmmRmRImm(cmpOp, byte(cmpImm), rm, tmp))
3446
+
3447
+ m.copyTo(tmp, m.c.VRegOf(ret))
3448
+ }
3449
+
3450
+ func (m *machine) lowerVIcmp(x, y ssa.Value, c ssa.IntegerCmpCond, ret ssa.Value, lane ssa.VecLane) {
3451
+ var eq, gt, maxu, minu, mins sseOpcode
3452
+ switch lane {
3453
+ case ssa.VecLaneI8x16:
3454
+ eq, gt, maxu, minu, mins = sseOpcodePcmpeqb, sseOpcodePcmpgtb, sseOpcodePmaxub, sseOpcodePminub, sseOpcodePminsb
3455
+ case ssa.VecLaneI16x8:
3456
+ eq, gt, maxu, minu, mins = sseOpcodePcmpeqw, sseOpcodePcmpgtw, sseOpcodePmaxuw, sseOpcodePminuw, sseOpcodePminsw
3457
+ case ssa.VecLaneI32x4:
3458
+ eq, gt, maxu, minu, mins = sseOpcodePcmpeqd, sseOpcodePcmpgtd, sseOpcodePmaxud, sseOpcodePminud, sseOpcodePminsd
3459
+ case ssa.VecLaneI64x2:
3460
+ eq, gt = sseOpcodePcmpeqq, sseOpcodePcmpgtq
3461
+ default:
3462
+ panic(fmt.Sprintf("invalid lane type: %s", lane))
3463
+ }
3464
+
3465
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3466
+ var op operand
3467
+ switch c {
3468
+ case ssa.IntegerCmpCondSignedLessThanOrEqual:
3469
+ if lane == ssa.VecLaneI64x2 {
3470
+ x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3471
+ // Copy x to tmp.
3472
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3473
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3474
+ } else {
3475
+ y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3476
+ // Copy y to tmp.
3477
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3478
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3479
+ }
3480
+ case ssa.IntegerCmpCondSignedGreaterThanOrEqual:
3481
+ if lane == ssa.VecLaneI64x2 {
3482
+ y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3483
+ // Copy y to tmp.
3484
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3485
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3486
+ } else {
3487
+ x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3488
+ // Copy x to tmp.
3489
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3490
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3491
+ }
3492
+ case ssa.IntegerCmpCondSignedLessThan, ssa.IntegerCmpCondUnsignedLessThan, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
3493
+ y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3494
+ // Copy y to tmp.
3495
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
3496
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3497
+ default:
3498
+ x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3499
+ // Copy x to tmp.
3500
+ m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
3501
+ op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
3502
+ }
3503
+
3504
+ switch c {
3505
+ case ssa.IntegerCmpCondEqual:
3506
+ m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3507
+ case ssa.IntegerCmpCondNotEqual:
3508
+ // First we compare for equality.
3509
+ m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3510
+ // Then flip the bits. To do so, we set all bits on tmp2.
3511
+ tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3512
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3513
+ m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3514
+ // And then xor with tmp.
3515
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3516
+ case ssa.IntegerCmpCondSignedGreaterThan, ssa.IntegerCmpCondSignedLessThan:
3517
+ m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
3518
+ case ssa.IntegerCmpCondSignedGreaterThanOrEqual, ssa.IntegerCmpCondSignedLessThanOrEqual:
3519
+ if lane == ssa.VecLaneI64x2 {
3520
+ m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
3521
+ // Then flip the bits. To do so, we set all bits on tmp2.
3522
+ tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3523
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3524
+ m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3525
+ // And then xor with tmp.
3526
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3527
+ } else {
3528
+ // First take min of x and y.
3529
+ m.insert(m.allocateInstr().asXmmRmR(mins, op, tmp))
3530
+ // Then compare for equality.
3531
+ m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3532
+ }
3533
+ case ssa.IntegerCmpCondUnsignedGreaterThan, ssa.IntegerCmpCondUnsignedLessThan:
3534
+ // First maxu of x and y.
3535
+ m.insert(m.allocateInstr().asXmmRmR(maxu, op, tmp))
3536
+ // Then compare for equality.
3537
+ m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3538
+ // Then flip the bits. To do so, we set all bits on tmp2.
3539
+ tmp2 := m.c.AllocateVReg(ssa.TypeV128)
3540
+ m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
3541
+ m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
3542
+ // And then xor with tmp.
3543
+ m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
3544
+ case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
3545
+ m.insert(m.allocateInstr().asXmmRmR(minu, op, tmp))
3546
+ m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
3547
+ default:
3548
+ panic("BUG")
3549
+ }
3550
+
3551
+ m.copyTo(tmp, m.c.VRegOf(ret))
3552
+ }
3553
+
3554
+ func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) {
3555
+ x, y := instr.Arg2()
3556
+ xDef := m.c.ValueDefinition(x)
3557
+ yDef := m.c.ValueDefinition(y)
3558
+ rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
3559
+ rd := m.c.VRegOf(instr.Return())
3560
+
3561
+ tmp := m.copyToTmp(rn.reg())
3562
+
3563
+ // pandn between rn, rm.
3564
+ pand := m.allocateInstr()
3565
+ pand.asXmmRmR(sseOpcodePandn, rm, tmp)
3566
+ m.insert(pand)
3567
+
3568
+ m.copyTo(tmp, rd)
3569
+ }
3570
+
3571
+ func (m *machine) lowerVbitselect(instr *ssa.Instruction) {
3572
+ c, x, y := instr.SelectData()
3573
+ xDef := m.c.ValueDefinition(x)
3574
+ yDef := m.c.ValueDefinition(y)
3575
+ rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
3576
+ creg := m.getOperand_Reg(m.c.ValueDefinition(c))
3577
+ rd := m.c.VRegOf(instr.Return())
3578
+
3579
+ tmpC := m.copyToTmp(creg.reg())
3580
+ tmpX := m.copyToTmp(rm.reg())
3581
+
3582
+ // And between c, x (overwrites x).
3583
+ pand := m.allocateInstr()
3584
+ pand.asXmmRmR(sseOpcodePand, creg, tmpX)
3585
+ m.insert(pand)
3586
+
3587
+ // Andn between y, c (overwrites c).
3588
+ pandn := m.allocateInstr()
3589
+ pandn.asXmmRmR(sseOpcodePandn, rn, tmpC)
3590
+ m.insert(pandn)
3591
+
3592
+ por := m.allocateInstr()
3593
+ por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX)
3594
+ m.insert(por)
3595
+
3596
+ m.copyTo(tmpX, rd)
3597
+ }
3598
+
3599
+ func (m *machine) lowerVFmin(instr *ssa.Instruction) {
3600
+ x, y, lane := instr.Arg2WithLane()
3601
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3602
+ rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3603
+ rd := m.c.VRegOf(instr.Return())
3604
+
3605
+ var min, cmp, andn, or, srl /* shift right logical */ sseOpcode
3606
+ var shiftNumToInverseNaN uint32
3607
+ if lane == ssa.VecLaneF32x4 {
3608
+ min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodePsrld, 0xa
3609
+ } else {
3610
+ min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodePsrlq, 0xd
3611
+ }
3612
+
3613
+ tmp1 := m.copyToTmp(rn.reg())
3614
+ tmp2 := m.copyToTmp(rm.reg())
3615
+
3616
+ // tmp1=min(rn, rm)
3617
+ minIns1 := m.allocateInstr()
3618
+ minIns1.asXmmRmR(min, rn, tmp2)
3619
+ m.insert(minIns1)
3620
+
3621
+ // tmp2=min(rm, rn)
3622
+ minIns2 := m.allocateInstr()
3623
+ minIns2.asXmmRmR(min, rm, tmp1)
3624
+ m.insert(minIns2)
3625
+
3626
+ // tmp3:=tmp1=min(rn, rm)
3627
+ tmp3 := m.copyToTmp(tmp1)
3628
+
3629
+ // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3630
+ // NaN if rn == NaN || rm == NaN
3631
+ // min(rm, rm) otherwise
3632
+ orIns := m.allocateInstr()
3633
+ orIns.asXmmRmR(or, newOperandReg(tmp2), tmp1)
3634
+ m.insert(orIns)
3635
+
3636
+ // tmp3 is originally min(rn,rm).
3637
+ // tmp3 = 0^ (set all bits) if rn == NaN || rm == NaN
3638
+ // 0 otherwise
3639
+ cmpIns := m.allocateInstr()
3640
+ cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp2), tmp3)
3641
+ m.insert(cmpIns)
3642
+
3643
+ // tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3644
+ // ^0 if rn == NaN || rm == NaN
3645
+ // min(v1, v2) otherwise
3646
+ orIns2 := m.allocateInstr()
3647
+ orIns2.asXmmRmR(or, newOperandReg(tmp3), tmp1)
3648
+ m.insert(orIns2)
3649
+
3650
+ // tmp3 = set all bits on the mantissa bits
3651
+ // 0 otherwise
3652
+ shift := m.allocateInstr()
3653
+ shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp3)
3654
+ m.insert(shift)
3655
+
3656
+ // tmp3 = tmp1 and !tmp3
3657
+ // = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
3658
+ // set all bits on exponential and sign bit (== NaN) if rn == NaN || rm == NaN
3659
+ // min(rn, rm) otherwise
3660
+ andnIns := m.allocateInstr()
3661
+ andnIns.asXmmRmR(andn, newOperandReg(tmp1), tmp3)
3662
+ m.insert(andnIns)
3663
+
3664
+ m.copyTo(tmp3, rd)
3665
+ }
3666
+
3667
+ func (m *machine) lowerVFmax(instr *ssa.Instruction) {
3668
+ x, y, lane := instr.Arg2WithLane()
3669
+ rn := m.getOperand_Reg(m.c.ValueDefinition(x))
3670
+ rm := m.getOperand_Reg(m.c.ValueDefinition(y))
3671
+ rd := m.c.VRegOf(instr.Return())
3672
+
3673
+ var max, cmp, andn, or, xor, sub, srl /* shift right logical */ sseOpcode
3674
+ var shiftNumToInverseNaN uint32
3675
+ if lane == ssa.VecLaneF32x4 {
3676
+ max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodeXorps, sseOpcodeSubps, sseOpcodePsrld, 0xa
3677
+ } else {
3678
+ max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodeXorpd, sseOpcodeSubpd, sseOpcodePsrlq, 0xd
3679
+ }
3680
+
3681
+ tmp0 := m.copyToTmp(rm.reg())
3682
+ tmp1 := m.copyToTmp(rn.reg())
3683
+
3684
+ // tmp0=max(rn, rm)
3685
+ maxIns1 := m.allocateInstr()
3686
+ maxIns1.asXmmRmR(max, rn, tmp0)
3687
+ m.insert(maxIns1)
3688
+
3689
+ // tmp1=max(rm, rn)
3690
+ maxIns2 := m.allocateInstr()
3691
+ maxIns2.asXmmRmR(max, rm, tmp1)
3692
+ m.insert(maxIns2)
3693
+
3694
+ // tmp2=max(rm, rn)
3695
+ tmp2 := m.copyToTmp(tmp1)
3696
+
3697
+ // tmp2 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
3698
+ // 0 if (rn == 0 && rm == 0)
3699
+ // -0 if (rn == -0 && rm == -0)
3700
+ // v1^v2 if rn == NaN || rm == NaN
3701
+ // 0 otherwise
3702
+ xorInstr := m.allocateInstr()
3703
+ xorInstr.asXmmRmR(xor, newOperandReg(tmp0), tmp2)
3704
+ m.insert(xorInstr)
3705
+ // tmp1 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
3706
+ // 0 if (rn == 0 && rm == 0)
3707
+ // -0 if (rn == -0 && rm == -0)
3708
+ // NaN if rn == NaN || rm == NaN
3709
+ // max(v1, v2) otherwise
3710
+ orInstr := m.allocateInstr()
3711
+ orInstr.asXmmRmR(or, newOperandReg(tmp2), tmp1)
3712
+ m.insert(orInstr)
3713
+
3714
+ tmp3 := m.copyToTmp(tmp1)
3715
+
3716
+ // tmp3 = 0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) || (rn == 0 && rm == 0)
3717
+ // -0 if (rn == -0 && rm == -0)
3718
+ // NaN if rn == NaN || rm == NaN
3719
+ // max(v1, v2) otherwise
3720
+ //
3721
+ // Note: -0 - (-0) = 0 (!= -0) in floating point operation.
3722
+ subIns := m.allocateInstr()
3723
+ subIns.asXmmRmR(sub, newOperandReg(tmp2), tmp3)
3724
+ m.insert(subIns)
3725
+
3726
+ // tmp1 = 0^ if rn == NaN || rm == NaN
3727
+ cmpIns := m.allocateInstr()
3728
+ cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp1), tmp1)
3729
+ m.insert(cmpIns)
3730
+
3731
+ // tmp1 = set all bits on the mantissa bits
3732
+ // 0 otherwise
3733
+ shift := m.allocateInstr()
3734
+ shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp1)
3735
+ m.insert(shift)
3736
+
3737
+ andnIns := m.allocateInstr()
3738
+ andnIns.asXmmRmR(andn, newOperandReg(tmp3), tmp1)
3739
+ m.insert(andnIns)
3740
+
3741
+ m.copyTo(tmp1, rd)
3742
+ }
3743
+
3744
+ func (m *machine) lowerVFabs(instr *ssa.Instruction) {
3745
+ x, lane := instr.ArgWithLane()
3746
+ rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
3747
+ rd := m.c.VRegOf(instr.Return())
3748
+
3749
+ tmp := m.c.AllocateVReg(ssa.TypeV128)
3750
+
3751
+ def := m.allocateInstr()
3752
+ def.asDefineUninitializedReg(tmp)
3753
+ m.insert(def)
3754
+
3755
+ // Set all bits on tmp.
3756
+ pcmp := m.allocateInstr()
3757
+ pcmp.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp)
3758
+ m.insert(pcmp)
3759
+
3760
+ switch lane {
3761
+ case ssa.VecLaneF32x4:
3762
+ // Shift right packed single floats by 1 to clear the sign bits.
3763
+ shift := m.allocateInstr()
3764
+ shift.asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp)
3765
+ m.insert(shift)
3766
+ // Clear the sign bit of rm.
3767
+ andp := m.allocateInstr()
3768
+ andp.asXmmRmR(sseOpcodeAndpd, rm, tmp)
3769
+ m.insert(andp)
3770
+ case ssa.VecLaneF64x2:
3771
+ // Shift right packed single floats by 1 to clear the sign bits.
3772
+ shift := m.allocateInstr()
3773
+ shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(1), tmp)
3774
+ m.insert(shift)
3775
+ // Clear the sign bit of rm.
3776
+ andp := m.allocateInstr()
3777
+ andp.asXmmRmR(sseOpcodeAndps, rm, tmp)
3778
+ m.insert(andp)
3779
+ }
3780
+
3781
+ m.copyTo(tmp, rd)
3782
+ }