typenative 0.6.2 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/bin/cli.js +18 -8
- package/bin/go/overrides.js +21 -1
- package/bin/index.js +48 -25
- package/bin/main.js +2 -1
- package/bin/node/assert.js +56 -17
- package/bin/node/async_hooks.js +69 -10
- package/bin/node/buffer.js +321 -7
- package/bin/node/child_process.js +220 -16
- package/bin/node/cluster.js +436 -32
- package/bin/node/crypto.js +814 -5
- package/bin/node/crypto_curves.js +142 -0
- package/bin/node/crypto_keys.js +1147 -48
- package/bin/node/crypto_pem.js +313 -0
- package/bin/node/crypto_pkcs12.js +17 -0
- package/bin/node/dgram.js +161 -10
- package/bin/node/diagnostics_channel.js +5 -1
- package/bin/node/events.js +358 -8
- package/bin/node/ffi.js +933 -0
- package/bin/node/fs.js +1346 -10
- package/bin/node/histogram.js +1991 -0
- package/bin/node/http.js +630 -55
- package/bin/node/http2.js +1141 -88
- package/bin/node/intl.js +4576 -0
- package/bin/node/intl_data.js +65412 -0
- package/bin/node/module_v8.js +155 -16
- package/bin/node/net.js +59 -10
- package/bin/node/path.js +5 -1
- package/bin/node/process.js +355 -4
- package/bin/node/readline.js +558 -8
- package/bin/node/registry.js +2 -0
- package/bin/node/sqlite.js +1539 -7
- package/bin/node/stream.js +371 -32
- package/bin/node/stream_web.js +422 -15
- package/bin/node/system_extra.js +40 -154
- package/bin/node/temporal.js +96 -0
- package/bin/node/temporal_runtime.js +5075 -0
- package/bin/node/temporal_zones.js +460 -0
- package/bin/node/test.js +191 -8
- package/bin/node/test_coverage.js +58 -0
- package/bin/node/test_report.js +50 -4
- package/bin/node/test_run.js +64 -3
- package/bin/node/timers.js +123 -0
- package/bin/node/tls.js +346 -12
- package/bin/node/tls_roots.js +122 -0
- package/bin/node/typed_arrays.js +867 -93
- package/bin/node/url.js +657 -0
- package/bin/node/util.js +449 -7
- package/bin/node/util_extra.js +6 -0
- package/bin/node/wasi.js +214 -0
- package/bin/node/webassembly.js +1137 -0
- package/bin/node/websocket.js +763 -0
- package/bin/node/webview.js +3257 -244
- package/bin/node/worker_threads.js +434 -52
- package/bin/node/zlib.js +455 -22
- package/bin/node/zlib_iter.js +585 -0
- package/bin/node/zlib_zip.js +2392 -0
- package/bin/transpiler.js +3417 -501
- package/bin/typecheck-node.js +49 -17
- package/bin/version.js +1 -1
- package/govendor/github.com/cloudflare/circl/.golangci.yaml +64 -0
- package/govendor/github.com/cloudflare/circl/.semgrepignore +1 -0
- package/govendor/github.com/cloudflare/circl/AGENTS.md +91 -0
- package/govendor/github.com/cloudflare/circl/CITATION.cff +30 -0
- package/govendor/github.com/cloudflare/circl/LICENSE +57 -0
- package/govendor/github.com/cloudflare/circl/Makefile +70 -0
- package/govendor/github.com/cloudflare/circl/README.md +206 -0
- package/govendor/github.com/cloudflare/circl/REVIEW.md +83 -0
- package/govendor/github.com/cloudflare/circl/codecov.yml +21 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve.go +104 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.go +30 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.h +111 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve_amd64.s +194 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve_generic.go +100 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/curve_noasm.go +11 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/doc.go +19 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/key.go +46 -0
- package/govendor/github.com/cloudflare/circl/dh/x448/table.go +460 -0
- package/govendor/github.com/cloudflare/circl/doc.go +8 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/constants.go +71 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/curve.go +90 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/isogeny.go +52 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/point.go +181 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/scalar.go +203 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twist.go +138 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twistPoint.go +135 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twistTables.go +216 -0
- package/govendor/github.com/cloudflare/circl/ecc/goldilocks/twist_basemult.go +62 -0
- package/govendor/github.com/cloudflare/circl/go.mod +3 -0
- package/govendor/github.com/cloudflare/circl/internal/conv/conv.go +173 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/doc.go +62 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/hashes.go +69 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/keccakf.go +391 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/rc.go +29 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/sha3.go +200 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/sha3_s390x.s +33 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/shake.go +119 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/xor.go +15 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/xor_generic.go +33 -0
- package/govendor/github.com/cloudflare/circl/internal/sha3/xor_unaligned.go +61 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp.go +164 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.go +43 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.h +591 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp_amd64.s +75 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp_generic.go +339 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fp_noasm.go +12 -0
- package/govendor/github.com/cloudflare/circl/math/fp448/fuzzer.go +75 -0
- package/govendor/github.com/cloudflare/circl/math/integer.go +16 -0
- package/govendor/github.com/cloudflare/circl/math/mlsbset/mlsbset.go +145 -0
- package/govendor/github.com/cloudflare/circl/math/mlsbset/power.go +64 -0
- package/govendor/github.com/cloudflare/circl/math/primes.go +34 -0
- package/govendor/github.com/cloudflare/circl/math/wnaf.go +84 -0
- package/govendor/github.com/cloudflare/circl/sign/ed448/ed448.go +417 -0
- package/govendor/github.com/cloudflare/circl/sign/ed448/signapi.go +89 -0
- package/govendor/github.com/cloudflare/circl/sign/sign.go +119 -0
- package/{examples/react-compiler/vendor/picocolors → govendor/github.com/decred/dcrd/dcrec/secp256k1/v4}/LICENSE +4 -2
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/README.md +72 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/compressedbytepoints.go +18 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve.go +1310 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve_embedded.go +14 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/curve_precompute.go +14 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/doc.go +59 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdh.go +21 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/README.md +52 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/doc.go +42 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/error.go +134 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa/signature.go +1001 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/ellipticadaptor.go +296 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/error.go +67 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/field.go +1696 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/go.mod +3 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/loadprecomputed.go +91 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/modnscalar.go +1105 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/nonce.go +263 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/privkey.go +111 -0
- package/govendor/github.com/decred/dcrd/dcrec/secp256k1/v4/pubkey.go +236 -0
- package/govendor/github.com/ebitengine/purego/LICENSE +201 -0
- package/govendor/github.com/ebitengine/purego/README.md +124 -0
- package/govendor/github.com/ebitengine/purego/abi_amd64.h +99 -0
- package/govendor/github.com/ebitengine/purego/abi_arm64.h +39 -0
- package/govendor/github.com/ebitengine/purego/abi_loong64.h +60 -0
- package/govendor/github.com/ebitengine/purego/cdecl.go +11 -0
- package/govendor/github.com/ebitengine/purego/cgo.go +19 -0
- package/govendor/github.com/ebitengine/purego/dlerror.go +17 -0
- package/govendor/github.com/ebitengine/purego/dlfcn.go +99 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_android.go +34 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_darwin.go +20 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_freebsd.go +14 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_linux.go +16 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_netbsd.go +15 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_freebsd.go +11 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_linux.go +19 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_nocgo_netbsd.go +11 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_playground.go +24 -0
- package/govendor/github.com/ebitengine/purego/dlfcn_stubs.s +22 -0
- package/govendor/github.com/ebitengine/purego/func.go +780 -0
- package/govendor/github.com/ebitengine/purego/gen.go +6 -0
- package/govendor/github.com/ebitengine/purego/go.mod +3 -0
- package/govendor/github.com/ebitengine/purego/go_runtime.go +19 -0
- package/govendor/github.com/ebitengine/purego/internal/cgo/dlfcn_cgo_unix.go +56 -0
- package/govendor/github.com/ebitengine/purego/internal/cgo/empty.go +6 -0
- package/govendor/github.com/ebitengine/purego/internal/cgo/syscall_cgo_unix.go +49 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_amd64.h +99 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_arm64.h +39 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_loong64.h +60 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_ppc64x.h +195 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/abi_riscv64.h +72 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_386.s +29 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_amd64.s +39 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_arm.s +58 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_arm64.s +35 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_loong64.s +38 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_ppc64le.s +79 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_riscv64.s +36 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/asm_s390x.s +52 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/callbacks.go +93 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/doc.go +32 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/fakecgo.go +14 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/freebsd.go +27 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_darwin.go +86 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_freebsd.go +77 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_libinit.go +72 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_linux.go +77 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_netbsd.go +83 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_setenv.go +18 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/go_util.go +38 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/iscgo.go +19 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo.go +37 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_darwin.go +37 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_freebsd.go +27 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_linux.go +27 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/libcgo_netbsd.go +36 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/linux.go +189 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/netbsd.go +23 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/setenv.go +19 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/threadentry_amd64.go +15 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/threadentry_noasm.go +24 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_386.s +130 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_amd64.s +124 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_arm.s +131 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_arm64.s +81 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_386.s +78 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_amd64.s +69 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_arm.s +69 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_arm64.s +60 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_loong64.s +60 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_ppc64le.s +69 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_riscv64.s +60 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_linux_s390x.s +51 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_loong64.s +78 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_ppc64le.s +132 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_riscv64.s +76 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/trampolines_s390x.s +158 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols.go +165 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_darwin.go +59 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_freebsd.go +48 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_linux.go +158 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/zsymbols_netbsd.go +59 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_darwin.s +19 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_freebsd.s +16 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_linux.s +46 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_netbsd.s +19 -0
- package/govendor/github.com/ebitengine/purego/internal/fakecgo/ztrampolines_stubs.s +55 -0
- package/govendor/github.com/ebitengine/purego/internal/strings/strings.go +40 -0
- package/govendor/github.com/ebitengine/purego/is_ios.go +13 -0
- package/govendor/github.com/ebitengine/purego/nocgo.go +25 -0
- package/govendor/github.com/ebitengine/purego/struct_amd64.go +518 -0
- package/govendor/github.com/ebitengine/purego/struct_arm.go +100 -0
- package/govendor/github.com/ebitengine/purego/struct_arm64.go +757 -0
- package/govendor/github.com/ebitengine/purego/struct_loong64.go +244 -0
- package/govendor/github.com/ebitengine/purego/struct_other.go +54 -0
- package/govendor/github.com/ebitengine/purego/struct_ppc64le.go +180 -0
- package/govendor/github.com/ebitengine/purego/struct_riscv64.go +158 -0
- package/govendor/github.com/ebitengine/purego/struct_s390x.go +158 -0
- package/govendor/github.com/ebitengine/purego/sys_386.s +153 -0
- package/govendor/github.com/ebitengine/purego/sys_amd64.s +206 -0
- package/govendor/github.com/ebitengine/purego/sys_arm.s +153 -0
- package/govendor/github.com/ebitengine/purego/sys_arm64.s +133 -0
- package/govendor/github.com/ebitengine/purego/sys_loong64.s +131 -0
- package/govendor/github.com/ebitengine/purego/sys_ppc64le.s +126 -0
- package/govendor/github.com/ebitengine/purego/sys_riscv64.s +135 -0
- package/govendor/github.com/ebitengine/purego/sys_s390x.s +148 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_386.s +234 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_arm.s +99 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_arm64.s +75 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_loong64.s +79 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_ppc64le.s +114 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_riscv64.s +79 -0
- package/govendor/github.com/ebitengine/purego/sys_unix_s390x.s +111 -0
- package/govendor/github.com/ebitengine/purego/syscall.go +95 -0
- package/govendor/github.com/ebitengine/purego/syscall_32bit.go +97 -0
- package/govendor/github.com/ebitengine/purego/syscall_cgo_linux.go +20 -0
- package/govendor/github.com/ebitengine/purego/syscall_notstackargs.go +55 -0
- package/govendor/github.com/ebitengine/purego/syscall_ppc64le.go +73 -0
- package/govendor/github.com/ebitengine/purego/syscall_stackargs_ppc64le.go +38 -0
- package/govendor/github.com/ebitengine/purego/syscall_stackargs_s390x.go +38 -0
- package/govendor/github.com/ebitengine/purego/syscall_unix.go +392 -0
- package/govendor/github.com/ebitengine/purego/syscall_windows.go +54 -0
- package/govendor/github.com/ebitengine/purego/zcallback_386.s +4014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_amd64.s +2014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_arm.s +4014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_arm64.s +4014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_loong64.s +4014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_ppc64le.s +4014 -0
- package/govendor/github.com/ebitengine/purego/zcallback_riscv64.s +4051 -0
- package/govendor/github.com/ebitengine/purego/zcallback_s390x.s +4015 -0
- package/govendor/github.com/klauspost/compress/.gitattributes +3 -0
- package/govendor/github.com/klauspost/compress/.goreleaser.yml +132 -0
- package/govendor/github.com/klauspost/compress/AGENTS.md +106 -0
- package/govendor/github.com/klauspost/compress/LICENSE +304 -0
- package/govendor/github.com/klauspost/compress/README.md +766 -0
- package/govendor/github.com/klauspost/compress/SECURITY.md +25 -0
- package/govendor/github.com/klauspost/compress/compressible.go +85 -0
- package/govendor/github.com/klauspost/compress/fse/README.md +79 -0
- package/govendor/github.com/klauspost/compress/fse/bitreader.go +122 -0
- package/govendor/github.com/klauspost/compress/fse/bitwriter.go +167 -0
- package/govendor/github.com/klauspost/compress/fse/bytereader.go +47 -0
- package/govendor/github.com/klauspost/compress/fse/compress.go +685 -0
- package/govendor/github.com/klauspost/compress/fse/decompress.go +383 -0
- package/govendor/github.com/klauspost/compress/fse/fse.go +150 -0
- package/govendor/github.com/klauspost/compress/gen.sh +4 -0
- package/govendor/github.com/klauspost/compress/go.mod +3 -0
- package/govendor/github.com/klauspost/compress/huff0/README.md +89 -0
- package/govendor/github.com/klauspost/compress/huff0/bitreader.go +323 -0
- package/govendor/github.com/klauspost/compress/huff0/bitwriter.go +97 -0
- package/govendor/github.com/klauspost/compress/huff0/build_table.go +168 -0
- package/govendor/github.com/klauspost/compress/huff0/compress.go +747 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress.go +1188 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_amd64.go +128 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_amd64.s +3476 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_arm64.go +72 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_arm64.s +1648 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_asm.go +252 -0
- package/govendor/github.com/klauspost/compress/huff0/decompress_generic.go +278 -0
- package/govendor/github.com/klauspost/compress/huff0/huff0.go +337 -0
- package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo.go +34 -0
- package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.go +10 -0
- package/govendor/github.com/klauspost/compress/internal/cpuinfo/cpuinfo_amd64.s +34 -0
- package/govendor/github.com/klauspost/compress/internal/le/le.go +5 -0
- package/govendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go +42 -0
- package/govendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go +52 -0
- package/govendor/github.com/klauspost/compress/internal/regmask/regmask.go +50 -0
- package/govendor/github.com/klauspost/compress/internal/regmask/regmask_386.go +39 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/LICENSE +27 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/decode.go +264 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/decode_other.go +113 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/encode.go +291 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/encode_other.go +250 -0
- package/govendor/github.com/klauspost/compress/internal/snapref/snappy.go +98 -0
- package/govendor/github.com/klauspost/compress/s2sx.mod +3 -0
- package/govendor/github.com/klauspost/compress/s2sx.sum +0 -0
- package/govendor/github.com/klauspost/compress/zstd/README.md +487 -0
- package/govendor/github.com/klauspost/compress/zstd/bitreader.go +176 -0
- package/govendor/github.com/klauspost/compress/zstd/bitwriter.go +151 -0
- package/govendor/github.com/klauspost/compress/zstd/blockdec.go +728 -0
- package/govendor/github.com/klauspost/compress/zstd/blockenc.go +923 -0
- package/govendor/github.com/klauspost/compress/zstd/blocktype_string.go +85 -0
- package/govendor/github.com/klauspost/compress/zstd/bytebuf.go +131 -0
- package/govendor/github.com/klauspost/compress/zstd/bytereader.go +82 -0
- package/govendor/github.com/klauspost/compress/zstd/decodeheader.go +261 -0
- package/govendor/github.com/klauspost/compress/zstd/decoder.go +995 -0
- package/govendor/github.com/klauspost/compress/zstd/decoder_options.go +213 -0
- package/govendor/github.com/klauspost/compress/zstd/dict.go +605 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_base.go +200 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_best.go +572 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_better.go +1262 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_dfast.go +1136 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_fast.go +892 -0
- package/govendor/github.com/klauspost/compress/zstd/enc_jobs.go +352 -0
- package/govendor/github.com/klauspost/compress/zstd/encoder.go +881 -0
- package/govendor/github.com/klauspost/compress/zstd/encoder_options.go +422 -0
- package/govendor/github.com/klauspost/compress/zstd/framedec.go +404 -0
- package/govendor/github.com/klauspost/compress/zstd/frameenc.go +137 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_decoder.go +333 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s +132 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s +148 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go +70 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go +71 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_encoder.go +701 -0
- package/govendor/github.com/klauspost/compress/zstd/fse_predefined.go +168 -0
- package/govendor/github.com/klauspost/compress/zstd/hash.go +57 -0
- package/govendor/github.com/klauspost/compress/zstd/history.go +119 -0
- package/{examples/react-compiler/vendor/jsesc/LICENSE-MIT.txt → govendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt} +3 -1
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md +71 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +242 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +210 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +184 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go +11 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +75 -0
- package/govendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go +11 -0
- package/govendor/github.com/klauspost/compress/zstd/matchlen_amd64.go +15 -0
- package/govendor/github.com/klauspost/compress/zstd/matchlen_amd64.s +66 -0
- package/govendor/github.com/klauspost/compress/zstd/matchlen_generic.go +37 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec.go +620 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +115 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +4762 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_arm64.go +81 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_arm64.s +2919 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_asm.go +327 -0
- package/govendor/github.com/klauspost/compress/zstd/seqdec_generic.go +258 -0
- package/govendor/github.com/klauspost/compress/zstd/seqenc.go +159 -0
- package/govendor/github.com/klauspost/compress/zstd/simple.go +54 -0
- package/govendor/github.com/klauspost/compress/zstd/snappy.go +447 -0
- package/govendor/github.com/klauspost/compress/zstd/zip.go +141 -0
- package/govendor/github.com/klauspost/compress/zstd/zstd.go +126 -0
- package/govendor/github.com/tetratelabs/wazero/.editorconfig +7 -0
- package/govendor/github.com/tetratelabs/wazero/.gitattributes +2 -0
- package/govendor/github.com/tetratelabs/wazero/.gitmodules +3 -0
- package/govendor/github.com/tetratelabs/wazero/CONTRIBUTING.md +75 -0
- package/govendor/github.com/tetratelabs/wazero/LICENSE +201 -0
- package/govendor/github.com/tetratelabs/wazero/Makefile +385 -0
- package/govendor/github.com/tetratelabs/wazero/NOTICE +2 -0
- package/govendor/github.com/tetratelabs/wazero/RATIONALE.md +1577 -0
- package/govendor/github.com/tetratelabs/wazero/README.md +135 -0
- package/govendor/github.com/tetratelabs/wazero/api/features.go +228 -0
- package/govendor/github.com/tetratelabs/wazero/api/wasm.go +766 -0
- package/govendor/github.com/tetratelabs/wazero/builder.go +367 -0
- package/govendor/github.com/tetratelabs/wazero/cache.go +123 -0
- package/govendor/github.com/tetratelabs/wazero/codecov.yml +9 -0
- package/govendor/github.com/tetratelabs/wazero/config.go +899 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/checkpoint.go +35 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/close.go +63 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/compilationworkers.go +19 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/experimental.go +41 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/features.go +39 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/importresolver.go +19 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/listener.go +324 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/memory.go +52 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/dir.go +92 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/errno.go +98 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/error.go +45 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/file.go +356 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/fs.go +292 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/oflag.go +70 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/poll.go +48 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno.go +106 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_notwindows.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_unsupported.go +7 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/syscall_errno_windows.go +37 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/time.go +10 -0
- package/govendor/github.com/tetratelabs/wazero/experimental/sys/unimplemented.go +160 -0
- package/govendor/github.com/tetratelabs/wazero/fsconfig.go +213 -0
- package/govendor/github.com/tetratelabs/wazero/go.mod +3 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/args.go +97 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/clock.go +116 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/environ.go +100 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/fs.go +2015 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/poll.go +247 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/proc.go +44 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/random.go +55 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/sched.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/sock.go +188 -0
- package/govendor/github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1/wasi.go +314 -0
- package/govendor/github.com/tetratelabs/wazero/internal/descriptor/table.go +149 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/compiler.go +3884 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/format.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/interpreter.go +4963 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/operations.go +2978 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/interpreter/signature.go +825 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/abi.go +170 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/backend.go +3 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler.go +402 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/compiler_lower.go +226 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/go_call.go +33 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi.go +186 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.go +9 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_amd64.s +29 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_entry_preamble.go +248 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/abi_go_call.go +440 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/cond.go +168 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/ext.go +35 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr.go +2495 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/instr_encoding.go +1714 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_constant.go +71 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/lower_mem.go +187 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go +3782 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_pro_epi_logue.go +334 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_regalloc.go +352 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine_vec.go +992 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/operands.go +336 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/reg.go +181 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/stack.go +130 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi.go +376 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.go +9 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_arm64.s +29 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_entry_preamble.go +233 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/abi_go_call.go +430 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/cond.go +215 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr.go +2562 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/instr_encoding.go +2357 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_constant.go +301 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr.go +2227 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_instr_operands.go +340 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/lower_mem.go +451 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine.go +631 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_pro_epi_logue.go +486 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_regalloc.go +351 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/machine_relocation.go +128 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/reg.go +397 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/arm64/unwind_stack.go +84 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/machine.go +119 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/api.go +124 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/reg.go +123 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regalloc.go +1189 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc/regset.go +96 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/vdef.go +19 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/call_engine.go +884 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine.go +1008 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/engine_cache.go +347 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_amd64.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_arm64.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/entrypoint_other.go +15 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/frontend.go +663 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/lower.go +5045 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/misc.go +10 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/frontend/sort_id.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/hostmodule.go +88 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_amd64.go +30 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_arm64.go +32 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/isa_other.go +29 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/memmove.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/module_engine.go +403 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block.go +381 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/basic_block_sort.go +32 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/builder.go +790 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/cmp.go +107 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/funcref.go +12 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/instructions.go +3024 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass.go +393 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_blk_layouts.go +334 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/pass_cfg.go +313 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/signature.go +49 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/ssa.go +14 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/type.go +115 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/ssa/vs.go +114 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/debug_options.go +189 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/exitcode.go +150 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/offsetdata.go +248 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap.go +96 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_disabled.go +5 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/perfmap_enabled.go +5 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/pool.go +215 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/ptr.go +15 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/queue.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi/resetmap.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/checkpoint.go +10 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/close.go +5 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/compilationworkers.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/expctxkeys.go +2 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/importresolver.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/listener.go +7 -0
- package/govendor/github.com/tetratelabs/wazero/internal/expctxkeys/memory.go +4 -0
- package/govendor/github.com/tetratelabs/wazero/internal/filecache/compilationcache.go +42 -0
- package/govendor/github.com/tetratelabs/wazero/internal/filecache/file_cache.go +76 -0
- package/govendor/github.com/tetratelabs/wazero/internal/ieee754/ieee754.go +29 -0
- package/govendor/github.com/tetratelabs/wazero/internal/internalapi/internal.go +9 -0
- package/govendor/github.com/tetratelabs/wazero/internal/leb128/leb128.go +285 -0
- package/govendor/github.com/tetratelabs/wazero/internal/moremath/moremath.go +271 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_amd64.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_arm64.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/cpuid_unsupported.go +5 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/crypto.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_linux.go +72 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_other.go +19 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_unix.go +14 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_unsupported.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/mmap_windows.go +31 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/path.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/path_windows.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/platform.go +71 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/time.go +76 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/time_notwindows.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/platform/time_windows.go +40 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sock/sock.go +89 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sock/sock_supported.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sock/sock_unsupported.go +10 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sys/fs.go +458 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sys/lazy.go +135 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sys/stdio.go +125 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sys/sys.go +228 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/adapter.go +105 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_linux.go +12 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/datasync_unsupported.go +14 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/dir.go +24 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/dirfs.go +137 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file.go +576 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_unix.go +39 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_unsupported.go +28 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/file_windows.go +133 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens.go +29 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.go +42 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_darwin.s +8 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_linux.go +38 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_unsupported.go +18 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/futimens_windows.go +42 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_unix.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_unsupported.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/ino_windows.go +28 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unix.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_unsupported.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/nonblock_windows.go +23 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/oflag.go +38 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_bsd.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_darwin.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_supported.go +30 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unix.go +20 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_unsupported.go +16 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/open_file_windows.go +163 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/osfile.go +318 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.go +55 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_darwin.s +8 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_linux.go +57 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_unsupported.go +12 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/poll_windows.go +224 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/readfs.go +117 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_unix.go +16 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_unsupported.go +16 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/rename_windows.go +55 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock.go +186 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_supported.go +76 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unix.go +48 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_unsupported.go +79 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sock_windows.go +80 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat.go +16 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_supported.go +40 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_unsupported.go +40 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/stat_windows.go +118 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sync.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sync_windows.go +20 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/syscall6_darwin.go +13 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/sysfs.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_unix.go +17 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_unsupported.go +14 -0
- package/govendor/github.com/tetratelabs/wazero/internal/sysfs/unlink_windows.go +25 -0
- package/govendor/github.com/tetratelabs/wazero/internal/u32/u32.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/u64/u64.go +15 -0
- package/govendor/github.com/tetratelabs/wazero/internal/version/version.go +52 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/args.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/clock.go +16 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/environ.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/errno.go +314 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/fs.go +164 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/poll.go +15 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/proc.go +6 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/random.go +3 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/rights.go +148 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/sched.go +3 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/sock.go +71 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasip1/wasi.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/code.go +124 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/const_expr.go +116 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/custom.go +22 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/data.go +79 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/decoder.go +235 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/element.go +250 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/errors.go +11 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/export.go +32 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/function.go +56 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/global.go +50 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/header.go +9 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/import.go +68 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/limits.go +47 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/memory.go +42 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/names.go +151 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/section.go +309 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/table.go +79 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/binary/value.go +97 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/const_expr.go +259 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/counts.go +53 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/engine.go +76 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/exception.go +9 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/func_validation.go +2804 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/function_definition.go +210 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/global.go +55 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/gofunc.go +279 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/host.go +179 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/instruction.go +1962 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/memory.go +475 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/memory_definition.go +128 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/module.go +1359 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/module_instance.go +251 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/module_instance_lookup.go +73 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/store.go +726 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/store_module_list.go +95 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasm/table.go +321 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasmdebug/debug.go +170 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasmdebug/dwarf.go +225 -0
- package/govendor/github.com/tetratelabs/wazero/internal/wasmruntime/errors.go +54 -0
- package/govendor/github.com/tetratelabs/wazero/netlify.toml +15 -0
- package/govendor/github.com/tetratelabs/wazero/runtime.go +399 -0
- package/govendor/github.com/tetratelabs/wazero/sys/clock.go +26 -0
- package/govendor/github.com/tetratelabs/wazero/sys/error.go +83 -0
- package/govendor/github.com/tetratelabs/wazero/sys/stat.go +105 -0
- package/govendor/github.com/tetratelabs/wazero/sys/stat_bsd.go +29 -0
- package/govendor/github.com/tetratelabs/wazero/sys/stat_posix.go +29 -0
- package/govendor/github.com/tetratelabs/wazero/sys/stat_unsupported.go +14 -0
- package/govendor/github.com/tetratelabs/wazero/sys/stat_windows.go +24 -0
- package/govendor/golang.org/x/crypto/LICENSE +27 -0
- package/govendor/golang.org/x/crypto/PATENTS +22 -0
- package/govendor/golang.org/x/crypto/cryptobyte/asn1/asn1.go +46 -0
- package/govendor/golang.org/x/crypto/cryptobyte/asn1.go +825 -0
- package/govendor/golang.org/x/crypto/cryptobyte/builder.go +350 -0
- package/govendor/golang.org/x/crypto/cryptobyte/string.go +183 -0
- package/govendor/golang.org/x/crypto/go.mod +3 -0
- package/govendor/golang.org/x/crypto/pbkdf2/pbkdf2.go +30 -0
- package/govendor/golang.org/x/net/LICENSE +27 -0
- package/govendor/golang.org/x/net/PATENTS +22 -0
- package/govendor/golang.org/x/net/dns/dnsmessage/message.go +2733 -0
- package/govendor/golang.org/x/net/dns/dnsmessage/svcb.go +333 -0
- package/govendor/golang.org/x/net/go.mod +3 -0
- package/govendor/golang.org/x/net/http/httpguts/guts.go +50 -0
- package/govendor/golang.org/x/net/http/httpguts/httplex.go +347 -0
- package/govendor/golang.org/x/net/http2/README.md +19 -0
- package/govendor/golang.org/x/net/http2/ascii.go +65 -0
- package/govendor/golang.org/x/net/http2/ciphers.go +641 -0
- package/govendor/golang.org/x/net/http2/client_conn_pool.go +301 -0
- package/govendor/golang.org/x/net/http2/client_priority_go126.go +20 -0
- package/govendor/golang.org/x/net/http2/client_priority_go127.go +13 -0
- package/govendor/golang.org/x/net/http2/clientconn.go +57 -0
- package/govendor/golang.org/x/net/http2/config.go +171 -0
- package/govendor/golang.org/x/net/http2/config_go125.go +15 -0
- package/govendor/golang.org/x/net/http2/config_go126.go +15 -0
- package/govendor/golang.org/x/net/http2/databuffer.go +146 -0
- package/govendor/golang.org/x/net/http2/errors.go +145 -0
- package/govendor/golang.org/x/net/http2/flow.go +175 -0
- package/govendor/golang.org/x/net/http2/frame.go +1887 -0
- package/govendor/golang.org/x/net/http2/gotrack.go +181 -0
- package/govendor/golang.org/x/net/http2/hpack/encode.go +244 -0
- package/govendor/golang.org/x/net/http2/hpack/hpack.go +522 -0
- package/govendor/golang.org/x/net/http2/hpack/huffman.go +226 -0
- package/govendor/golang.org/x/net/http2/hpack/static_table.go +188 -0
- package/govendor/golang.org/x/net/http2/hpack/tables.go +426 -0
- package/govendor/golang.org/x/net/http2/http2.go +415 -0
- package/govendor/golang.org/x/net/http2/pipe.go +184 -0
- package/govendor/golang.org/x/net/http2/server.go +3239 -0
- package/govendor/golang.org/x/net/http2/server_common.go +268 -0
- package/govendor/golang.org/x/net/http2/server_wrap.go +217 -0
- package/govendor/golang.org/x/net/http2/transport.go +3084 -0
- package/govendor/golang.org/x/net/http2/transport_common.go +516 -0
- package/govendor/golang.org/x/net/http2/transport_wrap.go +583 -0
- package/govendor/golang.org/x/net/http2/unencrypted.go +32 -0
- package/govendor/golang.org/x/net/http2/write.go +381 -0
- package/govendor/golang.org/x/net/http2/writesched.go +253 -0
- package/govendor/golang.org/x/net/http2/writesched_common.go +90 -0
- package/govendor/golang.org/x/net/http2/writesched_priority_rfc7540.go +420 -0
- package/govendor/golang.org/x/net/http2/writesched_priority_rfc9218.go +226 -0
- package/govendor/golang.org/x/net/http2/writesched_random.go +81 -0
- package/govendor/golang.org/x/net/http2/writesched_roundrobin.go +121 -0
- package/govendor/golang.org/x/net/idna/idna.go +884 -0
- package/govendor/golang.org/x/net/idna/punycode.go +220 -0
- package/govendor/golang.org/x/net/idna/tables15.0.0.go +5144 -0
- package/govendor/golang.org/x/net/idna/tables17.0.0.go +5302 -0
- package/govendor/golang.org/x/net/idna/trie.go +51 -0
- package/govendor/golang.org/x/net/idna/trieval.go +119 -0
- package/govendor/golang.org/x/net/internal/httpcommon/README.md +7 -0
- package/govendor/golang.org/x/net/internal/httpcommon/ascii.go +53 -0
- package/govendor/golang.org/x/net/internal/httpcommon/gzip.go +132 -0
- package/govendor/golang.org/x/net/internal/httpcommon/headermap.go +115 -0
- package/govendor/golang.org/x/net/internal/httpcommon/request.go +503 -0
- package/govendor/golang.org/x/net/internal/httpsfv/httpsfv.go +665 -0
- package/govendor/golang.org/x/sys/cpu/asm_aix_ppc64.s +17 -0
- package/govendor/golang.org/x/sys/cpu/asm_darwin_arm64_gc.s +12 -0
- package/govendor/golang.org/x/sys/cpu/asm_darwin_x86_gc.s +17 -0
- package/govendor/golang.org/x/sys/cpu/byteorder.go +66 -0
- package/govendor/golang.org/x/sys/cpu/cpu.go +357 -0
- package/govendor/golang.org/x/sys/cpu/cpu_aix.go +33 -0
- package/govendor/golang.org/x/sys/cpu/cpu_arm.go +73 -0
- package/govendor/golang.org/x/sys/cpu/cpu_arm64.go +191 -0
- package/govendor/golang.org/x/sys/cpu/cpu_arm64.s +35 -0
- package/govendor/golang.org/x/sys/cpu/cpu_darwin_arm64.go +67 -0
- package/govendor/golang.org/x/sys/cpu/cpu_darwin_arm64_other.go +31 -0
- package/govendor/golang.org/x/sys/cpu/cpu_darwin_x86.go +61 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gc_arm64.go +12 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gc_riscv64.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gc_s390x.go +21 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gc_x86.go +15 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gc_x86.s +26 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gccgo_arm64.go +12 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gccgo_s390x.go +22 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gccgo_x86.c +37 -0
- package/govendor/golang.org/x/sys/cpu/cpu_gccgo_x86.go +25 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux.go +15 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_arm.go +39 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_arm64.go +120 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_loong64.go +22 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_mips64x.go +22 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_noinit.go +9 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_ppc64x.go +34 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_riscv64.go +173 -0
- package/govendor/golang.org/x/sys/cpu/cpu_linux_s390x.go +40 -0
- package/govendor/golang.org/x/sys/cpu/cpu_loong64.go +62 -0
- package/govendor/golang.org/x/sys/cpu/cpu_loong64.s +13 -0
- package/govendor/golang.org/x/sys/cpu/cpu_mips64x.go +15 -0
- package/govendor/golang.org/x/sys/cpu/cpu_mipsx.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_netbsd_amd64.go +47 -0
- package/govendor/golang.org/x/sys/cpu/cpu_netbsd_arm64.go +173 -0
- package/govendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.go +65 -0
- package/govendor/golang.org/x/sys/cpu/cpu_openbsd_arm64.s +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_arm.go +9 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_arm64.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_mips64x.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_ppc64x.go +12 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_riscv64.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_other_x86.go +11 -0
- package/govendor/golang.org/x/sys/cpu/cpu_ppc64x.go +16 -0
- package/govendor/golang.org/x/sys/cpu/cpu_riscv64.go +33 -0
- package/govendor/golang.org/x/sys/cpu/cpu_riscv64.s +17 -0
- package/govendor/golang.org/x/sys/cpu/cpu_s390x.go +172 -0
- package/govendor/golang.org/x/sys/cpu/cpu_s390x.s +57 -0
- package/govendor/golang.org/x/sys/cpu/cpu_sparc64.go +12 -0
- package/govendor/golang.org/x/sys/cpu/cpu_wasm.go +17 -0
- package/govendor/golang.org/x/sys/cpu/cpu_windows.go +26 -0
- package/govendor/golang.org/x/sys/cpu/cpu_windows_arm64.go +38 -0
- package/govendor/golang.org/x/sys/cpu/cpu_x86.go +259 -0
- package/govendor/golang.org/x/sys/cpu/cpu_zos.go +10 -0
- package/govendor/golang.org/x/sys/cpu/cpu_zos_s390x.go +25 -0
- package/govendor/golang.org/x/sys/cpu/endian_big.go +10 -0
- package/govendor/golang.org/x/sys/cpu/endian_little.go +10 -0
- package/govendor/golang.org/x/sys/cpu/hwcap_linux.go +71 -0
- package/govendor/golang.org/x/sys/cpu/parse.go +55 -0
- package/govendor/golang.org/x/sys/cpu/proc_cpuinfo_linux.go +53 -0
- package/govendor/golang.org/x/sys/cpu/runtime_auxv.go +16 -0
- package/govendor/golang.org/x/sys/cpu/runtime_auxv_go121.go +18 -0
- package/govendor/golang.org/x/sys/cpu/syscall_aix_gccgo.go +26 -0
- package/govendor/golang.org/x/sys/cpu/syscall_aix_ppc64_gc.go +35 -0
- package/govendor/golang.org/x/sys/cpu/syscall_darwin_arm64_gc.go +54 -0
- package/govendor/golang.org/x/sys/cpu/syscall_darwin_x86_gc.go +98 -0
- package/govendor/golang.org/x/sys/cpu/zcpu_windows.go +48 -0
- package/govendor/golang.org/x/sys/unix/asm_linux_ppc64x.s +20 -0
- package/govendor/golang.org/x/sys/unix/asm_linux_sparc64.s +48 -0
- package/govendor/golang.org/x/sys/unix/asm_zos_s390x.s +4 -186
- package/govendor/golang.org/x/sys/unix/bpxsvc_zos.s +9 -9
- package/govendor/golang.org/x/sys/unix/fcntl_darwin.go +2 -2
- package/govendor/golang.org/x/sys/unix/mkerrors.sh +1 -1
- package/govendor/golang.org/x/sys/unix/ptrace_zos.go +537 -0
- package/govendor/golang.org/x/sys/unix/ptrace_zos_singlestep.go +425 -0
- package/govendor/golang.org/x/sys/unix/syscall_darwin.go +1 -0
- package/govendor/golang.org/x/sys/unix/syscall_freebsd.go +9 -0
- package/govendor/golang.org/x/sys/unix/syscall_linux.go +20 -1
- package/govendor/golang.org/x/sys/unix/syscall_linux_sparc64.go +3 -3
- package/govendor/golang.org/x/sys/unix/syscall_netbsd.go +9 -0
- package/govendor/golang.org/x/sys/unix/syscall_unix_gc.go +1 -1
- package/govendor/golang.org/x/sys/unix/syscall_zos_s390x.go +0 -4
- package/govendor/golang.org/x/sys/unix/xattr_bsd.go +12 -12
- package/govendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go +6 -0
- package/govendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go +6 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux.go +32 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_386.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_arm.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_loong64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_mips.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +2 -0
- package/govendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +2 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go +11 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go +11 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_linux.go +28 -1
- package/govendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go +3 -3
- package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go +159 -0
- package/govendor/golang.org/x/sys/unix/zsysnum_zos_s390x.go +13 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux.go +65 -36
- package/govendor/golang.org/x/sys/unix/ztypes_linux_386.go +6 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_arm.go +6 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_loong64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_mips.go +6 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go +6 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc.go +6 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +7 -0
- package/govendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go +10 -5
- package/govendor/golang.org/x/sys/unix/ztypes_zos_s390x.go +78 -0
- package/govendor/golang.org/x/text/.gitattributes +10 -0
- package/govendor/golang.org/x/text/CONTRIBUTING.md +26 -0
- package/govendor/golang.org/x/text/LICENSE +27 -0
- package/govendor/golang.org/x/text/PATENTS +22 -0
- package/govendor/golang.org/x/text/README.md +81 -0
- package/govendor/golang.org/x/text/cases/cases.go +162 -0
- package/govendor/golang.org/x/text/cases/context.go +376 -0
- package/govendor/golang.org/x/text/cases/fold.go +34 -0
- package/govendor/golang.org/x/text/cases/icu.go +61 -0
- package/govendor/golang.org/x/text/cases/info.go +82 -0
- package/govendor/golang.org/x/text/cases/map.go +816 -0
- package/govendor/golang.org/x/text/cases/tables15.0.0.go +2527 -0
- package/govendor/golang.org/x/text/cases/tables17.0.0.go +2642 -0
- package/govendor/golang.org/x/text/cases/trieval.go +217 -0
- package/govendor/golang.org/x/text/codereview.cfg +1 -0
- package/govendor/golang.org/x/text/collate/collate.go +402 -0
- package/govendor/golang.org/x/text/collate/index.go +32 -0
- package/govendor/golang.org/x/text/collate/option.go +239 -0
- package/govendor/golang.org/x/text/collate/sort.go +81 -0
- package/govendor/golang.org/x/text/collate/tables.go +73789 -0
- package/govendor/golang.org/x/text/currency/common.go +67 -0
- package/govendor/golang.org/x/text/currency/currency.go +185 -0
- package/govendor/golang.org/x/text/currency/format.go +220 -0
- package/govendor/golang.org/x/text/currency/query.go +152 -0
- package/govendor/golang.org/x/text/currency/tables.go +2629 -0
- package/govendor/golang.org/x/text/doc.go +16 -0
- package/govendor/golang.org/x/text/feature/plural/common.go +70 -0
- package/govendor/golang.org/x/text/feature/plural/message.go +244 -0
- package/govendor/golang.org/x/text/feature/plural/plural.go +262 -0
- package/govendor/golang.org/x/text/feature/plural/tables.go +552 -0
- package/govendor/golang.org/x/text/go.mod +3 -0
- package/govendor/golang.org/x/text/internal/catmsg/catmsg.go +417 -0
- package/govendor/golang.org/x/text/internal/catmsg/codec.go +407 -0
- package/govendor/golang.org/x/text/internal/catmsg/varint.go +62 -0
- package/govendor/golang.org/x/text/internal/colltab/collelem.go +374 -0
- package/govendor/golang.org/x/text/internal/colltab/colltab.go +105 -0
- package/govendor/golang.org/x/text/internal/colltab/contract.go +145 -0
- package/govendor/golang.org/x/text/internal/colltab/iter.go +178 -0
- package/govendor/golang.org/x/text/internal/colltab/numeric.go +236 -0
- package/govendor/golang.org/x/text/internal/colltab/table.go +275 -0
- package/govendor/golang.org/x/text/internal/colltab/trie.go +159 -0
- package/govendor/golang.org/x/text/internal/colltab/weighter.go +31 -0
- package/govendor/golang.org/x/text/internal/format/format.go +41 -0
- package/govendor/golang.org/x/text/internal/format/parser.go +358 -0
- package/govendor/golang.org/x/text/internal/internal.go +49 -0
- package/govendor/golang.org/x/text/internal/language/common.go +16 -0
- package/govendor/golang.org/x/text/internal/language/compact/compact.go +61 -0
- package/govendor/golang.org/x/text/internal/language/compact/language.go +260 -0
- package/govendor/golang.org/x/text/internal/language/compact/parents.go +120 -0
- package/govendor/golang.org/x/text/internal/language/compact/tables.go +1015 -0
- package/govendor/golang.org/x/text/internal/language/compact/tags.go +91 -0
- package/govendor/golang.org/x/text/internal/language/compact.go +29 -0
- package/govendor/golang.org/x/text/internal/language/compose.go +167 -0
- package/govendor/golang.org/x/text/internal/language/coverage.go +28 -0
- package/govendor/golang.org/x/text/internal/language/language.go +627 -0
- package/govendor/golang.org/x/text/internal/language/lookup.go +412 -0
- package/govendor/golang.org/x/text/internal/language/match.go +226 -0
- package/govendor/golang.org/x/text/internal/language/parse.go +608 -0
- package/govendor/golang.org/x/text/internal/language/tables.go +3494 -0
- package/govendor/golang.org/x/text/internal/language/tags.go +48 -0
- package/govendor/golang.org/x/text/internal/match.go +67 -0
- package/govendor/golang.org/x/text/internal/number/common.go +55 -0
- package/govendor/golang.org/x/text/internal/number/decimal.go +500 -0
- package/govendor/golang.org/x/text/internal/number/format.go +533 -0
- package/govendor/golang.org/x/text/internal/number/number.go +152 -0
- package/govendor/golang.org/x/text/internal/number/pattern.go +485 -0
- package/govendor/golang.org/x/text/internal/number/roundingmode_string.go +30 -0
- package/govendor/golang.org/x/text/internal/number/tables.go +1219 -0
- package/govendor/golang.org/x/text/internal/stringset/set.go +86 -0
- package/govendor/golang.org/x/text/internal/tag/tag.go +100 -0
- package/govendor/golang.org/x/text/language/coverage.go +187 -0
- package/govendor/golang.org/x/text/language/display/dict.go +92 -0
- package/govendor/golang.org/x/text/language/display/display.go +420 -0
- package/govendor/golang.org/x/text/language/display/lookup.go +253 -0
- package/govendor/golang.org/x/text/language/display/tables.go +53114 -0
- package/govendor/golang.org/x/text/language/doc.go +98 -0
- package/govendor/golang.org/x/text/language/language.go +605 -0
- package/govendor/golang.org/x/text/language/match.go +735 -0
- package/govendor/golang.org/x/text/language/parse.go +258 -0
- package/govendor/golang.org/x/text/language/tables.go +298 -0
- package/govendor/golang.org/x/text/language/tags.go +145 -0
- package/govendor/golang.org/x/text/message/catalog/catalog.go +365 -0
- package/govendor/golang.org/x/text/message/catalog/dict.go +133 -0
- package/govendor/golang.org/x/text/secure/bidirule/bidirule.go +346 -0
- package/govendor/golang.org/x/text/transform/transform.go +709 -0
- package/govendor/golang.org/x/text/unicode/bidi/bidi.go +359 -0
- package/govendor/golang.org/x/text/unicode/bidi/bracket.go +335 -0
- package/govendor/golang.org/x/text/unicode/bidi/core.go +1050 -0
- package/govendor/golang.org/x/text/unicode/bidi/prop.go +206 -0
- package/govendor/golang.org/x/text/unicode/bidi/tables15.0.0.go +2042 -0
- package/govendor/golang.org/x/text/unicode/bidi/tables17.0.0.go +2135 -0
- package/govendor/golang.org/x/text/unicode/bidi/trieval.go +48 -0
- package/govendor/golang.org/x/text/unicode/norm/composition.go +524 -0
- package/govendor/golang.org/x/text/unicode/norm/forminfo.go +296 -0
- package/govendor/golang.org/x/text/unicode/norm/input.go +109 -0
- package/govendor/golang.org/x/text/unicode/norm/iter.go +454 -0
- package/govendor/golang.org/x/text/unicode/norm/normalize.go +613 -0
- package/govendor/golang.org/x/text/unicode/norm/readwriter.go +125 -0
- package/govendor/golang.org/x/text/unicode/norm/tables15.0.0.go +7907 -0
- package/govendor/golang.org/x/text/unicode/norm/tables17.0.0.go +8104 -0
- package/govendor/golang.org/x/text/unicode/norm/transform.go +89 -0
- package/govendor/golang.org/x/text/unicode/norm/trie.go +54 -0
- package/govendor/manifest.txt +11 -2
- package/govendor/software.sslmate.com/src/go-pkcs12/.gitattributes +10 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/LICENSE +28 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/README.md +31 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/bmp-string.go +63 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/crypto.go +331 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/errors.go +23 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/go.mod +3 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/internal/rc2/rc2.go +271 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/mac.go +202 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/pbkdf.go +184 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/pkcs12.go +1000 -0
- package/govendor/software.sslmate.com/src/go-pkcs12/safebags.go +107 -0
- package/package.json +11 -1
- package/types/tsconfig.json +17 -0
- package/types/typenative-npm.d.ts +1327 -80
- package/types/typenative.d.ts +1312 -25
- package/.gitattributes +0 -1
- package/.github/workflows/ci.yml +0 -68
- package/.github/workflows/npm-publish-github-packages.yml +0 -88
- package/CHANGELOG.md +0 -1358
- package/CONTRIBUTION.md +0 -133
- package/conformance.md +0 -1450
- package/examples/README.md +0 -113
- package/examples/angular-templates/README.md +0 -141
- package/examples/angular-templates/bench.mjs +0 -173
- package/examples/angular-templates/dom-type-globals.d.ts +0 -21
- package/examples/angular-templates/dom-type-stubs.ts +0 -21
- package/examples/angular-templates/main.ts +0 -135
- package/examples/angular-templates/parity.mjs +0 -157
- package/examples/angular-templates/run-node.mjs +0 -69
- package/examples/angular-templates/vendor/angular/compiler/LICENSE +0 -21
- package/examples/desktop-notes/README.md +0 -106
- package/examples/desktop-notes/bench.mjs +0 -125
- package/examples/desktop-notes/main.ts +0 -131
- package/examples/desktop-notes/package-lock.json +0 -726
- package/examples/desktop-notes/package.json +0 -9
- package/examples/desktop-notes/ui.ts +0 -176
- package/examples/dupe-finder/README.md +0 -78
- package/examples/dupe-finder/bench.mjs +0 -120
- package/examples/dupe-finder/main.ts +0 -87
- package/examples/fibonacci/README.md +0 -89
- package/examples/fibonacci/bench.mjs +0 -87
- package/examples/fibonacci/main.ts +0 -21
- package/examples/hello-world/README.md +0 -62
- package/examples/hello-world/bench.mjs +0 -66
- package/examples/hello-world/main.ts +0 -7
- package/examples/json-server/README.md +0 -131
- package/examples/json-server/bench.mjs +0 -263
- package/examples/json-server/main.ts +0 -67
- package/examples/mandelbrot-png/README.md +0 -63
- package/examples/mandelbrot-png/bench.mjs +0 -95
- package/examples/mandelbrot-png/main.ts +0 -98
- package/examples/mandelbrot-png/preview.png +0 -0
- package/examples/marked-cli/README.md +0 -175
- package/examples/marked-cli/bench.mjs +0 -178
- package/examples/marked-cli/main.ts +0 -410
- package/examples/marked-cli/parity.mjs +0 -185
- package/examples/marked-cli/vendor/marked/LICENSE +0 -44
- package/examples/marked-cli/vendor/marked/VENDOR.md +0 -26
- package/examples/openai-chat/README.md +0 -148
- package/examples/openai-chat/bench.mjs +0 -175
- package/examples/openai-chat/main.ts +0 -488
- package/examples/react-compiler/README.md +0 -185
- package/examples/react-compiler/bench.mjs +0 -181
- package/examples/react-compiler/main.ts +0 -324
- package/examples/react-compiler/parity.mjs +0 -362
- package/examples/react-compiler/run-node.mjs +0 -90
- package/examples/react-compiler/vendor/babel/LICENSE +0 -22
- package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-lower.json +0 -15
- package/examples/react-compiler/vendor/babel/helper-globals/data/builtin-upper.json +0 -51
- package/examples/react-compiler/vendor/charcodes/package.json +0 -25
- package/examples/react-compiler/vendor/invariant/LICENSE +0 -21
- package/examples/react-compiler/vendor/invariant/invariant.cjs +0 -51
- package/examples/react-compiler/vendor/invariant/invariant.d.cts +0 -11
- package/examples/react-compiler/vendor/invariant/types/LICENSE +0 -21
- package/examples/react-compiler/vendor/jridgewell/gen-mapping/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/resolve-uri/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/sourcemap-codec/LICENSE +0 -19
- package/examples/react-compiler/vendor/jridgewell/trace-mapping/LICENSE +0 -19
- package/examples/react-compiler/vendor/js-tokens/LICENSE +0 -21
- package/examples/react-compiler/vendor/js-tokens/index.cjs +0 -23
- package/examples/react-compiler/vendor/js-tokens/index.d.cts +0 -13
- package/examples/react-compiler/vendor/jsesc/jsesc.cjs +0 -337
- package/examples/react-compiler/vendor/jsesc/jsesc.d.cts +0 -384
- package/examples/react-compiler/vendor/jsesc/types/LICENSE +0 -21
- package/examples/react-compiler/vendor/picocolors/picocolors.cjs +0 -75
- package/examples/react-compiler/vendor/picocolors/picocolors.d.cts +0 -5
- package/examples/react-compiler/vendor/picocolors/types.d.ts +0 -51
- package/examples/react-compiler/vendor/react-compiler/LICENSE +0 -21
- package/examples/react-compiler/vendor/stubs/babel-core.ts +0 -31
- package/examples/react-compiler/vendor/stubs/debug.ts +0 -18
- package/examples/react-compiler/vendor/stubs/js-tokens.d.ts +0 -13
- package/examples/react-compiler/vendor/stubs/pretty-format.ts +0 -18
- package/examples/react-compiler/vendor/zod/LICENSE +0 -21
- package/examples/react-compiler/vendor/zod-validation-error/LICENSE +0 -9
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/NonEmptyArray.ts +0 -5
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinPath.ts +0 -49
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/joinValues.ts +0 -48
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/stringify.ts +0 -50
- package/examples/react-compiler/vendor/zod-validation-error/lib/utils/titleCase.ts +0 -6
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/MessageBuilder.ts +0 -126
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/ValidationError.ts +0 -39
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/custom.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/errorMap.ts +0 -78
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/index.ts +0 -2
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidElement.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidKey.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidStringFormat.ts +0 -131
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidType.ts +0 -54
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidUnion.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/invalidValue.ts +0 -40
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/notMultipleOf.ts +0 -12
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooBig.ts +0 -76
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/tooSmall.ts +0 -76
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/types.ts +0 -27
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/errorMap/unrecognizedKeys.ts +0 -27
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromError.ts +0 -13
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodError.ts +0 -61
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/fromZodIssue.ts +0 -38
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/index.ts +0 -20
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationError.ts +0 -5
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isValidationErrorLike.ts +0 -8
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/isZodErrorLike.ts +0 -11
- package/examples/react-compiler/vendor/zod-validation-error/lib/v4/toValidationError.ts +0 -20
- package/examples/react-compiler/vendor.mjs +0 -688
- package/examples/todo-cli/README.md +0 -69
- package/examples/todo-cli/bench.mjs +0 -89
- package/examples/todo-cli/main.ts +0 -92
- package/examples/yaml-cli/README.md +0 -152
- package/examples/yaml-cli/bench.mjs +0 -265
- package/examples/yaml-cli/main.ts +0 -251
- package/examples/yaml-cli/parity.mjs +0 -156
- package/examples/yaml-cli/vendor/yaml/LICENSE +0 -13
- package/examples/yaml-cli/vendor/yaml/VENDOR.md +0 -25
- package/govendor/golang.org/x/sys/unix/syscall_unix_gc_ppc64x.go +0 -22
- package/govendor/golang.org/x/sys/windows/types_windows_arm.go +0 -35
- package/performance/README.md +0 -196
- package/performance/any-dynamic.ts +0 -21
- package/performance/any-record-view.ts +0 -20
- package/performance/arr-hof.ts +0 -21
- package/performance/arr-push-index.ts +0 -15
- package/performance/arr-sort.ts +0 -21
- package/performance/async-await.ts +0 -24
- package/performance/class-dispatch.ts +0 -52
- package/performance/events-emit.ts +0 -27
- package/performance/exc-throw-rare.ts +0 -25
- package/performance/exc-try-catch.ts +0 -26
- package/performance/fn-closures.ts +0 -29
- package/performance/fn-recursion-tree.ts +0 -31
- package/performance/gen-for-of.ts +0 -46
- package/performance/json-roundtrip.ts +0 -27
- package/performance/map-delete.ts +0 -24
- package/performance/map-string-keys.ts +0 -22
- package/performance/num-arith-loop.ts +0 -14
- package/performance/num-bitwise.ts +0 -21
- package/performance/num-mod-int.ts +0 -16
- package/performance/num-to-string.ts +0 -16
- package/performance/obj-alloc.ts +0 -32
- package/performance/regex-exec.ts +0 -29
- package/performance/regex-site.ts +0 -28
- package/performance/set-ops.ts +0 -20
- package/performance/str-char-scan.ts +0 -35
- package/performance/str-concat-plus.ts +0 -18
- package/performance/str-concat.ts +0 -24
- package/performance/str-short-scan.ts +0 -26
- package/performance/str-split-join.ts +0 -23
- package/performance/str-template.ts +0 -16
- package/performance/union-switch.ts +0 -57
- package/tsparser/check.go +0 -158
- package/tsparser/go.mod +0 -14
- package/tsparser/go.sum +0 -22
- package/tsparser/main.go +0 -363
- package/tsparser/tsparser.exe +0 -4
package/govendor/github.com/tetratelabs/wazero/internal/engine/wazevo/backend/isa/amd64/machine.go
ADDED
|
@@ -0,0 +1,3782 @@
|
|
|
1
|
+
package amd64
|
|
2
|
+
|
|
3
|
+
import (
|
|
4
|
+
"context"
|
|
5
|
+
"encoding/binary"
|
|
6
|
+
"fmt"
|
|
7
|
+
"math"
|
|
8
|
+
"strings"
|
|
9
|
+
|
|
10
|
+
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend"
|
|
11
|
+
"github.com/tetratelabs/wazero/internal/engine/wazevo/backend/regalloc"
|
|
12
|
+
"github.com/tetratelabs/wazero/internal/engine/wazevo/ssa"
|
|
13
|
+
"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
|
|
14
|
+
"github.com/tetratelabs/wazero/internal/platform"
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
// NewBackend returns a new backend for arm64.
|
|
18
|
+
func NewBackend() backend.Machine {
|
|
19
|
+
m := &machine{
|
|
20
|
+
cpuFeatures: platform.CpuFeatures,
|
|
21
|
+
regAlloc: regalloc.NewAllocator[*instruction, *labelPosition, *regAllocFn](regInfo),
|
|
22
|
+
spillSlots: map[regalloc.VRegID]int64{},
|
|
23
|
+
amodePool: wazevoapi.NewPool[amode](nil),
|
|
24
|
+
labelPositionPool: wazevoapi.NewIDedPool[labelPosition](resetLabelPosition),
|
|
25
|
+
instrPool: wazevoapi.NewPool[instruction](resetInstruction),
|
|
26
|
+
constSwizzleMaskConstIndex: -1,
|
|
27
|
+
constSqmulRoundSatIndex: -1,
|
|
28
|
+
constI8x16SHLMaskTableIndex: -1,
|
|
29
|
+
constI8x16LogicalSHRMaskTableIndex: -1,
|
|
30
|
+
constF64x2CvtFromIMaskIndex: -1,
|
|
31
|
+
constTwop52Index: -1,
|
|
32
|
+
constI32sMaxOnF64x2Index: -1,
|
|
33
|
+
constI32uMaxOnF64x2Index: -1,
|
|
34
|
+
constAllOnesI8x16Index: -1,
|
|
35
|
+
constAllOnesI16x8Index: -1,
|
|
36
|
+
constExtAddPairwiseI16x8uMask1Index: -1,
|
|
37
|
+
constExtAddPairwiseI16x8uMask2Index: -1,
|
|
38
|
+
}
|
|
39
|
+
m.regAllocFn.m = m
|
|
40
|
+
return m
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
type (
|
|
44
|
+
// machine implements backend.Machine for amd64.
|
|
45
|
+
machine struct {
|
|
46
|
+
c backend.Compiler
|
|
47
|
+
stackBoundsCheckDisabled bool
|
|
48
|
+
|
|
49
|
+
instrPool wazevoapi.Pool[instruction]
|
|
50
|
+
amodePool wazevoapi.Pool[amode]
|
|
51
|
+
|
|
52
|
+
cpuFeatures platform.CpuFeatureFlags
|
|
53
|
+
|
|
54
|
+
regAlloc regalloc.Allocator[*instruction, *labelPosition, *regAllocFn]
|
|
55
|
+
regAllocFn regAllocFn
|
|
56
|
+
regAllocStarted bool
|
|
57
|
+
|
|
58
|
+
// labelPositionPool is the pool of labelPosition. The id is the label where
|
|
59
|
+
// if the label is less than the maxSSABlockID, it's the ssa.BasicBlockID.
|
|
60
|
+
labelPositionPool wazevoapi.IDedPool[labelPosition]
|
|
61
|
+
// nextLabel is the next label to be allocated. The first free label comes after maxSSABlockID
|
|
62
|
+
// so that we can have an identical label for the SSA block ID, which is useful for debugging.
|
|
63
|
+
nextLabel label
|
|
64
|
+
// rootInstr is the first instruction of the function.
|
|
65
|
+
rootInstr *instruction
|
|
66
|
+
// currentLabelPos is the currently-compiled ssa.BasicBlock's labelPosition.
|
|
67
|
+
currentLabelPos *labelPosition
|
|
68
|
+
// orderedSSABlockLabelPos is the ordered list of labelPosition in the generated code for each ssa.BasicBlock.
|
|
69
|
+
orderedSSABlockLabelPos []*labelPosition
|
|
70
|
+
// returnLabelPos is the labelPosition for the return block.
|
|
71
|
+
returnLabelPos labelPosition
|
|
72
|
+
// perBlockHead and perBlockEnd are the head and tail of the instruction list per currently-compiled ssa.BasicBlock.
|
|
73
|
+
perBlockHead, perBlockEnd *instruction
|
|
74
|
+
// pendingInstructions are the instructions which are not yet emitted into the instruction list.
|
|
75
|
+
pendingInstructions []*instruction
|
|
76
|
+
// maxSSABlockID is the maximum ssa.BasicBlockID in the current function.
|
|
77
|
+
maxSSABlockID label
|
|
78
|
+
|
|
79
|
+
spillSlotSize int64
|
|
80
|
+
spillSlots map[regalloc.VRegID]int64
|
|
81
|
+
currentABI *backend.FunctionABI
|
|
82
|
+
clobberedRegs []regalloc.VReg
|
|
83
|
+
|
|
84
|
+
maxRequiredStackSizeForCalls int64
|
|
85
|
+
|
|
86
|
+
labelResolutionPends []labelResolutionPend
|
|
87
|
+
|
|
88
|
+
// jmpTableTargets holds the labels of the jump table targets.
|
|
89
|
+
jmpTableTargets [][]uint32
|
|
90
|
+
// jmpTableTargetNext is the index to the jmpTableTargets slice to be used for the next jump table.
|
|
91
|
+
jmpTableTargetsNext int
|
|
92
|
+
consts []_const
|
|
93
|
+
|
|
94
|
+
constSwizzleMaskConstIndex, constSqmulRoundSatIndex,
|
|
95
|
+
constI8x16SHLMaskTableIndex, constI8x16LogicalSHRMaskTableIndex,
|
|
96
|
+
constF64x2CvtFromIMaskIndex, constTwop52Index,
|
|
97
|
+
constI32sMaxOnF64x2Index, constI32uMaxOnF64x2Index,
|
|
98
|
+
constAllOnesI8x16Index, constAllOnesI16x8Index,
|
|
99
|
+
constExtAddPairwiseI16x8uMask1Index, constExtAddPairwiseI16x8uMask2Index int
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
_const struct {
|
|
103
|
+
lo, hi uint64
|
|
104
|
+
_var []byte
|
|
105
|
+
label label
|
|
106
|
+
labelPos *labelPosition
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
labelResolutionPend struct {
|
|
110
|
+
instr *instruction
|
|
111
|
+
instrOffset int64
|
|
112
|
+
// imm32Offset is the offset of the last 4 bytes of the instruction.
|
|
113
|
+
imm32Offset int64
|
|
114
|
+
}
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
type (
|
|
118
|
+
// label represents a position in the generated code which is either
|
|
119
|
+
// a real instruction or the constant InstructionPool (e.g. jump tables).
|
|
120
|
+
//
|
|
121
|
+
// This is exactly the same as the traditional "label" in assembly code.
|
|
122
|
+
label uint32
|
|
123
|
+
|
|
124
|
+
// labelPosition represents the regions of the generated code which the label represents.
|
|
125
|
+
// This implements regalloc.Block.
|
|
126
|
+
labelPosition struct {
|
|
127
|
+
// sb is not nil if this corresponds to a ssa.BasicBlock.
|
|
128
|
+
sb ssa.BasicBlock
|
|
129
|
+
// cur is used to walk through the instructions in the block during the register allocation.
|
|
130
|
+
cur,
|
|
131
|
+
// begin and end are the first and last instructions of the block.
|
|
132
|
+
begin, end *instruction
|
|
133
|
+
// binaryOffset is the offset in the binary where the label is located.
|
|
134
|
+
binaryOffset int64
|
|
135
|
+
}
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
// String implements backend.Machine.
|
|
139
|
+
func (l label) String() string {
|
|
140
|
+
return fmt.Sprintf("L%d", l)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
func resetLabelPosition(l *labelPosition) {
|
|
144
|
+
*l = labelPosition{}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
const labelReturn = math.MaxUint32
|
|
148
|
+
|
|
149
|
+
func ssaBlockLabel(sb ssa.BasicBlock) label {
|
|
150
|
+
if sb.ReturnBlock() {
|
|
151
|
+
return labelReturn
|
|
152
|
+
}
|
|
153
|
+
return label(sb.ID())
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
// getOrAllocateSSABlockLabelPosition returns the labelPosition for the given basic block.
|
|
157
|
+
func (m *machine) getOrAllocateSSABlockLabelPosition(sb ssa.BasicBlock) *labelPosition {
|
|
158
|
+
if sb.ReturnBlock() {
|
|
159
|
+
m.returnLabelPos.sb = sb
|
|
160
|
+
return &m.returnLabelPos
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
l := ssaBlockLabel(sb)
|
|
164
|
+
pos := m.labelPositionPool.GetOrAllocate(int(l))
|
|
165
|
+
pos.sb = sb
|
|
166
|
+
return pos
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
func (m *machine) getOrAllocateConstLabel(i *int, _var []byte) label {
|
|
170
|
+
index := *i
|
|
171
|
+
if index == -1 {
|
|
172
|
+
l, pos := m.allocateLabel()
|
|
173
|
+
index = len(m.consts)
|
|
174
|
+
m.consts = append(m.consts, _const{
|
|
175
|
+
_var: _var,
|
|
176
|
+
label: l,
|
|
177
|
+
labelPos: pos,
|
|
178
|
+
})
|
|
179
|
+
*i = index
|
|
180
|
+
}
|
|
181
|
+
return m.consts[index].label
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
// Reset implements backend.Machine.
|
|
185
|
+
func (m *machine) Reset() {
|
|
186
|
+
m.consts = m.consts[:0]
|
|
187
|
+
m.clobberedRegs = m.clobberedRegs[:0]
|
|
188
|
+
for key := range m.spillSlots {
|
|
189
|
+
m.clobberedRegs = append(m.clobberedRegs, regalloc.VReg(key))
|
|
190
|
+
}
|
|
191
|
+
for _, key := range m.clobberedRegs {
|
|
192
|
+
delete(m.spillSlots, regalloc.VRegID(key))
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
m.stackBoundsCheckDisabled = false
|
|
196
|
+
m.regAlloc.Reset()
|
|
197
|
+
m.labelPositionPool.Reset()
|
|
198
|
+
m.instrPool.Reset()
|
|
199
|
+
m.regAllocStarted = false
|
|
200
|
+
m.clobberedRegs = m.clobberedRegs[:0]
|
|
201
|
+
|
|
202
|
+
m.spillSlotSize = 0
|
|
203
|
+
m.maxRequiredStackSizeForCalls = 0
|
|
204
|
+
m.perBlockHead, m.perBlockEnd, m.rootInstr = nil, nil, nil
|
|
205
|
+
m.pendingInstructions = m.pendingInstructions[:0]
|
|
206
|
+
m.orderedSSABlockLabelPos = m.orderedSSABlockLabelPos[:0]
|
|
207
|
+
|
|
208
|
+
m.amodePool.Reset()
|
|
209
|
+
m.jmpTableTargetsNext = 0
|
|
210
|
+
m.constSwizzleMaskConstIndex = -1
|
|
211
|
+
m.constSqmulRoundSatIndex = -1
|
|
212
|
+
m.constI8x16SHLMaskTableIndex = -1
|
|
213
|
+
m.constI8x16LogicalSHRMaskTableIndex = -1
|
|
214
|
+
m.constF64x2CvtFromIMaskIndex = -1
|
|
215
|
+
m.constTwop52Index = -1
|
|
216
|
+
m.constI32sMaxOnF64x2Index = -1
|
|
217
|
+
m.constI32uMaxOnF64x2Index = -1
|
|
218
|
+
m.constAllOnesI8x16Index = -1
|
|
219
|
+
m.constAllOnesI16x8Index = -1
|
|
220
|
+
m.constExtAddPairwiseI16x8uMask1Index = -1
|
|
221
|
+
m.constExtAddPairwiseI16x8uMask2Index = -1
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
// StartLoweringFunction implements backend.Machine StartLoweringFunction.
|
|
225
|
+
func (m *machine) StartLoweringFunction(maxBlockID ssa.BasicBlockID) {
|
|
226
|
+
m.maxSSABlockID = label(maxBlockID)
|
|
227
|
+
m.nextLabel = label(maxBlockID) + 1
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
// LinkAdjacentBlocks implements backend.Machine.
|
|
231
|
+
func (m *machine) LinkAdjacentBlocks(prev, next ssa.BasicBlock) {
|
|
232
|
+
prevPos, nextPos := m.getOrAllocateSSABlockLabelPosition(prev), m.getOrAllocateSSABlockLabelPosition(next)
|
|
233
|
+
prevPos.end.next = nextPos.begin
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// StartBlock implements backend.Machine.
|
|
237
|
+
func (m *machine) StartBlock(blk ssa.BasicBlock) {
|
|
238
|
+
m.currentLabelPos = m.getOrAllocateSSABlockLabelPosition(blk)
|
|
239
|
+
labelPos := m.currentLabelPos
|
|
240
|
+
end := m.allocateNop()
|
|
241
|
+
m.perBlockHead, m.perBlockEnd = end, end
|
|
242
|
+
labelPos.begin, labelPos.end = end, end
|
|
243
|
+
m.orderedSSABlockLabelPos = append(m.orderedSSABlockLabelPos, labelPos)
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// EndBlock implements ExecutableContext.
|
|
247
|
+
func (m *machine) EndBlock() {
|
|
248
|
+
// Insert nop0 as the head of the block for convenience to simplify the logic of inserting instructions.
|
|
249
|
+
m.insertAtPerBlockHead(m.allocateNop())
|
|
250
|
+
|
|
251
|
+
m.currentLabelPos.begin = m.perBlockHead
|
|
252
|
+
|
|
253
|
+
if m.currentLabelPos.sb.EntryBlock() {
|
|
254
|
+
m.rootInstr = m.perBlockHead
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
func (m *machine) insertAtPerBlockHead(i *instruction) {
|
|
259
|
+
if m.perBlockHead == nil {
|
|
260
|
+
m.perBlockHead = i
|
|
261
|
+
m.perBlockEnd = i
|
|
262
|
+
return
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
i.next = m.perBlockHead
|
|
266
|
+
m.perBlockHead.prev = i
|
|
267
|
+
m.perBlockHead = i
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// FlushPendingInstructions implements backend.Machine.
|
|
271
|
+
func (m *machine) FlushPendingInstructions() {
|
|
272
|
+
l := len(m.pendingInstructions)
|
|
273
|
+
if l == 0 {
|
|
274
|
+
return
|
|
275
|
+
}
|
|
276
|
+
for i := l - 1; i >= 0; i-- { // reverse because we lower instructions in reverse order.
|
|
277
|
+
m.insertAtPerBlockHead(m.pendingInstructions[i])
|
|
278
|
+
}
|
|
279
|
+
m.pendingInstructions = m.pendingInstructions[:0]
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// DisableStackCheck implements backend.Machine.
|
|
283
|
+
func (m *machine) DisableStackCheck() { m.stackBoundsCheckDisabled = true }
|
|
284
|
+
|
|
285
|
+
// SetCompiler implements backend.Machine.
|
|
286
|
+
func (m *machine) SetCompiler(c backend.Compiler) {
|
|
287
|
+
m.c = c
|
|
288
|
+
m.regAllocFn.ssaB = c.SSABuilder()
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// SetCurrentABI implements backend.Machine.
|
|
292
|
+
func (m *machine) SetCurrentABI(abi *backend.FunctionABI) { m.currentABI = abi }
|
|
293
|
+
|
|
294
|
+
// RegAlloc implements backend.Machine.
|
|
295
|
+
func (m *machine) RegAlloc() {
|
|
296
|
+
rf := m.regAllocFn
|
|
297
|
+
m.regAllocStarted = true
|
|
298
|
+
m.regAlloc.DoAllocation(&rf)
|
|
299
|
+
// Now that we know the final spill slot size, we must align spillSlotSize to 16 bytes.
|
|
300
|
+
m.spillSlotSize = (m.spillSlotSize + 15) &^ 15
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// InsertReturn implements backend.Machine.
|
|
304
|
+
func (m *machine) InsertReturn() {
|
|
305
|
+
i := m.allocateInstr().asRet()
|
|
306
|
+
m.insert(i)
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
// LowerSingleBranch implements backend.Machine.
|
|
310
|
+
func (m *machine) LowerSingleBranch(b *ssa.Instruction) {
|
|
311
|
+
switch b.Opcode() {
|
|
312
|
+
case ssa.OpcodeJump:
|
|
313
|
+
_, _, targetBlkID := b.BranchData()
|
|
314
|
+
if b.IsFallthroughJump() {
|
|
315
|
+
return
|
|
316
|
+
}
|
|
317
|
+
jmp := m.allocateInstr()
|
|
318
|
+
target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
|
|
319
|
+
if target == labelReturn {
|
|
320
|
+
jmp.asRet()
|
|
321
|
+
} else {
|
|
322
|
+
jmp.asJmp(newOperandLabel(target))
|
|
323
|
+
}
|
|
324
|
+
m.insert(jmp)
|
|
325
|
+
case ssa.OpcodeBrTable:
|
|
326
|
+
index, targetBlkIDs := b.BrTableData()
|
|
327
|
+
m.lowerBrTable(index, targetBlkIDs)
|
|
328
|
+
default:
|
|
329
|
+
panic("BUG: unexpected branch opcode" + b.Opcode().String())
|
|
330
|
+
}
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
func (m *machine) addJmpTableTarget(targets ssa.Values) (index int) {
|
|
334
|
+
if m.jmpTableTargetsNext == len(m.jmpTableTargets) {
|
|
335
|
+
m.jmpTableTargets = append(m.jmpTableTargets, make([]uint32, 0, len(targets.View())))
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
index = m.jmpTableTargetsNext
|
|
339
|
+
m.jmpTableTargetsNext++
|
|
340
|
+
m.jmpTableTargets[index] = m.jmpTableTargets[index][:0]
|
|
341
|
+
for _, targetBlockID := range targets.View() {
|
|
342
|
+
target := m.c.SSABuilder().BasicBlock(ssa.BasicBlockID(targetBlockID))
|
|
343
|
+
m.jmpTableTargets[index] = append(m.jmpTableTargets[index], uint32(ssaBlockLabel(target)))
|
|
344
|
+
}
|
|
345
|
+
return
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
var condBranchMatches = [...]ssa.Opcode{ssa.OpcodeIcmp, ssa.OpcodeFcmp}
|
|
349
|
+
|
|
350
|
+
func (m *machine) lowerBrTable(index ssa.Value, targets ssa.Values) {
|
|
351
|
+
_v := m.getOperand_Reg(m.c.ValueDefinition(index))
|
|
352
|
+
v := m.copyToTmp(_v.reg())
|
|
353
|
+
|
|
354
|
+
targetCount := len(targets.View())
|
|
355
|
+
|
|
356
|
+
// First, we need to do the bounds check.
|
|
357
|
+
maxIndex := m.c.AllocateVReg(ssa.TypeI32)
|
|
358
|
+
m.lowerIconst(maxIndex, uint64(targetCount-1), false)
|
|
359
|
+
cmp := m.allocateInstr().asCmpRmiR(true, newOperandReg(maxIndex), v, false)
|
|
360
|
+
m.insert(cmp)
|
|
361
|
+
|
|
362
|
+
// Then do the conditional move maxIndex to v if v > maxIndex.
|
|
363
|
+
cmov := m.allocateInstr().asCmove(condNB, newOperandReg(maxIndex), v, false)
|
|
364
|
+
m.insert(cmov)
|
|
365
|
+
|
|
366
|
+
// Now that v has the correct index. Load the address of the jump table into the addr.
|
|
367
|
+
addr := m.c.AllocateVReg(ssa.TypeI64)
|
|
368
|
+
leaJmpTableAddr := m.allocateInstr()
|
|
369
|
+
m.insert(leaJmpTableAddr)
|
|
370
|
+
|
|
371
|
+
// Then add the target's offset into jmpTableAddr.
|
|
372
|
+
loadTargetOffsetFromJmpTable := m.allocateInstr().asAluRmiR(aluRmiROpcodeAdd,
|
|
373
|
+
// Shift by 3 because each entry is 8 bytes.
|
|
374
|
+
newOperandMem(m.newAmodeRegRegShift(0, addr, v, 3)), addr, true)
|
|
375
|
+
m.insert(loadTargetOffsetFromJmpTable)
|
|
376
|
+
|
|
377
|
+
// Now ready to jump.
|
|
378
|
+
jmp := m.allocateInstr().asJmp(newOperandReg(addr))
|
|
379
|
+
m.insert(jmp)
|
|
380
|
+
|
|
381
|
+
jmpTableBegin, jmpTableBeginLabel := m.allocateBrTarget()
|
|
382
|
+
m.insert(jmpTableBegin)
|
|
383
|
+
leaJmpTableAddr.asLEA(newOperandLabel(jmpTableBeginLabel), addr)
|
|
384
|
+
|
|
385
|
+
jmpTable := m.allocateInstr()
|
|
386
|
+
targetSliceIndex := m.addJmpTableTarget(targets)
|
|
387
|
+
jmpTable.asJmpTableSequence(targetSliceIndex, targetCount)
|
|
388
|
+
m.insert(jmpTable)
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// LowerConditionalBranch implements backend.Machine.
|
|
392
|
+
func (m *machine) LowerConditionalBranch(b *ssa.Instruction) {
|
|
393
|
+
cval, args, targetBlkID := b.BranchData()
|
|
394
|
+
if len(args) > 0 {
|
|
395
|
+
panic(fmt.Sprintf(
|
|
396
|
+
"conditional branch shouldn't have args; likely a bug in critical edge splitting: from %s to %s",
|
|
397
|
+
m.currentLabelPos.sb,
|
|
398
|
+
targetBlkID,
|
|
399
|
+
))
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
target := ssaBlockLabel(m.c.SSABuilder().BasicBlock(targetBlkID))
|
|
403
|
+
cvalDef := m.c.ValueDefinition(cval)
|
|
404
|
+
|
|
405
|
+
switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
|
|
406
|
+
case ssa.OpcodeIcmp:
|
|
407
|
+
cvalInstr := cvalDef.Instr
|
|
408
|
+
x, y, c := cvalInstr.IcmpData()
|
|
409
|
+
|
|
410
|
+
cc := condFromSSAIntCmpCond(c)
|
|
411
|
+
if b.Opcode() == ssa.OpcodeBrz {
|
|
412
|
+
cc = cc.invert()
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
// First, perform the comparison and set the flag.
|
|
416
|
+
xd, yd := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
417
|
+
if !m.tryLowerBandToFlag(xd, yd) {
|
|
418
|
+
m.lowerIcmpToFlag(xd, yd, x.Type() == ssa.TypeI64)
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
// Then perform the conditional branch.
|
|
422
|
+
m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
|
|
423
|
+
cvalDef.Instr.MarkLowered()
|
|
424
|
+
case ssa.OpcodeFcmp:
|
|
425
|
+
cvalInstr := cvalDef.Instr
|
|
426
|
+
|
|
427
|
+
f1, f2, and := m.lowerFcmpToFlags(cvalInstr)
|
|
428
|
+
isBrz := b.Opcode() == ssa.OpcodeBrz
|
|
429
|
+
if isBrz {
|
|
430
|
+
f1 = f1.invert()
|
|
431
|
+
}
|
|
432
|
+
if f2 == condInvalid {
|
|
433
|
+
m.insert(m.allocateInstr().asJmpIf(f1, newOperandLabel(target)))
|
|
434
|
+
} else {
|
|
435
|
+
if isBrz {
|
|
436
|
+
f2 = f2.invert()
|
|
437
|
+
and = !and
|
|
438
|
+
}
|
|
439
|
+
jmp1, jmp2 := m.allocateInstr(), m.allocateInstr()
|
|
440
|
+
m.insert(jmp1)
|
|
441
|
+
m.insert(jmp2)
|
|
442
|
+
notTaken, notTakenLabel := m.allocateBrTarget()
|
|
443
|
+
m.insert(notTaken)
|
|
444
|
+
if and {
|
|
445
|
+
jmp1.asJmpIf(f1.invert(), newOperandLabel(notTakenLabel))
|
|
446
|
+
jmp2.asJmpIf(f2, newOperandLabel(target))
|
|
447
|
+
} else {
|
|
448
|
+
jmp1.asJmpIf(f1, newOperandLabel(target))
|
|
449
|
+
jmp2.asJmpIf(f2, newOperandLabel(target))
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
cvalDef.Instr.MarkLowered()
|
|
454
|
+
default:
|
|
455
|
+
v := m.getOperand_Reg(cvalDef)
|
|
456
|
+
|
|
457
|
+
var cc cond
|
|
458
|
+
if b.Opcode() == ssa.OpcodeBrz {
|
|
459
|
+
cc = condZ
|
|
460
|
+
} else {
|
|
461
|
+
cc = condNZ
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// Perform test %v, %v to set the flag.
|
|
465
|
+
cmp := m.allocateInstr().asCmpRmiR(false, v, v.reg(), false)
|
|
466
|
+
m.insert(cmp)
|
|
467
|
+
m.insert(m.allocateInstr().asJmpIf(cc, newOperandLabel(target)))
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// LowerInstr implements backend.Machine.
|
|
472
|
+
func (m *machine) LowerInstr(instr *ssa.Instruction) {
|
|
473
|
+
if l := instr.SourceOffset(); l.Valid() {
|
|
474
|
+
info := m.allocateInstr().asEmitSourceOffsetInfo(l)
|
|
475
|
+
m.insert(info)
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
switch op := instr.Opcode(); op {
|
|
479
|
+
case ssa.OpcodeBrz, ssa.OpcodeBrnz, ssa.OpcodeJump, ssa.OpcodeBrTable:
|
|
480
|
+
panic("BUG: branching instructions are handled by LowerBranches")
|
|
481
|
+
case ssa.OpcodeReturn:
|
|
482
|
+
panic("BUG: return must be handled by backend.Compiler")
|
|
483
|
+
case ssa.OpcodeIconst, ssa.OpcodeF32const, ssa.OpcodeF64const: // Constant instructions are inlined.
|
|
484
|
+
case ssa.OpcodeCall, ssa.OpcodeCallIndirect:
|
|
485
|
+
m.lowerCall(instr)
|
|
486
|
+
case ssa.OpcodeStore, ssa.OpcodeIstore8, ssa.OpcodeIstore16, ssa.OpcodeIstore32:
|
|
487
|
+
m.lowerStore(instr)
|
|
488
|
+
case ssa.OpcodeIadd:
|
|
489
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeAdd)
|
|
490
|
+
case ssa.OpcodeIsub:
|
|
491
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeSub)
|
|
492
|
+
case ssa.OpcodeImul:
|
|
493
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeMul)
|
|
494
|
+
case ssa.OpcodeSdiv, ssa.OpcodeUdiv, ssa.OpcodeSrem, ssa.OpcodeUrem:
|
|
495
|
+
isDiv := op == ssa.OpcodeSdiv || op == ssa.OpcodeUdiv
|
|
496
|
+
isSigned := op == ssa.OpcodeSdiv || op == ssa.OpcodeSrem
|
|
497
|
+
m.lowerIDivRem(instr, isDiv, isSigned)
|
|
498
|
+
case ssa.OpcodeBand:
|
|
499
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeAnd)
|
|
500
|
+
case ssa.OpcodeBor:
|
|
501
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeOr)
|
|
502
|
+
case ssa.OpcodeBxor:
|
|
503
|
+
m.lowerAluRmiROp(instr, aluRmiROpcodeXor)
|
|
504
|
+
case ssa.OpcodeIshl:
|
|
505
|
+
m.lowerShiftR(instr, shiftROpShiftLeft)
|
|
506
|
+
case ssa.OpcodeSshr:
|
|
507
|
+
m.lowerShiftR(instr, shiftROpShiftRightArithmetic)
|
|
508
|
+
case ssa.OpcodeUshr:
|
|
509
|
+
m.lowerShiftR(instr, shiftROpShiftRightLogical)
|
|
510
|
+
case ssa.OpcodeRotl:
|
|
511
|
+
m.lowerShiftR(instr, shiftROpRotateLeft)
|
|
512
|
+
case ssa.OpcodeRotr:
|
|
513
|
+
m.lowerShiftR(instr, shiftROpRotateRight)
|
|
514
|
+
case ssa.OpcodeClz:
|
|
515
|
+
m.lowerClz(instr)
|
|
516
|
+
case ssa.OpcodeCtz:
|
|
517
|
+
m.lowerCtz(instr)
|
|
518
|
+
case ssa.OpcodePopcnt:
|
|
519
|
+
m.lowerUnaryRmR(instr, unaryRmROpcodePopcnt)
|
|
520
|
+
case ssa.OpcodeFadd, ssa.OpcodeFsub, ssa.OpcodeFmul, ssa.OpcodeFdiv:
|
|
521
|
+
m.lowerXmmRmR(instr)
|
|
522
|
+
case ssa.OpcodeFabs:
|
|
523
|
+
m.lowerFabsFneg(instr)
|
|
524
|
+
case ssa.OpcodeFneg:
|
|
525
|
+
m.lowerFabsFneg(instr)
|
|
526
|
+
case ssa.OpcodeCeil:
|
|
527
|
+
m.lowerRound(instr, roundingModeUp)
|
|
528
|
+
case ssa.OpcodeFloor:
|
|
529
|
+
m.lowerRound(instr, roundingModeDown)
|
|
530
|
+
case ssa.OpcodeTrunc:
|
|
531
|
+
m.lowerRound(instr, roundingModeZero)
|
|
532
|
+
case ssa.OpcodeNearest:
|
|
533
|
+
m.lowerRound(instr, roundingModeNearest)
|
|
534
|
+
case ssa.OpcodeFmin, ssa.OpcodeFmax:
|
|
535
|
+
m.lowerFminFmax(instr)
|
|
536
|
+
case ssa.OpcodeFcopysign:
|
|
537
|
+
m.lowerFcopysign(instr)
|
|
538
|
+
case ssa.OpcodeBitcast:
|
|
539
|
+
m.lowerBitcast(instr)
|
|
540
|
+
case ssa.OpcodeSqrt:
|
|
541
|
+
m.lowerSqrt(instr)
|
|
542
|
+
case ssa.OpcodeFpromote:
|
|
543
|
+
v := instr.Arg()
|
|
544
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(v))
|
|
545
|
+
rd := m.c.VRegOf(instr.Return())
|
|
546
|
+
cnt := m.allocateInstr()
|
|
547
|
+
cnt.asXmmUnaryRmR(sseOpcodeCvtss2sd, rn, rd)
|
|
548
|
+
m.insert(cnt)
|
|
549
|
+
case ssa.OpcodeFdemote:
|
|
550
|
+
v := instr.Arg()
|
|
551
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(v))
|
|
552
|
+
rd := m.c.VRegOf(instr.Return())
|
|
553
|
+
cnt := m.allocateInstr()
|
|
554
|
+
cnt.asXmmUnaryRmR(sseOpcodeCvtsd2ss, rn, rd)
|
|
555
|
+
m.insert(cnt)
|
|
556
|
+
case ssa.OpcodeFcvtToSint, ssa.OpcodeFcvtToSintSat:
|
|
557
|
+
x, ctx := instr.Arg2()
|
|
558
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
559
|
+
rd := m.c.VRegOf(instr.Return())
|
|
560
|
+
ctxVReg := m.c.VRegOf(ctx)
|
|
561
|
+
m.lowerFcvtToSint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
|
|
562
|
+
instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToSintSat)
|
|
563
|
+
case ssa.OpcodeFcvtToUint, ssa.OpcodeFcvtToUintSat:
|
|
564
|
+
x, ctx := instr.Arg2()
|
|
565
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
566
|
+
rd := m.c.VRegOf(instr.Return())
|
|
567
|
+
ctxVReg := m.c.VRegOf(ctx)
|
|
568
|
+
m.lowerFcvtToUint(ctxVReg, rn.reg(), rd, x.Type() == ssa.TypeF64,
|
|
569
|
+
instr.Return().Type().Bits() == 64, op == ssa.OpcodeFcvtToUintSat)
|
|
570
|
+
case ssa.OpcodeFcvtFromSint:
|
|
571
|
+
x := instr.Arg()
|
|
572
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
573
|
+
rd := newOperandReg(m.c.VRegOf(instr.Return()))
|
|
574
|
+
m.lowerFcvtFromSint(rn, rd,
|
|
575
|
+
x.Type() == ssa.TypeI64, instr.Return().Type().Bits() == 64)
|
|
576
|
+
case ssa.OpcodeFcvtFromUint:
|
|
577
|
+
x := instr.Arg()
|
|
578
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
579
|
+
rd := newOperandReg(m.c.VRegOf(instr.Return()))
|
|
580
|
+
m.lowerFcvtFromUint(rn, rd, x.Type() == ssa.TypeI64,
|
|
581
|
+
instr.Return().Type().Bits() == 64)
|
|
582
|
+
case ssa.OpcodeVanyTrue:
|
|
583
|
+
m.lowerVanyTrue(instr)
|
|
584
|
+
case ssa.OpcodeVallTrue:
|
|
585
|
+
m.lowerVallTrue(instr)
|
|
586
|
+
case ssa.OpcodeVhighBits:
|
|
587
|
+
m.lowerVhighBits(instr)
|
|
588
|
+
case ssa.OpcodeVbnot:
|
|
589
|
+
m.lowerVbnot(instr)
|
|
590
|
+
case ssa.OpcodeVband:
|
|
591
|
+
x, y := instr.Arg2()
|
|
592
|
+
m.lowerVbBinOp(sseOpcodePand, x, y, instr.Return())
|
|
593
|
+
case ssa.OpcodeVbor:
|
|
594
|
+
x, y := instr.Arg2()
|
|
595
|
+
m.lowerVbBinOp(sseOpcodePor, x, y, instr.Return())
|
|
596
|
+
case ssa.OpcodeVbxor:
|
|
597
|
+
x, y := instr.Arg2()
|
|
598
|
+
m.lowerVbBinOp(sseOpcodePxor, x, y, instr.Return())
|
|
599
|
+
case ssa.OpcodeVbandnot:
|
|
600
|
+
m.lowerVbandnot(instr, sseOpcodePandn)
|
|
601
|
+
case ssa.OpcodeVbitselect:
|
|
602
|
+
m.lowerVbitselect(instr)
|
|
603
|
+
case ssa.OpcodeVIadd:
|
|
604
|
+
x, y, lane := instr.Arg2WithLane()
|
|
605
|
+
var vecOp sseOpcode
|
|
606
|
+
switch lane {
|
|
607
|
+
case ssa.VecLaneI8x16:
|
|
608
|
+
vecOp = sseOpcodePaddb
|
|
609
|
+
case ssa.VecLaneI16x8:
|
|
610
|
+
vecOp = sseOpcodePaddw
|
|
611
|
+
case ssa.VecLaneI32x4:
|
|
612
|
+
vecOp = sseOpcodePaddd
|
|
613
|
+
case ssa.VecLaneI64x2:
|
|
614
|
+
vecOp = sseOpcodePaddq
|
|
615
|
+
}
|
|
616
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
617
|
+
|
|
618
|
+
case ssa.OpcodeVSaddSat:
|
|
619
|
+
x, y, lane := instr.Arg2WithLane()
|
|
620
|
+
var vecOp sseOpcode
|
|
621
|
+
switch lane {
|
|
622
|
+
case ssa.VecLaneI8x16:
|
|
623
|
+
vecOp = sseOpcodePaddsb
|
|
624
|
+
case ssa.VecLaneI16x8:
|
|
625
|
+
vecOp = sseOpcodePaddsw
|
|
626
|
+
}
|
|
627
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
628
|
+
|
|
629
|
+
case ssa.OpcodeVUaddSat:
|
|
630
|
+
x, y, lane := instr.Arg2WithLane()
|
|
631
|
+
var vecOp sseOpcode
|
|
632
|
+
switch lane {
|
|
633
|
+
case ssa.VecLaneI8x16:
|
|
634
|
+
vecOp = sseOpcodePaddusb
|
|
635
|
+
case ssa.VecLaneI16x8:
|
|
636
|
+
vecOp = sseOpcodePaddusw
|
|
637
|
+
}
|
|
638
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
639
|
+
|
|
640
|
+
case ssa.OpcodeVIsub:
|
|
641
|
+
x, y, lane := instr.Arg2WithLane()
|
|
642
|
+
var vecOp sseOpcode
|
|
643
|
+
switch lane {
|
|
644
|
+
case ssa.VecLaneI8x16:
|
|
645
|
+
vecOp = sseOpcodePsubb
|
|
646
|
+
case ssa.VecLaneI16x8:
|
|
647
|
+
vecOp = sseOpcodePsubw
|
|
648
|
+
case ssa.VecLaneI32x4:
|
|
649
|
+
vecOp = sseOpcodePsubd
|
|
650
|
+
case ssa.VecLaneI64x2:
|
|
651
|
+
vecOp = sseOpcodePsubq
|
|
652
|
+
}
|
|
653
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
654
|
+
|
|
655
|
+
case ssa.OpcodeVSsubSat:
|
|
656
|
+
x, y, lane := instr.Arg2WithLane()
|
|
657
|
+
var vecOp sseOpcode
|
|
658
|
+
switch lane {
|
|
659
|
+
case ssa.VecLaneI8x16:
|
|
660
|
+
vecOp = sseOpcodePsubsb
|
|
661
|
+
case ssa.VecLaneI16x8:
|
|
662
|
+
vecOp = sseOpcodePsubsw
|
|
663
|
+
}
|
|
664
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
665
|
+
|
|
666
|
+
case ssa.OpcodeVUsubSat:
|
|
667
|
+
x, y, lane := instr.Arg2WithLane()
|
|
668
|
+
var vecOp sseOpcode
|
|
669
|
+
switch lane {
|
|
670
|
+
case ssa.VecLaneI8x16:
|
|
671
|
+
vecOp = sseOpcodePsubusb
|
|
672
|
+
case ssa.VecLaneI16x8:
|
|
673
|
+
vecOp = sseOpcodePsubusw
|
|
674
|
+
}
|
|
675
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
676
|
+
|
|
677
|
+
case ssa.OpcodeVImul:
|
|
678
|
+
m.lowerVImul(instr)
|
|
679
|
+
case ssa.OpcodeVIneg:
|
|
680
|
+
x, lane := instr.ArgWithLane()
|
|
681
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
682
|
+
rd := m.c.VRegOf(instr.Return())
|
|
683
|
+
var vecOp sseOpcode
|
|
684
|
+
switch lane {
|
|
685
|
+
case ssa.VecLaneI8x16:
|
|
686
|
+
vecOp = sseOpcodePsubb
|
|
687
|
+
case ssa.VecLaneI16x8:
|
|
688
|
+
vecOp = sseOpcodePsubw
|
|
689
|
+
case ssa.VecLaneI32x4:
|
|
690
|
+
vecOp = sseOpcodePsubd
|
|
691
|
+
case ssa.VecLaneI64x2:
|
|
692
|
+
vecOp = sseOpcodePsubq
|
|
693
|
+
default:
|
|
694
|
+
panic("BUG")
|
|
695
|
+
}
|
|
696
|
+
|
|
697
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
698
|
+
m.insert(m.allocateInstr().asZeros(tmp))
|
|
699
|
+
|
|
700
|
+
i := m.allocateInstr()
|
|
701
|
+
i.asXmmRmR(vecOp, rn, tmp)
|
|
702
|
+
m.insert(i)
|
|
703
|
+
|
|
704
|
+
m.copyTo(tmp, rd)
|
|
705
|
+
case ssa.OpcodeVFadd:
|
|
706
|
+
x, y, lane := instr.Arg2WithLane()
|
|
707
|
+
var vecOp sseOpcode
|
|
708
|
+
switch lane {
|
|
709
|
+
case ssa.VecLaneF32x4:
|
|
710
|
+
vecOp = sseOpcodeAddps
|
|
711
|
+
case ssa.VecLaneF64x2:
|
|
712
|
+
vecOp = sseOpcodeAddpd
|
|
713
|
+
}
|
|
714
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
715
|
+
|
|
716
|
+
case ssa.OpcodeVFsub:
|
|
717
|
+
x, y, lane := instr.Arg2WithLane()
|
|
718
|
+
var vecOp sseOpcode
|
|
719
|
+
switch lane {
|
|
720
|
+
case ssa.VecLaneF32x4:
|
|
721
|
+
vecOp = sseOpcodeSubps
|
|
722
|
+
case ssa.VecLaneF64x2:
|
|
723
|
+
vecOp = sseOpcodeSubpd
|
|
724
|
+
}
|
|
725
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
726
|
+
|
|
727
|
+
case ssa.OpcodeVFdiv:
|
|
728
|
+
x, y, lane := instr.Arg2WithLane()
|
|
729
|
+
var vecOp sseOpcode
|
|
730
|
+
switch lane {
|
|
731
|
+
case ssa.VecLaneF32x4:
|
|
732
|
+
vecOp = sseOpcodeDivps
|
|
733
|
+
case ssa.VecLaneF64x2:
|
|
734
|
+
vecOp = sseOpcodeDivpd
|
|
735
|
+
}
|
|
736
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
737
|
+
|
|
738
|
+
case ssa.OpcodeVFmul:
|
|
739
|
+
x, y, lane := instr.Arg2WithLane()
|
|
740
|
+
var vecOp sseOpcode
|
|
741
|
+
switch lane {
|
|
742
|
+
case ssa.VecLaneF32x4:
|
|
743
|
+
vecOp = sseOpcodeMulps
|
|
744
|
+
case ssa.VecLaneF64x2:
|
|
745
|
+
vecOp = sseOpcodeMulpd
|
|
746
|
+
}
|
|
747
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
748
|
+
|
|
749
|
+
case ssa.OpcodeVFneg:
|
|
750
|
+
x, lane := instr.ArgWithLane()
|
|
751
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
752
|
+
rd := m.c.VRegOf(instr.Return())
|
|
753
|
+
|
|
754
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
755
|
+
|
|
756
|
+
var shiftOp, xorOp sseOpcode
|
|
757
|
+
var shiftAmt uint32
|
|
758
|
+
switch lane {
|
|
759
|
+
case ssa.VecLaneF32x4:
|
|
760
|
+
shiftOp, shiftAmt, xorOp = sseOpcodePslld, 31, sseOpcodeXorps
|
|
761
|
+
case ssa.VecLaneF64x2:
|
|
762
|
+
shiftOp, shiftAmt, xorOp = sseOpcodePsllq, 63, sseOpcodeXorpd
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
zero := m.allocateInstr()
|
|
766
|
+
zero.asZeros(tmp)
|
|
767
|
+
m.insert(zero)
|
|
768
|
+
|
|
769
|
+
// Set all bits on tmp by CMPPD with arg=0 (== pseudo CMPEQPD instruction).
|
|
770
|
+
// See https://www.felixcloutier.com/x86/cmpps
|
|
771
|
+
//
|
|
772
|
+
// Note: if we do not clear all the bits ^ with XORPS, this might end up not setting ones on some lane
|
|
773
|
+
// if the lane is NaN.
|
|
774
|
+
cmp := m.allocateInstr()
|
|
775
|
+
cmp.asXmmRmRImm(sseOpcodeCmppd, uint8(cmpPredEQ_UQ), newOperandReg(tmp), tmp)
|
|
776
|
+
m.insert(cmp)
|
|
777
|
+
|
|
778
|
+
// Do the left shift on each lane to set only the most significant bit in each.
|
|
779
|
+
i := m.allocateInstr()
|
|
780
|
+
i.asXmmRmiReg(shiftOp, newOperandImm32(shiftAmt), tmp)
|
|
781
|
+
m.insert(i)
|
|
782
|
+
|
|
783
|
+
// Get the negated result by XOR on each lane with tmp.
|
|
784
|
+
i = m.allocateInstr()
|
|
785
|
+
i.asXmmRmR(xorOp, rn, tmp)
|
|
786
|
+
m.insert(i)
|
|
787
|
+
|
|
788
|
+
m.copyTo(tmp, rd)
|
|
789
|
+
|
|
790
|
+
case ssa.OpcodeVSqrt:
|
|
791
|
+
x, lane := instr.ArgWithLane()
|
|
792
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
793
|
+
rd := m.c.VRegOf(instr.Return())
|
|
794
|
+
|
|
795
|
+
var vecOp sseOpcode
|
|
796
|
+
switch lane {
|
|
797
|
+
case ssa.VecLaneF32x4:
|
|
798
|
+
vecOp = sseOpcodeSqrtps
|
|
799
|
+
case ssa.VecLaneF64x2:
|
|
800
|
+
vecOp = sseOpcodeSqrtpd
|
|
801
|
+
}
|
|
802
|
+
i := m.allocateInstr()
|
|
803
|
+
i.asXmmUnaryRmR(vecOp, rn, rd)
|
|
804
|
+
m.insert(i)
|
|
805
|
+
|
|
806
|
+
case ssa.OpcodeVImin:
|
|
807
|
+
x, y, lane := instr.Arg2WithLane()
|
|
808
|
+
var vecOp sseOpcode
|
|
809
|
+
switch lane {
|
|
810
|
+
case ssa.VecLaneI8x16:
|
|
811
|
+
vecOp = sseOpcodePminsb
|
|
812
|
+
case ssa.VecLaneI16x8:
|
|
813
|
+
vecOp = sseOpcodePminsw
|
|
814
|
+
case ssa.VecLaneI32x4:
|
|
815
|
+
vecOp = sseOpcodePminsd
|
|
816
|
+
}
|
|
817
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
818
|
+
|
|
819
|
+
case ssa.OpcodeVUmin:
|
|
820
|
+
x, y, lane := instr.Arg2WithLane()
|
|
821
|
+
var vecOp sseOpcode
|
|
822
|
+
switch lane {
|
|
823
|
+
case ssa.VecLaneI8x16:
|
|
824
|
+
vecOp = sseOpcodePminub
|
|
825
|
+
case ssa.VecLaneI16x8:
|
|
826
|
+
vecOp = sseOpcodePminuw
|
|
827
|
+
case ssa.VecLaneI32x4:
|
|
828
|
+
vecOp = sseOpcodePminud
|
|
829
|
+
}
|
|
830
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
831
|
+
|
|
832
|
+
case ssa.OpcodeVImax:
|
|
833
|
+
x, y, lane := instr.Arg2WithLane()
|
|
834
|
+
var vecOp sseOpcode
|
|
835
|
+
switch lane {
|
|
836
|
+
case ssa.VecLaneI8x16:
|
|
837
|
+
vecOp = sseOpcodePmaxsb
|
|
838
|
+
case ssa.VecLaneI16x8:
|
|
839
|
+
vecOp = sseOpcodePmaxsw
|
|
840
|
+
case ssa.VecLaneI32x4:
|
|
841
|
+
vecOp = sseOpcodePmaxsd
|
|
842
|
+
}
|
|
843
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
844
|
+
|
|
845
|
+
case ssa.OpcodeVUmax:
|
|
846
|
+
x, y, lane := instr.Arg2WithLane()
|
|
847
|
+
var vecOp sseOpcode
|
|
848
|
+
switch lane {
|
|
849
|
+
case ssa.VecLaneI8x16:
|
|
850
|
+
vecOp = sseOpcodePmaxub
|
|
851
|
+
case ssa.VecLaneI16x8:
|
|
852
|
+
vecOp = sseOpcodePmaxuw
|
|
853
|
+
case ssa.VecLaneI32x4:
|
|
854
|
+
vecOp = sseOpcodePmaxud
|
|
855
|
+
}
|
|
856
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
857
|
+
|
|
858
|
+
case ssa.OpcodeVAvgRound:
|
|
859
|
+
x, y, lane := instr.Arg2WithLane()
|
|
860
|
+
var vecOp sseOpcode
|
|
861
|
+
switch lane {
|
|
862
|
+
case ssa.VecLaneI8x16:
|
|
863
|
+
vecOp = sseOpcodePavgb
|
|
864
|
+
case ssa.VecLaneI16x8:
|
|
865
|
+
vecOp = sseOpcodePavgw
|
|
866
|
+
}
|
|
867
|
+
m.lowerVbBinOp(vecOp, x, y, instr.Return())
|
|
868
|
+
|
|
869
|
+
case ssa.OpcodeVIcmp:
|
|
870
|
+
x, y, c, lane := instr.VIcmpData()
|
|
871
|
+
m.lowerVIcmp(x, y, c, instr.Return(), lane)
|
|
872
|
+
|
|
873
|
+
case ssa.OpcodeVFcmp:
|
|
874
|
+
x, y, c, lane := instr.VFcmpData()
|
|
875
|
+
m.lowerVFcmp(x, y, c, instr.Return(), lane)
|
|
876
|
+
|
|
877
|
+
case ssa.OpcodeExtractlane:
|
|
878
|
+
x, index, signed, lane := instr.ExtractlaneData()
|
|
879
|
+
m.lowerExtractLane(x, index, signed, instr.Return(), lane)
|
|
880
|
+
|
|
881
|
+
case ssa.OpcodeInsertlane:
|
|
882
|
+
x, y, index, lane := instr.InsertlaneData()
|
|
883
|
+
m.lowerInsertLane(x, y, index, instr.Return(), lane)
|
|
884
|
+
|
|
885
|
+
case ssa.OpcodeSwizzle:
|
|
886
|
+
x, y, _ := instr.Arg2WithLane()
|
|
887
|
+
m.lowerSwizzle(x, y, instr.Return())
|
|
888
|
+
|
|
889
|
+
case ssa.OpcodeShuffle:
|
|
890
|
+
x, y, lo, hi := instr.ShuffleData()
|
|
891
|
+
m.lowerShuffle(x, y, lo, hi, instr.Return())
|
|
892
|
+
|
|
893
|
+
case ssa.OpcodeSplat:
|
|
894
|
+
x, lane := instr.ArgWithLane()
|
|
895
|
+
m.lowerSplat(x, instr.Return(), lane)
|
|
896
|
+
|
|
897
|
+
case ssa.OpcodeSqmulRoundSat:
|
|
898
|
+
x, y := instr.Arg2()
|
|
899
|
+
m.lowerSqmulRoundSat(x, y, instr.Return())
|
|
900
|
+
|
|
901
|
+
case ssa.OpcodeVZeroExtLoad:
|
|
902
|
+
ptr, offset, typ := instr.VZeroExtLoadData()
|
|
903
|
+
var sseOp sseOpcode
|
|
904
|
+
// Both movss and movsd clears the higher bits of the destination register upt 128 bits.
|
|
905
|
+
// https://www.felixcloutier.com/x86/movss
|
|
906
|
+
// https://www.felixcloutier.com/x86/movsd
|
|
907
|
+
if typ == ssa.TypeF32 {
|
|
908
|
+
sseOp = sseOpcodeMovss
|
|
909
|
+
} else {
|
|
910
|
+
sseOp = sseOpcodeMovsd
|
|
911
|
+
}
|
|
912
|
+
mem := m.lowerToAddressMode(ptr, offset)
|
|
913
|
+
dst := m.c.VRegOf(instr.Return())
|
|
914
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOp, newOperandMem(mem), dst))
|
|
915
|
+
|
|
916
|
+
case ssa.OpcodeVMinPseudo:
|
|
917
|
+
x, y, lane := instr.Arg2WithLane()
|
|
918
|
+
var vecOp sseOpcode
|
|
919
|
+
switch lane {
|
|
920
|
+
case ssa.VecLaneF32x4:
|
|
921
|
+
vecOp = sseOpcodeMinps
|
|
922
|
+
case ssa.VecLaneF64x2:
|
|
923
|
+
vecOp = sseOpcodeMinpd
|
|
924
|
+
default:
|
|
925
|
+
panic("BUG: unexpected lane type")
|
|
926
|
+
}
|
|
927
|
+
m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
|
|
928
|
+
|
|
929
|
+
case ssa.OpcodeVMaxPseudo:
|
|
930
|
+
x, y, lane := instr.Arg2WithLane()
|
|
931
|
+
var vecOp sseOpcode
|
|
932
|
+
switch lane {
|
|
933
|
+
case ssa.VecLaneF32x4:
|
|
934
|
+
vecOp = sseOpcodeMaxps
|
|
935
|
+
case ssa.VecLaneF64x2:
|
|
936
|
+
vecOp = sseOpcodeMaxpd
|
|
937
|
+
default:
|
|
938
|
+
panic("BUG: unexpected lane type")
|
|
939
|
+
}
|
|
940
|
+
m.lowerVbBinOpUnaligned(vecOp, y, x, instr.Return())
|
|
941
|
+
|
|
942
|
+
case ssa.OpcodeVIshl:
|
|
943
|
+
x, y, lane := instr.Arg2WithLane()
|
|
944
|
+
m.lowerVIshl(x, y, instr.Return(), lane)
|
|
945
|
+
|
|
946
|
+
case ssa.OpcodeVSshr:
|
|
947
|
+
x, y, lane := instr.Arg2WithLane()
|
|
948
|
+
m.lowerVSshr(x, y, instr.Return(), lane)
|
|
949
|
+
|
|
950
|
+
case ssa.OpcodeVUshr:
|
|
951
|
+
x, y, lane := instr.Arg2WithLane()
|
|
952
|
+
m.lowerVUshr(x, y, instr.Return(), lane)
|
|
953
|
+
|
|
954
|
+
case ssa.OpcodeVCeil:
|
|
955
|
+
x, lane := instr.ArgWithLane()
|
|
956
|
+
m.lowerVRound(x, instr.Return(), 0x2, lane == ssa.VecLaneF64x2)
|
|
957
|
+
|
|
958
|
+
case ssa.OpcodeVFloor:
|
|
959
|
+
x, lane := instr.ArgWithLane()
|
|
960
|
+
m.lowerVRound(x, instr.Return(), 0x1, lane == ssa.VecLaneF64x2)
|
|
961
|
+
|
|
962
|
+
case ssa.OpcodeVTrunc:
|
|
963
|
+
x, lane := instr.ArgWithLane()
|
|
964
|
+
m.lowerVRound(x, instr.Return(), 0x3, lane == ssa.VecLaneF64x2)
|
|
965
|
+
|
|
966
|
+
case ssa.OpcodeVNearest:
|
|
967
|
+
x, lane := instr.ArgWithLane()
|
|
968
|
+
m.lowerVRound(x, instr.Return(), 0x0, lane == ssa.VecLaneF64x2)
|
|
969
|
+
|
|
970
|
+
case ssa.OpcodeExtIaddPairwise:
|
|
971
|
+
x, lane, signed := instr.ExtIaddPairwiseData()
|
|
972
|
+
m.lowerExtIaddPairwise(x, instr.Return(), lane, signed)
|
|
973
|
+
|
|
974
|
+
case ssa.OpcodeUwidenLow, ssa.OpcodeSwidenLow:
|
|
975
|
+
x, lane := instr.ArgWithLane()
|
|
976
|
+
m.lowerWidenLow(x, instr.Return(), lane, op == ssa.OpcodeSwidenLow)
|
|
977
|
+
|
|
978
|
+
case ssa.OpcodeUwidenHigh, ssa.OpcodeSwidenHigh:
|
|
979
|
+
x, lane := instr.ArgWithLane()
|
|
980
|
+
m.lowerWidenHigh(x, instr.Return(), lane, op == ssa.OpcodeSwidenHigh)
|
|
981
|
+
|
|
982
|
+
case ssa.OpcodeLoadSplat:
|
|
983
|
+
ptr, offset, lane := instr.LoadSplatData()
|
|
984
|
+
m.lowerLoadSplat(ptr, offset, instr.Return(), lane)
|
|
985
|
+
|
|
986
|
+
case ssa.OpcodeVFcvtFromUint, ssa.OpcodeVFcvtFromSint:
|
|
987
|
+
x, lane := instr.ArgWithLane()
|
|
988
|
+
m.lowerVFcvtFromInt(x, instr.Return(), lane, op == ssa.OpcodeVFcvtFromSint)
|
|
989
|
+
|
|
990
|
+
case ssa.OpcodeVFcvtToSintSat, ssa.OpcodeVFcvtToUintSat:
|
|
991
|
+
x, lane := instr.ArgWithLane()
|
|
992
|
+
m.lowerVFcvtToIntSat(x, instr.Return(), lane, op == ssa.OpcodeVFcvtToSintSat)
|
|
993
|
+
|
|
994
|
+
case ssa.OpcodeSnarrow, ssa.OpcodeUnarrow:
|
|
995
|
+
x, y, lane := instr.Arg2WithLane()
|
|
996
|
+
m.lowerNarrow(x, y, instr.Return(), lane, op == ssa.OpcodeSnarrow)
|
|
997
|
+
|
|
998
|
+
case ssa.OpcodeFvpromoteLow:
|
|
999
|
+
x := instr.Arg()
|
|
1000
|
+
src := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
1001
|
+
dst := m.c.VRegOf(instr.Return())
|
|
1002
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtps2pd, src, dst))
|
|
1003
|
+
|
|
1004
|
+
case ssa.OpcodeFvdemote:
|
|
1005
|
+
x := instr.Arg()
|
|
1006
|
+
src := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
1007
|
+
dst := m.c.VRegOf(instr.Return())
|
|
1008
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeCvtpd2ps, src, dst))
|
|
1009
|
+
|
|
1010
|
+
case ssa.OpcodeWideningPairwiseDotProductS:
|
|
1011
|
+
x, y := instr.Arg2()
|
|
1012
|
+
m.lowerWideningPairwiseDotProductS(x, y, instr.Return())
|
|
1013
|
+
|
|
1014
|
+
case ssa.OpcodeVIabs:
|
|
1015
|
+
m.lowerVIabs(instr)
|
|
1016
|
+
case ssa.OpcodeVIpopcnt:
|
|
1017
|
+
m.lowerVIpopcnt(instr)
|
|
1018
|
+
case ssa.OpcodeVFmin:
|
|
1019
|
+
m.lowerVFmin(instr)
|
|
1020
|
+
case ssa.OpcodeVFmax:
|
|
1021
|
+
m.lowerVFmax(instr)
|
|
1022
|
+
case ssa.OpcodeVFabs:
|
|
1023
|
+
m.lowerVFabs(instr)
|
|
1024
|
+
case ssa.OpcodeUndefined:
|
|
1025
|
+
m.insert(m.allocateInstr().asUD2())
|
|
1026
|
+
case ssa.OpcodeExitWithCode:
|
|
1027
|
+
execCtx, code := instr.ExitWithCodeData()
|
|
1028
|
+
m.lowerExitWithCode(m.c.VRegOf(execCtx), code)
|
|
1029
|
+
case ssa.OpcodeExitIfTrueWithCode:
|
|
1030
|
+
execCtx, c, code := instr.ExitIfTrueWithCodeData()
|
|
1031
|
+
m.lowerExitIfTrueWithCode(m.c.VRegOf(execCtx), c, code)
|
|
1032
|
+
case ssa.OpcodeLoad:
|
|
1033
|
+
ptr, offset, typ := instr.LoadData()
|
|
1034
|
+
dst := m.c.VRegOf(instr.Return())
|
|
1035
|
+
m.lowerLoad(ptr, offset, typ, dst)
|
|
1036
|
+
case ssa.OpcodeUload8, ssa.OpcodeUload16, ssa.OpcodeUload32, ssa.OpcodeSload8, ssa.OpcodeSload16, ssa.OpcodeSload32:
|
|
1037
|
+
ptr, offset, _ := instr.LoadData()
|
|
1038
|
+
ret := m.c.VRegOf(instr.Return())
|
|
1039
|
+
m.lowerExtLoad(op, ptr, offset, ret)
|
|
1040
|
+
case ssa.OpcodeVconst:
|
|
1041
|
+
result := m.c.VRegOf(instr.Return())
|
|
1042
|
+
lo, hi := instr.VconstData()
|
|
1043
|
+
m.lowerVconst(result, lo, hi)
|
|
1044
|
+
case ssa.OpcodeSExtend, ssa.OpcodeUExtend:
|
|
1045
|
+
from, to, signed := instr.ExtendData()
|
|
1046
|
+
m.lowerExtend(instr.Arg(), instr.Return(), from, to, signed)
|
|
1047
|
+
case ssa.OpcodeIcmp:
|
|
1048
|
+
m.lowerIcmp(instr)
|
|
1049
|
+
case ssa.OpcodeFcmp:
|
|
1050
|
+
m.lowerFcmp(instr)
|
|
1051
|
+
case ssa.OpcodeSelect:
|
|
1052
|
+
cval, x, y := instr.SelectData()
|
|
1053
|
+
m.lowerSelect(x, y, cval, instr.Return())
|
|
1054
|
+
case ssa.OpcodeIreduce:
|
|
1055
|
+
rn := m.getOperand_Mem_Reg(m.c.ValueDefinition(instr.Arg()))
|
|
1056
|
+
retVal := instr.Return()
|
|
1057
|
+
rd := m.c.VRegOf(retVal)
|
|
1058
|
+
|
|
1059
|
+
if retVal.Type() != ssa.TypeI32 {
|
|
1060
|
+
panic("TODO?: Ireduce to non-i32")
|
|
1061
|
+
}
|
|
1062
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, rd))
|
|
1063
|
+
|
|
1064
|
+
case ssa.OpcodeAtomicLoad:
|
|
1065
|
+
ptr := instr.Arg()
|
|
1066
|
+
size := instr.AtomicTargetSize()
|
|
1067
|
+
dst := m.c.VRegOf(instr.Return())
|
|
1068
|
+
|
|
1069
|
+
// At this point, the ptr is ensured to be aligned, so using a normal load is atomic.
|
|
1070
|
+
// https://github.com/golang/go/blob/go1.24.0/src/internal/runtime/atomic/atomic_amd64.go#L29
|
|
1071
|
+
mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
|
|
1072
|
+
load := m.allocateInstr()
|
|
1073
|
+
switch size {
|
|
1074
|
+
case 8:
|
|
1075
|
+
load.asMov64MR(mem, dst)
|
|
1076
|
+
case 4:
|
|
1077
|
+
load.asMovzxRmR(extModeLQ, mem, dst)
|
|
1078
|
+
case 2:
|
|
1079
|
+
load.asMovzxRmR(extModeWQ, mem, dst)
|
|
1080
|
+
case 1:
|
|
1081
|
+
load.asMovzxRmR(extModeBQ, mem, dst)
|
|
1082
|
+
default:
|
|
1083
|
+
panic("BUG")
|
|
1084
|
+
}
|
|
1085
|
+
m.insert(load)
|
|
1086
|
+
|
|
1087
|
+
case ssa.OpcodeFence:
|
|
1088
|
+
m.insert(m.allocateInstr().asMFence())
|
|
1089
|
+
|
|
1090
|
+
case ssa.OpcodeAtomicStore:
|
|
1091
|
+
ptr, _val := instr.Arg2()
|
|
1092
|
+
size := instr.AtomicTargetSize()
|
|
1093
|
+
|
|
1094
|
+
val := m.getOperand_Reg(m.c.ValueDefinition(_val))
|
|
1095
|
+
// The content on the val register will be overwritten by xchg, so we need to copy it to a temporary register.
|
|
1096
|
+
copied := m.copyToTmp(val.reg())
|
|
1097
|
+
|
|
1098
|
+
mem := newOperandMem(m.lowerToAddressMode(ptr, 0))
|
|
1099
|
+
store := m.allocateInstr().asXCHG(copied, mem, byte(size))
|
|
1100
|
+
m.insert(store)
|
|
1101
|
+
|
|
1102
|
+
case ssa.OpcodeAtomicCas:
|
|
1103
|
+
addr, exp, repl := instr.Arg3()
|
|
1104
|
+
size := instr.AtomicTargetSize()
|
|
1105
|
+
m.lowerAtomicCas(addr, exp, repl, size, instr.Return())
|
|
1106
|
+
|
|
1107
|
+
case ssa.OpcodeAtomicRmw:
|
|
1108
|
+
addr, val := instr.Arg2()
|
|
1109
|
+
atomicOp, size := instr.AtomicRmwData()
|
|
1110
|
+
m.lowerAtomicRmw(atomicOp, addr, val, size, instr.Return())
|
|
1111
|
+
|
|
1112
|
+
case ssa.OpcodeTailCallReturnCall, ssa.OpcodeTailCallReturnCallIndirect:
|
|
1113
|
+
m.lowerTailCall(instr)
|
|
1114
|
+
|
|
1115
|
+
default:
|
|
1116
|
+
panic("TODO: lowering " + op.String())
|
|
1117
|
+
}
|
|
1118
|
+
}
|
|
1119
|
+
|
|
1120
|
+
func (m *machine) lowerAtomicRmw(op ssa.AtomicRmwOp, addr, val ssa.Value, size uint64, ret ssa.Value) {
|
|
1121
|
+
mem := m.lowerToAddressMode(addr, 0)
|
|
1122
|
+
_val := m.getOperand_Reg(m.c.ValueDefinition(val))
|
|
1123
|
+
|
|
1124
|
+
switch op {
|
|
1125
|
+
case ssa.AtomicRmwOpAdd, ssa.AtomicRmwOpSub:
|
|
1126
|
+
valCopied := m.copyToTmp(_val.reg())
|
|
1127
|
+
if op == ssa.AtomicRmwOpSub {
|
|
1128
|
+
// Negate the value.
|
|
1129
|
+
m.insert(m.allocateInstr().asNeg(newOperandReg(valCopied), true))
|
|
1130
|
+
}
|
|
1131
|
+
m.insert(m.allocateInstr().asLockXAdd(valCopied, mem, byte(size)))
|
|
1132
|
+
m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
|
|
1133
|
+
m.copyTo(valCopied, m.c.VRegOf(ret))
|
|
1134
|
+
|
|
1135
|
+
case ssa.AtomicRmwOpAnd, ssa.AtomicRmwOpOr, ssa.AtomicRmwOpXor:
|
|
1136
|
+
accumulator := raxVReg
|
|
1137
|
+
// Reserve rax for the accumulator to make regalloc happy.
|
|
1138
|
+
// Note: do this initialization before defining valCopied, because it might be the same register and
|
|
1139
|
+
// if that happens, the unnecessary load/store will be performed inside the loop.
|
|
1140
|
+
// This can be mitigated in any way once the register allocator is clever enough.
|
|
1141
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(accumulator))
|
|
1142
|
+
|
|
1143
|
+
// Copy the value to a temporary register.
|
|
1144
|
+
valCopied := m.copyToTmp(_val.reg())
|
|
1145
|
+
m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
|
|
1146
|
+
|
|
1147
|
+
memOp := newOperandMem(mem)
|
|
1148
|
+
tmp := m.c.AllocateVReg(ssa.TypeI64)
|
|
1149
|
+
beginLoop, beginLoopLabel := m.allocateBrTarget()
|
|
1150
|
+
{
|
|
1151
|
+
m.insert(beginLoop)
|
|
1152
|
+
// Reset the value on tmp by the original value.
|
|
1153
|
+
m.copyTo(valCopied, tmp)
|
|
1154
|
+
// Load the current value at the memory location into accumulator.
|
|
1155
|
+
switch size {
|
|
1156
|
+
case 1:
|
|
1157
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, memOp, accumulator))
|
|
1158
|
+
case 2:
|
|
1159
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, memOp, accumulator))
|
|
1160
|
+
case 4:
|
|
1161
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, memOp, accumulator))
|
|
1162
|
+
case 8:
|
|
1163
|
+
m.insert(m.allocateInstr().asMov64MR(memOp, accumulator))
|
|
1164
|
+
default:
|
|
1165
|
+
panic("BUG")
|
|
1166
|
+
}
|
|
1167
|
+
// Then perform the logical operation on the accumulator and the value on tmp.
|
|
1168
|
+
switch op {
|
|
1169
|
+
case ssa.AtomicRmwOpAnd:
|
|
1170
|
+
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeAnd, newOperandReg(accumulator), tmp, true))
|
|
1171
|
+
case ssa.AtomicRmwOpOr:
|
|
1172
|
+
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeOr, newOperandReg(accumulator), tmp, true))
|
|
1173
|
+
case ssa.AtomicRmwOpXor:
|
|
1174
|
+
m.insert(m.allocateInstr().asAluRmiR(aluRmiROpcodeXor, newOperandReg(accumulator), tmp, true))
|
|
1175
|
+
default:
|
|
1176
|
+
panic("BUG")
|
|
1177
|
+
}
|
|
1178
|
+
// Finally, try compare-exchange the value at the memory location with the tmp.
|
|
1179
|
+
m.insert(m.allocateInstr().asLockCmpXCHG(tmp, memOp.addressMode(), byte(size)))
|
|
1180
|
+
// If it succeeds, ZF will be set, and we can break the loop.
|
|
1181
|
+
m.insert(m.allocateInstr().asJmpIf(condNZ, newOperandLabel(beginLoopLabel)))
|
|
1182
|
+
}
|
|
1183
|
+
|
|
1184
|
+
// valCopied must be alive at the end of the loop.
|
|
1185
|
+
m.insert(m.allocateInstr().asNopUseReg(valCopied))
|
|
1186
|
+
|
|
1187
|
+
// At this point, accumulator contains the result.
|
|
1188
|
+
m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
|
|
1189
|
+
m.copyTo(accumulator, m.c.VRegOf(ret))
|
|
1190
|
+
|
|
1191
|
+
case ssa.AtomicRmwOpXchg:
|
|
1192
|
+
valCopied := m.copyToTmp(_val.reg())
|
|
1193
|
+
|
|
1194
|
+
m.insert(m.allocateInstr().asXCHG(valCopied, newOperandMem(mem), byte(size)))
|
|
1195
|
+
m.clearHigherBitsForAtomic(valCopied, size, ret.Type())
|
|
1196
|
+
m.copyTo(valCopied, m.c.VRegOf(ret))
|
|
1197
|
+
|
|
1198
|
+
default:
|
|
1199
|
+
panic("BUG")
|
|
1200
|
+
}
|
|
1201
|
+
}
|
|
1202
|
+
|
|
1203
|
+
func (m *machine) lowerAtomicCas(addr, exp, repl ssa.Value, size uint64, ret ssa.Value) {
|
|
1204
|
+
mem := m.lowerToAddressMode(addr, 0)
|
|
1205
|
+
expOp := m.getOperand_Reg(m.c.ValueDefinition(exp))
|
|
1206
|
+
replOp := m.getOperand_Reg(m.c.ValueDefinition(repl))
|
|
1207
|
+
|
|
1208
|
+
accumulator := raxVReg
|
|
1209
|
+
m.copyTo(expOp.reg(), accumulator)
|
|
1210
|
+
m.insert(m.allocateInstr().asLockCmpXCHG(replOp.reg(), mem, byte(size)))
|
|
1211
|
+
m.clearHigherBitsForAtomic(accumulator, size, ret.Type())
|
|
1212
|
+
m.copyTo(accumulator, m.c.VRegOf(ret))
|
|
1213
|
+
}
|
|
1214
|
+
|
|
1215
|
+
func (m *machine) clearHigherBitsForAtomic(r regalloc.VReg, valSize uint64, resultType ssa.Type) {
|
|
1216
|
+
switch resultType {
|
|
1217
|
+
case ssa.TypeI32:
|
|
1218
|
+
switch valSize {
|
|
1219
|
+
case 1:
|
|
1220
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBL, newOperandReg(r), r))
|
|
1221
|
+
case 2:
|
|
1222
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeWL, newOperandReg(r), r))
|
|
1223
|
+
}
|
|
1224
|
+
case ssa.TypeI64:
|
|
1225
|
+
switch valSize {
|
|
1226
|
+
case 1:
|
|
1227
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(r), r))
|
|
1228
|
+
case 2:
|
|
1229
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeWQ, newOperandReg(r), r))
|
|
1230
|
+
case 4:
|
|
1231
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, newOperandReg(r), r))
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
func (m *machine) lowerFcmp(instr *ssa.Instruction) {
|
|
1237
|
+
f1, f2, and := m.lowerFcmpToFlags(instr)
|
|
1238
|
+
rd := m.c.VRegOf(instr.Return())
|
|
1239
|
+
if f2 == condInvalid {
|
|
1240
|
+
tmp := m.c.AllocateVReg(ssa.TypeI32)
|
|
1241
|
+
m.insert(m.allocateInstr().asSetcc(f1, tmp))
|
|
1242
|
+
// On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
|
|
1243
|
+
// the semantics of Icmp that sets either 0 or 1.
|
|
1244
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
|
|
1245
|
+
} else {
|
|
1246
|
+
tmp1, tmp2 := m.c.AllocateVReg(ssa.TypeI32), m.c.AllocateVReg(ssa.TypeI32)
|
|
1247
|
+
m.insert(m.allocateInstr().asSetcc(f1, tmp1))
|
|
1248
|
+
m.insert(m.allocateInstr().asSetcc(f2, tmp2))
|
|
1249
|
+
var op aluRmiROpcode
|
|
1250
|
+
if and {
|
|
1251
|
+
op = aluRmiROpcodeAnd
|
|
1252
|
+
} else {
|
|
1253
|
+
op = aluRmiROpcodeOr
|
|
1254
|
+
}
|
|
1255
|
+
m.insert(m.allocateInstr().asAluRmiR(op, newOperandReg(tmp1), tmp2, false))
|
|
1256
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp2), rd))
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
|
|
1260
|
+
func (m *machine) lowerIcmp(instr *ssa.Instruction) {
|
|
1261
|
+
x, y, c := instr.IcmpData()
|
|
1262
|
+
m.lowerIcmpToFlag(m.c.ValueDefinition(x), m.c.ValueDefinition(y), x.Type() == ssa.TypeI64)
|
|
1263
|
+
rd := m.c.VRegOf(instr.Return())
|
|
1264
|
+
tmp := m.c.AllocateVReg(ssa.TypeI32)
|
|
1265
|
+
m.insert(m.allocateInstr().asSetcc(condFromSSAIntCmpCond(c), tmp))
|
|
1266
|
+
// On amd64, setcc only sets the first byte of the register, so we need to zero extend it to match
|
|
1267
|
+
// the semantics of Icmp that sets either 0 or 1.
|
|
1268
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeBQ, newOperandReg(tmp), rd))
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
func (m *machine) lowerSelect(x, y, cval, ret ssa.Value) {
|
|
1272
|
+
xo, yo := m.getOperand_Mem_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
1273
|
+
rd := m.c.VRegOf(ret)
|
|
1274
|
+
|
|
1275
|
+
var cond cond
|
|
1276
|
+
cvalDef := m.c.ValueDefinition(cval)
|
|
1277
|
+
switch m.c.MatchInstrOneOf(cvalDef, condBranchMatches[:]) {
|
|
1278
|
+
case ssa.OpcodeIcmp:
|
|
1279
|
+
icmp := cvalDef.Instr
|
|
1280
|
+
xc, yc, cc := icmp.IcmpData()
|
|
1281
|
+
m.lowerIcmpToFlag(m.c.ValueDefinition(xc), m.c.ValueDefinition(yc), xc.Type() == ssa.TypeI64)
|
|
1282
|
+
cond = condFromSSAIntCmpCond(cc)
|
|
1283
|
+
icmp.Lowered()
|
|
1284
|
+
default: // TODO: match ssa.OpcodeFcmp for optimization, but seems a bit complex.
|
|
1285
|
+
cv := m.getOperand_Reg(cvalDef)
|
|
1286
|
+
test := m.allocateInstr().asCmpRmiR(false, cv, cv.reg(), false)
|
|
1287
|
+
m.insert(test)
|
|
1288
|
+
cond = condNZ
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
if typ := x.Type(); typ.IsInt() {
|
|
1292
|
+
_64 := typ.Bits() == 64
|
|
1293
|
+
mov := m.allocateInstr()
|
|
1294
|
+
tmp := m.c.AllocateVReg(typ)
|
|
1295
|
+
switch yo.kind {
|
|
1296
|
+
case operandKindReg:
|
|
1297
|
+
mov.asMovRR(yo.reg(), tmp, _64)
|
|
1298
|
+
case operandKindMem:
|
|
1299
|
+
if _64 {
|
|
1300
|
+
mov.asMov64MR(yo, tmp)
|
|
1301
|
+
} else {
|
|
1302
|
+
mov.asMovzxRmR(extModeLQ, yo, tmp)
|
|
1303
|
+
}
|
|
1304
|
+
default:
|
|
1305
|
+
panic("BUG")
|
|
1306
|
+
}
|
|
1307
|
+
m.insert(mov)
|
|
1308
|
+
cmov := m.allocateInstr().asCmove(cond, xo, tmp, _64)
|
|
1309
|
+
m.insert(cmov)
|
|
1310
|
+
m.insert(m.allocateInstr().asMovRR(tmp, rd, _64))
|
|
1311
|
+
} else {
|
|
1312
|
+
mov := m.allocateInstr()
|
|
1313
|
+
tmp := m.c.AllocateVReg(typ)
|
|
1314
|
+
switch typ {
|
|
1315
|
+
case ssa.TypeF32:
|
|
1316
|
+
mov.asXmmUnaryRmR(sseOpcodeMovss, yo, tmp)
|
|
1317
|
+
case ssa.TypeF64:
|
|
1318
|
+
mov.asXmmUnaryRmR(sseOpcodeMovsd, yo, tmp)
|
|
1319
|
+
case ssa.TypeV128:
|
|
1320
|
+
mov.asXmmUnaryRmR(sseOpcodeMovdqu, yo, tmp)
|
|
1321
|
+
default:
|
|
1322
|
+
panic("BUG")
|
|
1323
|
+
}
|
|
1324
|
+
m.insert(mov)
|
|
1325
|
+
|
|
1326
|
+
cmov := m.allocateInstr().asXmmCMov(cond, xo, tmp, typ.Size())
|
|
1327
|
+
m.insert(cmov)
|
|
1328
|
+
|
|
1329
|
+
m.copyTo(tmp, rd)
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
func (m *machine) lowerXmmCmovAfterRegAlloc(i *instruction) {
|
|
1334
|
+
x := i.op1
|
|
1335
|
+
rd := i.op2.reg()
|
|
1336
|
+
cond := cond(i.u1)
|
|
1337
|
+
|
|
1338
|
+
jcc := m.allocateInstr()
|
|
1339
|
+
m.insert(jcc)
|
|
1340
|
+
|
|
1341
|
+
mov := m.allocateInstr()
|
|
1342
|
+
switch i.u2 {
|
|
1343
|
+
case 4:
|
|
1344
|
+
mov.asXmmUnaryRmR(sseOpcodeMovss, x, rd)
|
|
1345
|
+
case 8:
|
|
1346
|
+
mov.asXmmUnaryRmR(sseOpcodeMovsd, x, rd)
|
|
1347
|
+
case 16:
|
|
1348
|
+
mov.asXmmUnaryRmR(sseOpcodeMovdqu, x, rd)
|
|
1349
|
+
default:
|
|
1350
|
+
panic("BUG")
|
|
1351
|
+
}
|
|
1352
|
+
m.insert(mov)
|
|
1353
|
+
|
|
1354
|
+
nop, end := m.allocateBrTarget()
|
|
1355
|
+
m.insert(nop)
|
|
1356
|
+
jcc.asJmpIf(cond.invert(), newOperandLabel(end))
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1359
|
+
func (m *machine) lowerExtend(_arg, ret ssa.Value, from, to byte, signed bool) {
|
|
1360
|
+
rd0 := m.c.VRegOf(ret)
|
|
1361
|
+
arg := m.getOperand_Mem_Reg(m.c.ValueDefinition(_arg))
|
|
1362
|
+
|
|
1363
|
+
rd := m.c.AllocateVReg(ret.Type())
|
|
1364
|
+
|
|
1365
|
+
ext := m.allocateInstr()
|
|
1366
|
+
switch {
|
|
1367
|
+
case from == 8 && to == 16 && signed:
|
|
1368
|
+
ext.asMovsxRmR(extModeBQ, arg, rd)
|
|
1369
|
+
case from == 8 && to == 16 && !signed:
|
|
1370
|
+
ext.asMovzxRmR(extModeBL, arg, rd)
|
|
1371
|
+
case from == 8 && to == 32 && signed:
|
|
1372
|
+
ext.asMovsxRmR(extModeBL, arg, rd)
|
|
1373
|
+
case from == 8 && to == 32 && !signed:
|
|
1374
|
+
ext.asMovzxRmR(extModeBQ, arg, rd)
|
|
1375
|
+
case from == 8 && to == 64 && signed:
|
|
1376
|
+
ext.asMovsxRmR(extModeBQ, arg, rd)
|
|
1377
|
+
case from == 8 && to == 64 && !signed:
|
|
1378
|
+
ext.asMovzxRmR(extModeBQ, arg, rd)
|
|
1379
|
+
case from == 16 && to == 32 && signed:
|
|
1380
|
+
ext.asMovsxRmR(extModeWL, arg, rd)
|
|
1381
|
+
case from == 16 && to == 32 && !signed:
|
|
1382
|
+
ext.asMovzxRmR(extModeWL, arg, rd)
|
|
1383
|
+
case from == 16 && to == 64 && signed:
|
|
1384
|
+
ext.asMovsxRmR(extModeWQ, arg, rd)
|
|
1385
|
+
case from == 16 && to == 64 && !signed:
|
|
1386
|
+
ext.asMovzxRmR(extModeWQ, arg, rd)
|
|
1387
|
+
case from == 32 && to == 64 && signed:
|
|
1388
|
+
ext.asMovsxRmR(extModeLQ, arg, rd)
|
|
1389
|
+
case from == 32 && to == 64 && !signed:
|
|
1390
|
+
ext.asMovzxRmR(extModeLQ, arg, rd)
|
|
1391
|
+
default:
|
|
1392
|
+
panic(fmt.Sprintf("BUG: unhandled extend: from=%d, to=%d, signed=%t", from, to, signed))
|
|
1393
|
+
}
|
|
1394
|
+
m.insert(ext)
|
|
1395
|
+
|
|
1396
|
+
m.copyTo(rd, rd0)
|
|
1397
|
+
}
|
|
1398
|
+
|
|
1399
|
+
func (m *machine) lowerVconst(dst regalloc.VReg, lo, hi uint64) {
|
|
1400
|
+
if lo == 0 && hi == 0 {
|
|
1401
|
+
m.insert(m.allocateInstr().asZeros(dst))
|
|
1402
|
+
return
|
|
1403
|
+
}
|
|
1404
|
+
|
|
1405
|
+
load := m.allocateInstr()
|
|
1406
|
+
l, pos := m.allocateLabel()
|
|
1407
|
+
m.consts = append(m.consts, _const{label: l, labelPos: pos, lo: lo, hi: hi})
|
|
1408
|
+
load.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(l)), dst)
|
|
1409
|
+
m.insert(load)
|
|
1410
|
+
}
|
|
1411
|
+
|
|
1412
|
+
func (m *machine) lowerCtz(instr *ssa.Instruction) {
|
|
1413
|
+
if m.cpuFeatures.Has(platform.CpuFeatureAmd64BMI1) {
|
|
1414
|
+
m.lowerUnaryRmR(instr, unaryRmROpcodeTzcnt)
|
|
1415
|
+
} else {
|
|
1416
|
+
// On processors that do not support TZCNT, the BSF instruction is
|
|
1417
|
+
// executed instead. The key difference between TZCNT and BSF
|
|
1418
|
+
// instruction is that if source operand is zero, the content of
|
|
1419
|
+
// destination operand is undefined.
|
|
1420
|
+
// https://www.felixcloutier.com/x86/tzcnt.html
|
|
1421
|
+
|
|
1422
|
+
x := instr.Arg()
|
|
1423
|
+
if !x.Type().IsInt() {
|
|
1424
|
+
panic("BUG?")
|
|
1425
|
+
}
|
|
1426
|
+
_64 := x.Type().Bits() == 64
|
|
1427
|
+
|
|
1428
|
+
xDef := m.c.ValueDefinition(x)
|
|
1429
|
+
tmp := m.c.AllocateVReg(x.Type())
|
|
1430
|
+
rm := m.getOperand_Reg(xDef)
|
|
1431
|
+
|
|
1432
|
+
// First, we have to check if the target is non-zero.
|
|
1433
|
+
test := m.allocateInstr()
|
|
1434
|
+
test.asCmpRmiR(false, rm, rm.reg(), _64)
|
|
1435
|
+
m.insert(test)
|
|
1436
|
+
|
|
1437
|
+
jmpNz := m.allocateInstr()
|
|
1438
|
+
m.insert(jmpNz)
|
|
1439
|
+
|
|
1440
|
+
// If the value is zero, we just push the const value.
|
|
1441
|
+
m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
|
|
1442
|
+
|
|
1443
|
+
// Now jump right after the non-zero case.
|
|
1444
|
+
jmpAtEnd := m.allocateInstr()
|
|
1445
|
+
m.insert(jmpAtEnd)
|
|
1446
|
+
|
|
1447
|
+
// jmpNz target label is set here.
|
|
1448
|
+
nop, nz := m.allocateBrTarget()
|
|
1449
|
+
jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
|
|
1450
|
+
m.insert(nop)
|
|
1451
|
+
|
|
1452
|
+
// Emit the non-zero case.
|
|
1453
|
+
bsr := m.allocateInstr()
|
|
1454
|
+
bsr.asUnaryRmR(unaryRmROpcodeBsf, rm, tmp, _64)
|
|
1455
|
+
m.insert(bsr)
|
|
1456
|
+
|
|
1457
|
+
// jmpAtEnd target label is set here.
|
|
1458
|
+
nopEnd, end := m.allocateBrTarget()
|
|
1459
|
+
jmpAtEnd.asJmp(newOperandLabel(end))
|
|
1460
|
+
m.insert(nopEnd)
|
|
1461
|
+
|
|
1462
|
+
m.copyTo(tmp, m.c.VRegOf(instr.Return()))
|
|
1463
|
+
}
|
|
1464
|
+
}
|
|
1465
|
+
|
|
1466
|
+
func (m *machine) lowerClz(instr *ssa.Instruction) {
|
|
1467
|
+
if m.cpuFeatures.Has(platform.CpuFeatureAmd64ABM) {
|
|
1468
|
+
m.lowerUnaryRmR(instr, unaryRmROpcodeLzcnt)
|
|
1469
|
+
} else {
|
|
1470
|
+
// On processors that do not support LZCNT, we combine BSR (calculating
|
|
1471
|
+
// most significant set bit) with XOR. This logic is described in
|
|
1472
|
+
// "Replace Raw Assembly Code with Builtin Intrinsics" section in:
|
|
1473
|
+
// https://developer.apple.com/documentation/apple-silicon/addressing-architectural-differences-in-your-macos-code.
|
|
1474
|
+
|
|
1475
|
+
x := instr.Arg()
|
|
1476
|
+
if !x.Type().IsInt() {
|
|
1477
|
+
panic("BUG?")
|
|
1478
|
+
}
|
|
1479
|
+
_64 := x.Type().Bits() == 64
|
|
1480
|
+
|
|
1481
|
+
xDef := m.c.ValueDefinition(x)
|
|
1482
|
+
rm := m.getOperand_Reg(xDef)
|
|
1483
|
+
tmp := m.c.AllocateVReg(x.Type())
|
|
1484
|
+
|
|
1485
|
+
// First, we have to check if the rm is non-zero as BSR is undefined
|
|
1486
|
+
// on zero. See https://www.felixcloutier.com/x86/bsr.
|
|
1487
|
+
test := m.allocateInstr()
|
|
1488
|
+
test.asCmpRmiR(false, rm, rm.reg(), _64)
|
|
1489
|
+
m.insert(test)
|
|
1490
|
+
|
|
1491
|
+
jmpNz := m.allocateInstr()
|
|
1492
|
+
m.insert(jmpNz)
|
|
1493
|
+
|
|
1494
|
+
// If the value is zero, we just push the const value.
|
|
1495
|
+
m.lowerIconst(tmp, uint64(x.Type().Bits()), _64)
|
|
1496
|
+
|
|
1497
|
+
// Now jump right after the non-zero case.
|
|
1498
|
+
jmpAtEnd := m.allocateInstr()
|
|
1499
|
+
m.insert(jmpAtEnd)
|
|
1500
|
+
|
|
1501
|
+
// jmpNz target label is set here.
|
|
1502
|
+
nop, nz := m.allocateBrTarget()
|
|
1503
|
+
jmpNz.asJmpIf(condNZ, newOperandLabel(nz))
|
|
1504
|
+
m.insert(nop)
|
|
1505
|
+
|
|
1506
|
+
// Emit the non-zero case.
|
|
1507
|
+
bsr := m.allocateInstr()
|
|
1508
|
+
bsr.asUnaryRmR(unaryRmROpcodeBsr, rm, tmp, _64)
|
|
1509
|
+
m.insert(bsr)
|
|
1510
|
+
|
|
1511
|
+
// Now we XOR the value with the bit length minus one.
|
|
1512
|
+
xor := m.allocateInstr()
|
|
1513
|
+
xor.asAluRmiR(aluRmiROpcodeXor, newOperandImm32(uint32(x.Type().Bits()-1)), tmp, _64)
|
|
1514
|
+
m.insert(xor)
|
|
1515
|
+
|
|
1516
|
+
// jmpAtEnd target label is set here.
|
|
1517
|
+
nopEnd, end := m.allocateBrTarget()
|
|
1518
|
+
jmpAtEnd.asJmp(newOperandLabel(end))
|
|
1519
|
+
m.insert(nopEnd)
|
|
1520
|
+
|
|
1521
|
+
m.copyTo(tmp, m.c.VRegOf(instr.Return()))
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
func (m *machine) lowerUnaryRmR(si *ssa.Instruction, op unaryRmROpcode) {
|
|
1526
|
+
x := si.Arg()
|
|
1527
|
+
if !x.Type().IsInt() {
|
|
1528
|
+
panic("BUG?")
|
|
1529
|
+
}
|
|
1530
|
+
_64 := x.Type().Bits() == 64
|
|
1531
|
+
|
|
1532
|
+
xDef := m.c.ValueDefinition(x)
|
|
1533
|
+
rm := m.getOperand_Mem_Reg(xDef)
|
|
1534
|
+
rd := m.c.VRegOf(si.Return())
|
|
1535
|
+
|
|
1536
|
+
instr := m.allocateInstr()
|
|
1537
|
+
instr.asUnaryRmR(op, rm, rd, _64)
|
|
1538
|
+
m.insert(instr)
|
|
1539
|
+
}
|
|
1540
|
+
|
|
1541
|
+
func (m *machine) lowerLoad(ptr ssa.Value, offset uint32, typ ssa.Type, dst regalloc.VReg) {
|
|
1542
|
+
mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
|
|
1543
|
+
load := m.allocateInstr()
|
|
1544
|
+
switch typ {
|
|
1545
|
+
case ssa.TypeI32:
|
|
1546
|
+
load.asMovzxRmR(extModeLQ, mem, dst)
|
|
1547
|
+
case ssa.TypeI64:
|
|
1548
|
+
load.asMov64MR(mem, dst)
|
|
1549
|
+
case ssa.TypeF32:
|
|
1550
|
+
load.asXmmUnaryRmR(sseOpcodeMovss, mem, dst)
|
|
1551
|
+
case ssa.TypeF64:
|
|
1552
|
+
load.asXmmUnaryRmR(sseOpcodeMovsd, mem, dst)
|
|
1553
|
+
case ssa.TypeV128:
|
|
1554
|
+
load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, dst)
|
|
1555
|
+
default:
|
|
1556
|
+
panic("BUG")
|
|
1557
|
+
}
|
|
1558
|
+
m.insert(load)
|
|
1559
|
+
}
|
|
1560
|
+
|
|
1561
|
+
func (m *machine) lowerExtLoad(op ssa.Opcode, ptr ssa.Value, offset uint32, dst regalloc.VReg) {
|
|
1562
|
+
mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
|
|
1563
|
+
load := m.allocateInstr()
|
|
1564
|
+
switch op {
|
|
1565
|
+
case ssa.OpcodeUload8:
|
|
1566
|
+
load.asMovzxRmR(extModeBQ, mem, dst)
|
|
1567
|
+
case ssa.OpcodeUload16:
|
|
1568
|
+
load.asMovzxRmR(extModeWQ, mem, dst)
|
|
1569
|
+
case ssa.OpcodeUload32:
|
|
1570
|
+
load.asMovzxRmR(extModeLQ, mem, dst)
|
|
1571
|
+
case ssa.OpcodeSload8:
|
|
1572
|
+
load.asMovsxRmR(extModeBQ, mem, dst)
|
|
1573
|
+
case ssa.OpcodeSload16:
|
|
1574
|
+
load.asMovsxRmR(extModeWQ, mem, dst)
|
|
1575
|
+
case ssa.OpcodeSload32:
|
|
1576
|
+
load.asMovsxRmR(extModeLQ, mem, dst)
|
|
1577
|
+
default:
|
|
1578
|
+
panic("BUG")
|
|
1579
|
+
}
|
|
1580
|
+
m.insert(load)
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
func (m *machine) lowerExitIfTrueWithCode(execCtx regalloc.VReg, cond ssa.Value, code wazevoapi.ExitCode) {
|
|
1584
|
+
condDef := m.c.ValueDefinition(cond)
|
|
1585
|
+
if !m.c.MatchInstr(condDef, ssa.OpcodeIcmp) {
|
|
1586
|
+
panic("TODO: ExitIfTrue must come after Icmp at the moment: " + condDef.Instr.Opcode().String())
|
|
1587
|
+
}
|
|
1588
|
+
cvalInstr := condDef.Instr
|
|
1589
|
+
cvalInstr.MarkLowered()
|
|
1590
|
+
|
|
1591
|
+
// We need to copy the execution context to a temp register, because if it's spilled,
|
|
1592
|
+
// it might end up being reloaded inside the exiting branch.
|
|
1593
|
+
execCtxTmp := m.copyToTmp(execCtx)
|
|
1594
|
+
|
|
1595
|
+
x, y, c := cvalInstr.IcmpData()
|
|
1596
|
+
xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
1597
|
+
if !m.tryLowerBandToFlag(xx, yy) {
|
|
1598
|
+
m.lowerIcmpToFlag(xx, yy, x.Type() == ssa.TypeI64)
|
|
1599
|
+
}
|
|
1600
|
+
|
|
1601
|
+
jmpIf := m.allocateInstr()
|
|
1602
|
+
m.insert(jmpIf)
|
|
1603
|
+
l := m.lowerExitWithCode(execCtxTmp, code)
|
|
1604
|
+
jmpIf.asJmpIf(condFromSSAIntCmpCond(c).invert(), newOperandLabel(l))
|
|
1605
|
+
}
|
|
1606
|
+
|
|
1607
|
+
func (m *machine) tryLowerBandToFlag(x, y backend.SSAValueDefinition) (ok bool) {
|
|
1608
|
+
var target backend.SSAValueDefinition
|
|
1609
|
+
var got bool
|
|
1610
|
+
if x.IsFromInstr() && x.Instr.Constant() && x.Instr.ConstantVal() == 0 {
|
|
1611
|
+
if m.c.MatchInstr(y, ssa.OpcodeBand) {
|
|
1612
|
+
target = y
|
|
1613
|
+
got = true
|
|
1614
|
+
}
|
|
1615
|
+
}
|
|
1616
|
+
|
|
1617
|
+
if y.IsFromInstr() && y.Instr.Constant() && y.Instr.ConstantVal() == 0 {
|
|
1618
|
+
if m.c.MatchInstr(x, ssa.OpcodeBand) {
|
|
1619
|
+
target = x
|
|
1620
|
+
got = true
|
|
1621
|
+
}
|
|
1622
|
+
}
|
|
1623
|
+
|
|
1624
|
+
if !got {
|
|
1625
|
+
return false
|
|
1626
|
+
}
|
|
1627
|
+
|
|
1628
|
+
bandInstr := target.Instr
|
|
1629
|
+
bandX, bandY := bandInstr.Arg2()
|
|
1630
|
+
|
|
1631
|
+
xx := m.getOperand_Reg(m.c.ValueDefinition(bandX))
|
|
1632
|
+
yy := m.getOperand_Mem_Imm32_Reg(m.c.ValueDefinition(bandY))
|
|
1633
|
+
test := m.allocateInstr().asCmpRmiR(false, yy, xx.reg(), bandX.Type() == ssa.TypeI64)
|
|
1634
|
+
m.insert(test)
|
|
1635
|
+
bandInstr.MarkLowered()
|
|
1636
|
+
return true
|
|
1637
|
+
}
|
|
1638
|
+
|
|
1639
|
+
func (m *machine) allocateExitInstructions(execCtx, exitCodeReg regalloc.VReg) (saveRsp, saveRbp, setExitCode *instruction) {
|
|
1640
|
+
saveRsp = m.allocateInstr().asMovRM(
|
|
1641
|
+
rspVReg,
|
|
1642
|
+
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetStackPointerBeforeGoCall.U32(), execCtx)),
|
|
1643
|
+
8,
|
|
1644
|
+
)
|
|
1645
|
+
|
|
1646
|
+
saveRbp = m.allocateInstr().asMovRM(
|
|
1647
|
+
rbpVReg,
|
|
1648
|
+
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetFramePointerBeforeGoCall.U32(), execCtx)),
|
|
1649
|
+
8,
|
|
1650
|
+
)
|
|
1651
|
+
setExitCode = m.allocateInstr().asMovRM(
|
|
1652
|
+
exitCodeReg,
|
|
1653
|
+
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetExitCodeOffset.U32(), execCtx)),
|
|
1654
|
+
4,
|
|
1655
|
+
)
|
|
1656
|
+
return
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
func (m *machine) lowerExitWithCode(execCtx regalloc.VReg, code wazevoapi.ExitCode) (afterLabel label) {
|
|
1660
|
+
exitCodeReg := rbpVReg
|
|
1661
|
+
saveRsp, saveRbp, setExitCode := m.allocateExitInstructions(execCtx, exitCodeReg)
|
|
1662
|
+
|
|
1663
|
+
// Set save RSP, RBP, and write exit code.
|
|
1664
|
+
m.insert(saveRsp)
|
|
1665
|
+
m.insert(saveRbp)
|
|
1666
|
+
m.lowerIconst(exitCodeReg, uint64(code), false)
|
|
1667
|
+
m.insert(setExitCode)
|
|
1668
|
+
|
|
1669
|
+
ripReg := rbpVReg
|
|
1670
|
+
|
|
1671
|
+
// Next is to save the current address for stack unwinding.
|
|
1672
|
+
nop, currentAddrLabel := m.allocateBrTarget()
|
|
1673
|
+
m.insert(nop)
|
|
1674
|
+
readRip := m.allocateInstr().asLEA(newOperandLabel(currentAddrLabel), ripReg)
|
|
1675
|
+
m.insert(readRip)
|
|
1676
|
+
saveRip := m.allocateInstr().asMovRM(
|
|
1677
|
+
ripReg,
|
|
1678
|
+
newOperandMem(m.newAmodeImmReg(wazevoapi.ExecutionContextOffsetGoCallReturnAddress.U32(), execCtx)),
|
|
1679
|
+
8,
|
|
1680
|
+
)
|
|
1681
|
+
m.insert(saveRip)
|
|
1682
|
+
|
|
1683
|
+
// Finally exit.
|
|
1684
|
+
exitSq := m.allocateExitSeq(execCtx)
|
|
1685
|
+
m.insert(exitSq)
|
|
1686
|
+
|
|
1687
|
+
// Return the label for continuation.
|
|
1688
|
+
continuation, afterLabel := m.allocateBrTarget()
|
|
1689
|
+
m.insert(continuation)
|
|
1690
|
+
return afterLabel
|
|
1691
|
+
}
|
|
1692
|
+
|
|
1693
|
+
func (m *machine) lowerAluRmiROp(si *ssa.Instruction, op aluRmiROpcode) {
|
|
1694
|
+
x, y := si.Arg2()
|
|
1695
|
+
if !x.Type().IsInt() {
|
|
1696
|
+
panic("BUG?")
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1699
|
+
_64 := x.Type().Bits() == 64
|
|
1700
|
+
|
|
1701
|
+
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
1702
|
+
|
|
1703
|
+
// TODO: commutative args can be swapped if one of them is an immediate.
|
|
1704
|
+
rn := m.getOperand_Reg(xDef)
|
|
1705
|
+
rm := m.getOperand_Mem_Imm32_Reg(yDef)
|
|
1706
|
+
rd := m.c.VRegOf(si.Return())
|
|
1707
|
+
|
|
1708
|
+
// rn is being overwritten, so we first copy its value to a temp register,
|
|
1709
|
+
// in case it is referenced again later.
|
|
1710
|
+
tmp := m.copyToTmp(rn.reg())
|
|
1711
|
+
|
|
1712
|
+
alu := m.allocateInstr()
|
|
1713
|
+
alu.asAluRmiR(op, rm, tmp, _64)
|
|
1714
|
+
m.insert(alu)
|
|
1715
|
+
|
|
1716
|
+
// tmp now contains the result, we copy it to the dest register.
|
|
1717
|
+
m.copyTo(tmp, rd)
|
|
1718
|
+
}
|
|
1719
|
+
|
|
1720
|
+
func (m *machine) lowerShiftR(si *ssa.Instruction, op shiftROp) {
|
|
1721
|
+
x, amt := si.Arg2()
|
|
1722
|
+
if !x.Type().IsInt() {
|
|
1723
|
+
panic("BUG?")
|
|
1724
|
+
}
|
|
1725
|
+
_64 := x.Type().Bits() == 64
|
|
1726
|
+
|
|
1727
|
+
xDef, amtDef := m.c.ValueDefinition(x), m.c.ValueDefinition(amt)
|
|
1728
|
+
|
|
1729
|
+
opAmt := m.getOperand_Imm32_Reg(amtDef)
|
|
1730
|
+
rx := m.getOperand_Reg(xDef)
|
|
1731
|
+
rd := m.c.VRegOf(si.Return())
|
|
1732
|
+
|
|
1733
|
+
// rx is being overwritten, so we first copy its value to a temp register,
|
|
1734
|
+
// in case it is referenced again later.
|
|
1735
|
+
tmpDst := m.copyToTmp(rx.reg())
|
|
1736
|
+
|
|
1737
|
+
if opAmt.kind == operandKindReg {
|
|
1738
|
+
// If opAmt is a register we must copy its value to rcx,
|
|
1739
|
+
// because shiftR encoding mandates that the shift amount is in rcx.
|
|
1740
|
+
m.copyTo(opAmt.reg(), rcxVReg)
|
|
1741
|
+
|
|
1742
|
+
alu := m.allocateInstr()
|
|
1743
|
+
alu.asShiftR(op, newOperandReg(rcxVReg), tmpDst, _64)
|
|
1744
|
+
m.insert(alu)
|
|
1745
|
+
|
|
1746
|
+
} else {
|
|
1747
|
+
alu := m.allocateInstr()
|
|
1748
|
+
alu.asShiftR(op, opAmt, tmpDst, _64)
|
|
1749
|
+
m.insert(alu)
|
|
1750
|
+
}
|
|
1751
|
+
|
|
1752
|
+
// tmp now contains the result, we copy it to the dest register.
|
|
1753
|
+
m.copyTo(tmpDst, rd)
|
|
1754
|
+
}
|
|
1755
|
+
|
|
1756
|
+
func (m *machine) lowerXmmRmR(instr *ssa.Instruction) {
|
|
1757
|
+
x, y := instr.Arg2()
|
|
1758
|
+
if !x.Type().IsFloat() {
|
|
1759
|
+
panic("BUG?")
|
|
1760
|
+
}
|
|
1761
|
+
_64 := x.Type().Bits() == 64
|
|
1762
|
+
|
|
1763
|
+
var op sseOpcode
|
|
1764
|
+
if _64 {
|
|
1765
|
+
switch instr.Opcode() {
|
|
1766
|
+
case ssa.OpcodeFadd:
|
|
1767
|
+
op = sseOpcodeAddsd
|
|
1768
|
+
case ssa.OpcodeFsub:
|
|
1769
|
+
op = sseOpcodeSubsd
|
|
1770
|
+
case ssa.OpcodeFmul:
|
|
1771
|
+
op = sseOpcodeMulsd
|
|
1772
|
+
case ssa.OpcodeFdiv:
|
|
1773
|
+
op = sseOpcodeDivsd
|
|
1774
|
+
default:
|
|
1775
|
+
panic("BUG")
|
|
1776
|
+
}
|
|
1777
|
+
} else {
|
|
1778
|
+
switch instr.Opcode() {
|
|
1779
|
+
case ssa.OpcodeFadd:
|
|
1780
|
+
op = sseOpcodeAddss
|
|
1781
|
+
case ssa.OpcodeFsub:
|
|
1782
|
+
op = sseOpcodeSubss
|
|
1783
|
+
case ssa.OpcodeFmul:
|
|
1784
|
+
op = sseOpcodeMulss
|
|
1785
|
+
case ssa.OpcodeFdiv:
|
|
1786
|
+
op = sseOpcodeDivss
|
|
1787
|
+
default:
|
|
1788
|
+
panic("BUG")
|
|
1789
|
+
}
|
|
1790
|
+
}
|
|
1791
|
+
|
|
1792
|
+
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
1793
|
+
rn := m.getOperand_Reg(yDef)
|
|
1794
|
+
rm := m.getOperand_Reg(xDef)
|
|
1795
|
+
rd := m.c.VRegOf(instr.Return())
|
|
1796
|
+
|
|
1797
|
+
// rm is being overwritten, so we first copy its value to a temp register,
|
|
1798
|
+
// in case it is referenced again later.
|
|
1799
|
+
tmp := m.copyToTmp(rm.reg())
|
|
1800
|
+
|
|
1801
|
+
xmm := m.allocateInstr().asXmmRmR(op, rn, tmp)
|
|
1802
|
+
m.insert(xmm)
|
|
1803
|
+
|
|
1804
|
+
m.copyTo(tmp, rd)
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
func (m *machine) lowerSqrt(instr *ssa.Instruction) {
|
|
1808
|
+
x := instr.Arg()
|
|
1809
|
+
if !x.Type().IsFloat() {
|
|
1810
|
+
panic("BUG")
|
|
1811
|
+
}
|
|
1812
|
+
_64 := x.Type().Bits() == 64
|
|
1813
|
+
var op sseOpcode
|
|
1814
|
+
if _64 {
|
|
1815
|
+
op = sseOpcodeSqrtsd
|
|
1816
|
+
} else {
|
|
1817
|
+
op = sseOpcodeSqrtss
|
|
1818
|
+
}
|
|
1819
|
+
|
|
1820
|
+
xDef := m.c.ValueDefinition(x)
|
|
1821
|
+
rm := m.getOperand_Mem_Reg(xDef)
|
|
1822
|
+
rd := m.c.VRegOf(instr.Return())
|
|
1823
|
+
|
|
1824
|
+
xmm := m.allocateInstr().asXmmUnaryRmR(op, rm, rd)
|
|
1825
|
+
m.insert(xmm)
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
func (m *machine) lowerFabsFneg(instr *ssa.Instruction) {
|
|
1829
|
+
x := instr.Arg()
|
|
1830
|
+
if !x.Type().IsFloat() {
|
|
1831
|
+
panic("BUG")
|
|
1832
|
+
}
|
|
1833
|
+
_64 := x.Type().Bits() == 64
|
|
1834
|
+
var op sseOpcode
|
|
1835
|
+
var mask uint64
|
|
1836
|
+
if _64 {
|
|
1837
|
+
switch instr.Opcode() {
|
|
1838
|
+
case ssa.OpcodeFabs:
|
|
1839
|
+
mask, op = 0x7fffffffffffffff, sseOpcodeAndpd
|
|
1840
|
+
case ssa.OpcodeFneg:
|
|
1841
|
+
mask, op = 0x8000000000000000, sseOpcodeXorpd
|
|
1842
|
+
}
|
|
1843
|
+
} else {
|
|
1844
|
+
switch instr.Opcode() {
|
|
1845
|
+
case ssa.OpcodeFabs:
|
|
1846
|
+
mask, op = 0x7fffffff, sseOpcodeAndps
|
|
1847
|
+
case ssa.OpcodeFneg:
|
|
1848
|
+
mask, op = 0x80000000, sseOpcodeXorps
|
|
1849
|
+
}
|
|
1850
|
+
}
|
|
1851
|
+
|
|
1852
|
+
tmp := m.c.AllocateVReg(x.Type())
|
|
1853
|
+
|
|
1854
|
+
xDef := m.c.ValueDefinition(x)
|
|
1855
|
+
rm := m.getOperand_Reg(xDef)
|
|
1856
|
+
rd := m.c.VRegOf(instr.Return())
|
|
1857
|
+
|
|
1858
|
+
m.lowerFconst(tmp, mask, _64)
|
|
1859
|
+
|
|
1860
|
+
xmm := m.allocateInstr().asXmmRmR(op, rm, tmp)
|
|
1861
|
+
m.insert(xmm)
|
|
1862
|
+
|
|
1863
|
+
m.copyTo(tmp, rd)
|
|
1864
|
+
}
|
|
1865
|
+
|
|
1866
|
+
func (m *machine) lowerStore(si *ssa.Instruction) {
|
|
1867
|
+
value, ptr, offset, storeSizeInBits := si.StoreData()
|
|
1868
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(value))
|
|
1869
|
+
mem := newOperandMem(m.lowerToAddressMode(ptr, offset))
|
|
1870
|
+
|
|
1871
|
+
store := m.allocateInstr()
|
|
1872
|
+
switch value.Type() {
|
|
1873
|
+
case ssa.TypeI32:
|
|
1874
|
+
store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
|
|
1875
|
+
case ssa.TypeI64:
|
|
1876
|
+
store.asMovRM(rm.reg(), mem, storeSizeInBits/8)
|
|
1877
|
+
case ssa.TypeF32:
|
|
1878
|
+
store.asXmmMovRM(sseOpcodeMovss, rm.reg(), mem)
|
|
1879
|
+
case ssa.TypeF64:
|
|
1880
|
+
store.asXmmMovRM(sseOpcodeMovsd, rm.reg(), mem)
|
|
1881
|
+
case ssa.TypeV128:
|
|
1882
|
+
store.asXmmMovRM(sseOpcodeMovdqu, rm.reg(), mem)
|
|
1883
|
+
default:
|
|
1884
|
+
panic("BUG")
|
|
1885
|
+
}
|
|
1886
|
+
m.insert(store)
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
func (m *machine) lowerCall(si *ssa.Instruction) {
|
|
1890
|
+
isDirectCall := si.Opcode() == ssa.OpcodeCall
|
|
1891
|
+
indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
|
|
1892
|
+
|
|
1893
|
+
if isMemmove {
|
|
1894
|
+
// Go's memmove *might* use all xmm0-xmm15, so we need to release them.
|
|
1895
|
+
// https://github.com/golang/go/blob/go1.24.0/src/cmd/compile/abi-internal.md#architecture-specifics
|
|
1896
|
+
// https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L286-L301
|
|
1897
|
+
for i := regalloc.RealReg(0); i < 16; i++ {
|
|
1898
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[xmm0+i]))
|
|
1899
|
+
}
|
|
1900
|
+
// Since Go 1.24 it may also use DX, which is not reserved for the function call's 3 args.
|
|
1901
|
+
// https://github.com/golang/go/blob/go1.24.0/src/runtime/memmove_amd64.s#L123
|
|
1902
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(regInfo.RealRegToVReg[rdx]))
|
|
1903
|
+
}
|
|
1904
|
+
|
|
1905
|
+
if isDirectCall {
|
|
1906
|
+
call := m.allocateInstr().asCall(directCallee, calleeABI)
|
|
1907
|
+
m.insert(call)
|
|
1908
|
+
} else {
|
|
1909
|
+
ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
|
|
1910
|
+
callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
|
|
1911
|
+
m.insert(callInd)
|
|
1912
|
+
}
|
|
1913
|
+
|
|
1914
|
+
if isMemmove {
|
|
1915
|
+
for i := regalloc.RealReg(0); i < 16; i++ {
|
|
1916
|
+
m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[xmm0+i]))
|
|
1917
|
+
}
|
|
1918
|
+
m.insert(m.allocateInstr().asNopUseReg(regInfo.RealRegToVReg[rdx]))
|
|
1919
|
+
}
|
|
1920
|
+
|
|
1921
|
+
m.insertReturns(si, calleeABI, stackSlotSize)
|
|
1922
|
+
}
|
|
1923
|
+
|
|
1924
|
+
func (m *machine) prepareCall(si *ssa.Instruction, isDirectCall bool) (ssa.Value, ssa.FuncRef, bool, *backend.FunctionABI, int64) {
|
|
1925
|
+
var indirectCalleePtr ssa.Value
|
|
1926
|
+
var directCallee ssa.FuncRef
|
|
1927
|
+
var sigID ssa.SignatureID
|
|
1928
|
+
var args []ssa.Value
|
|
1929
|
+
var isMemmove bool
|
|
1930
|
+
if isDirectCall {
|
|
1931
|
+
directCallee, sigID, args = si.CallData()
|
|
1932
|
+
} else {
|
|
1933
|
+
indirectCalleePtr, sigID, args, isMemmove = si.CallIndirectData()
|
|
1934
|
+
}
|
|
1935
|
+
calleeABI := m.c.GetFunctionABI(m.c.SSABuilder().ResolveSignature(sigID))
|
|
1936
|
+
|
|
1937
|
+
stackSlotSize := int64(calleeABI.AlignedArgResultStackSlotSize())
|
|
1938
|
+
if m.maxRequiredStackSizeForCalls < stackSlotSize+16 {
|
|
1939
|
+
m.maxRequiredStackSizeForCalls = stackSlotSize + 16 // 16 == return address + RBP.
|
|
1940
|
+
}
|
|
1941
|
+
|
|
1942
|
+
// Note: See machine.SetupPrologue for the stack layout.
|
|
1943
|
+
// The stack pointer decrease/increase will be inserted later in the compilation.
|
|
1944
|
+
|
|
1945
|
+
for i, arg := range args {
|
|
1946
|
+
reg := m.c.VRegOf(arg)
|
|
1947
|
+
def := m.c.ValueDefinition(arg)
|
|
1948
|
+
m.callerGenVRegToFunctionArg(calleeABI, i, reg, def, stackSlotSize)
|
|
1949
|
+
}
|
|
1950
|
+
return indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
func (m *machine) insertReturns(si *ssa.Instruction, calleeABI *backend.FunctionABI, stackSlotSize int64) {
|
|
1954
|
+
var index int
|
|
1955
|
+
r1, rs := si.Returns()
|
|
1956
|
+
if r1.Valid() {
|
|
1957
|
+
m.callerGenFunctionReturnVReg(calleeABI, 0, m.c.VRegOf(r1), stackSlotSize)
|
|
1958
|
+
index++
|
|
1959
|
+
}
|
|
1960
|
+
|
|
1961
|
+
for _, r := range rs {
|
|
1962
|
+
m.callerGenFunctionReturnVReg(calleeABI, index, m.c.VRegOf(r), stackSlotSize)
|
|
1963
|
+
index++
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
|
|
1967
|
+
func (m *machine) lowerTailCall(si *ssa.Instruction) {
|
|
1968
|
+
isDirectCall := si.Opcode() == ssa.OpcodeTailCallReturnCall
|
|
1969
|
+
indirectCalleePtr, directCallee, isMemmove, calleeABI, stackSlotSize := m.prepareCall(si, isDirectCall)
|
|
1970
|
+
if isMemmove {
|
|
1971
|
+
panic("memmove not supported in tail calls")
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
isAllRegs := stackSlotSize == 0
|
|
1975
|
+
|
|
1976
|
+
switch {
|
|
1977
|
+
case isDirectCall && isAllRegs:
|
|
1978
|
+
call := m.allocateInstr().asTailCallReturnCall(directCallee, calleeABI)
|
|
1979
|
+
m.insert(call)
|
|
1980
|
+
case !isDirectCall && isAllRegs:
|
|
1981
|
+
// In a tail call we insert the epilogue before the jump instruction,
|
|
1982
|
+
// so an arbitrary register might be overwritten while restoring the stack.
|
|
1983
|
+
// So, as compared to a regular indirect call, we ensure the pointer is stored
|
|
1984
|
+
// in a caller-saved register (r11).
|
|
1985
|
+
// For details, see internal/engine/RATIONALE.md
|
|
1986
|
+
ptrOp := m.getOperand_Reg(m.c.ValueDefinition(indirectCalleePtr))
|
|
1987
|
+
tmpJmp := r11VReg
|
|
1988
|
+
m.InsertMove(tmpJmp, ptrOp.reg(), ssa.TypeI64)
|
|
1989
|
+
callInd := m.allocateInstr().asTailCallReturnCallIndirect(newOperandReg(tmpJmp), calleeABI)
|
|
1990
|
+
m.insert(callInd)
|
|
1991
|
+
case isDirectCall && !isAllRegs:
|
|
1992
|
+
call := m.allocateInstr().asCall(directCallee, calleeABI)
|
|
1993
|
+
m.insert(call)
|
|
1994
|
+
case !isDirectCall && !isAllRegs:
|
|
1995
|
+
ptrOp := m.getOperand_Mem_Reg(m.c.ValueDefinition(indirectCalleePtr))
|
|
1996
|
+
callInd := m.allocateInstr().asCallIndirect(ptrOp, calleeABI)
|
|
1997
|
+
m.insert(callInd)
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// If this is a proper tail call, returns will be cleared in the postRegAlloc phase.
|
|
2001
|
+
m.insertReturns(si, calleeABI, stackSlotSize)
|
|
2002
|
+
}
|
|
2003
|
+
|
|
2004
|
+
// callerGenVRegToFunctionArg is the opposite of GenFunctionArgToVReg, which is used to generate the
|
|
2005
|
+
// caller side of the function call.
|
|
2006
|
+
func (m *machine) callerGenVRegToFunctionArg(a *backend.FunctionABI, argIndex int, reg regalloc.VReg, def backend.SSAValueDefinition, stackSlotSize int64) {
|
|
2007
|
+
arg := &a.Args[argIndex]
|
|
2008
|
+
if def.IsFromInstr() {
|
|
2009
|
+
// Constant instructions are inlined.
|
|
2010
|
+
if inst := def.Instr; inst.Constant() {
|
|
2011
|
+
m.insertLoadConstant(inst, reg)
|
|
2012
|
+
}
|
|
2013
|
+
}
|
|
2014
|
+
if arg.Kind == backend.ABIArgKindReg {
|
|
2015
|
+
m.InsertMove(arg.Reg, reg, arg.Type)
|
|
2016
|
+
} else {
|
|
2017
|
+
store := m.allocateInstr()
|
|
2018
|
+
mem := newOperandMem(m.newAmodeImmReg(
|
|
2019
|
+
// -stackSlotSize because the stack pointer is not yet decreased.
|
|
2020
|
+
uint32(arg.Offset-stackSlotSize), rspVReg))
|
|
2021
|
+
switch arg.Type {
|
|
2022
|
+
case ssa.TypeI32:
|
|
2023
|
+
store.asMovRM(reg, mem, 4)
|
|
2024
|
+
case ssa.TypeI64:
|
|
2025
|
+
store.asMovRM(reg, mem, 8)
|
|
2026
|
+
case ssa.TypeF32:
|
|
2027
|
+
store.asXmmMovRM(sseOpcodeMovss, reg, mem)
|
|
2028
|
+
case ssa.TypeF64:
|
|
2029
|
+
store.asXmmMovRM(sseOpcodeMovsd, reg, mem)
|
|
2030
|
+
case ssa.TypeV128:
|
|
2031
|
+
store.asXmmMovRM(sseOpcodeMovdqu, reg, mem)
|
|
2032
|
+
default:
|
|
2033
|
+
panic("BUG")
|
|
2034
|
+
}
|
|
2035
|
+
m.insert(store)
|
|
2036
|
+
}
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
func (m *machine) callerGenFunctionReturnVReg(a *backend.FunctionABI, retIndex int, reg regalloc.VReg, stackSlotSize int64) {
|
|
2040
|
+
r := &a.Rets[retIndex]
|
|
2041
|
+
if r.Kind == backend.ABIArgKindReg {
|
|
2042
|
+
m.InsertMove(reg, r.Reg, r.Type)
|
|
2043
|
+
} else {
|
|
2044
|
+
load := m.allocateInstr()
|
|
2045
|
+
mem := newOperandMem(m.newAmodeImmReg(
|
|
2046
|
+
// -stackSlotSize because the stack pointer is not yet decreased.
|
|
2047
|
+
uint32(a.ArgStackSize+r.Offset-stackSlotSize), rspVReg))
|
|
2048
|
+
switch r.Type {
|
|
2049
|
+
case ssa.TypeI32:
|
|
2050
|
+
load.asMovzxRmR(extModeLQ, mem, reg)
|
|
2051
|
+
case ssa.TypeI64:
|
|
2052
|
+
load.asMov64MR(mem, reg)
|
|
2053
|
+
case ssa.TypeF32:
|
|
2054
|
+
load.asXmmUnaryRmR(sseOpcodeMovss, mem, reg)
|
|
2055
|
+
case ssa.TypeF64:
|
|
2056
|
+
load.asXmmUnaryRmR(sseOpcodeMovsd, mem, reg)
|
|
2057
|
+
case ssa.TypeV128:
|
|
2058
|
+
load.asXmmUnaryRmR(sseOpcodeMovdqu, mem, reg)
|
|
2059
|
+
default:
|
|
2060
|
+
panic("BUG")
|
|
2061
|
+
}
|
|
2062
|
+
m.insert(load)
|
|
2063
|
+
}
|
|
2064
|
+
}
|
|
2065
|
+
|
|
2066
|
+
// InsertMove implements backend.Machine.
|
|
2067
|
+
func (m *machine) InsertMove(dst, src regalloc.VReg, typ ssa.Type) {
|
|
2068
|
+
switch typ {
|
|
2069
|
+
case ssa.TypeI32, ssa.TypeI64:
|
|
2070
|
+
i := m.allocateInstr().asMovRR(src, dst, typ.Bits() == 64)
|
|
2071
|
+
m.insert(i)
|
|
2072
|
+
case ssa.TypeF32, ssa.TypeF64, ssa.TypeV128:
|
|
2073
|
+
var op sseOpcode
|
|
2074
|
+
switch typ {
|
|
2075
|
+
case ssa.TypeF32:
|
|
2076
|
+
op = sseOpcodeMovss
|
|
2077
|
+
case ssa.TypeF64:
|
|
2078
|
+
op = sseOpcodeMovsd
|
|
2079
|
+
case ssa.TypeV128:
|
|
2080
|
+
op = sseOpcodeMovdqa
|
|
2081
|
+
}
|
|
2082
|
+
i := m.allocateInstr().asXmmUnaryRmR(op, newOperandReg(src), dst)
|
|
2083
|
+
m.insert(i)
|
|
2084
|
+
default:
|
|
2085
|
+
panic("BUG")
|
|
2086
|
+
}
|
|
2087
|
+
}
|
|
2088
|
+
|
|
2089
|
+
// Format implements backend.Machine.
|
|
2090
|
+
func (m *machine) Format() string {
|
|
2091
|
+
begins := map[*instruction]label{}
|
|
2092
|
+
for l := label(0); l < m.nextLabel; l++ {
|
|
2093
|
+
pos := m.labelPositionPool.Get(int(l))
|
|
2094
|
+
if pos != nil {
|
|
2095
|
+
begins[pos.begin] = l
|
|
2096
|
+
}
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
var lines []string
|
|
2100
|
+
for cur := m.rootInstr; cur != nil; cur = cur.next {
|
|
2101
|
+
if l, ok := begins[cur]; ok {
|
|
2102
|
+
var labelStr string
|
|
2103
|
+
if l <= m.maxSSABlockID {
|
|
2104
|
+
labelStr = fmt.Sprintf("%s (SSA Block: blk%d):", l, l)
|
|
2105
|
+
} else {
|
|
2106
|
+
labelStr = fmt.Sprintf("%s:", l)
|
|
2107
|
+
}
|
|
2108
|
+
lines = append(lines, labelStr)
|
|
2109
|
+
}
|
|
2110
|
+
if cur.kind == nop0 {
|
|
2111
|
+
continue
|
|
2112
|
+
}
|
|
2113
|
+
lines = append(lines, "\t"+cur.String())
|
|
2114
|
+
}
|
|
2115
|
+
for _, vc := range m.consts {
|
|
2116
|
+
if vc._var == nil {
|
|
2117
|
+
lines = append(lines, fmt.Sprintf("%s: const [%d %d]", vc.label, vc.lo, vc.hi))
|
|
2118
|
+
} else {
|
|
2119
|
+
lines = append(lines, fmt.Sprintf("%s: const %#x", vc.label, vc._var))
|
|
2120
|
+
}
|
|
2121
|
+
}
|
|
2122
|
+
return "\n" + strings.Join(lines, "\n") + "\n"
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
func (m *machine) encodeWithoutSSA(root *instruction) {
|
|
2126
|
+
m.labelResolutionPends = m.labelResolutionPends[:0]
|
|
2127
|
+
bufPtr := m.c.BufPtr()
|
|
2128
|
+
for cur := root; cur != nil; cur = cur.next {
|
|
2129
|
+
offset := int64(len(*bufPtr))
|
|
2130
|
+
if cur.kind == nop0 {
|
|
2131
|
+
l := cur.nop0Label()
|
|
2132
|
+
pos := m.labelPositionPool.Get(int(l))
|
|
2133
|
+
if pos != nil {
|
|
2134
|
+
pos.binaryOffset = offset
|
|
2135
|
+
}
|
|
2136
|
+
}
|
|
2137
|
+
|
|
2138
|
+
needLabelResolution := cur.encode(m.c)
|
|
2139
|
+
if needLabelResolution {
|
|
2140
|
+
m.labelResolutionPends = append(m.labelResolutionPends,
|
|
2141
|
+
labelResolutionPend{instr: cur, imm32Offset: int64(len(*bufPtr)) - 4},
|
|
2142
|
+
)
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
for i := range m.labelResolutionPends {
|
|
2147
|
+
p := &m.labelResolutionPends[i]
|
|
2148
|
+
switch p.instr.kind {
|
|
2149
|
+
case jmp, jmpIf, lea:
|
|
2150
|
+
target := p.instr.jmpLabel()
|
|
2151
|
+
targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
|
|
2152
|
+
imm32Offset := p.imm32Offset
|
|
2153
|
+
jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
|
|
2154
|
+
binary.LittleEndian.PutUint32((*bufPtr)[imm32Offset:], uint32(jmpOffset))
|
|
2155
|
+
default:
|
|
2156
|
+
panic("BUG")
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
// Encode implements backend.Machine Encode.
|
|
2162
|
+
func (m *machine) Encode(ctx context.Context) (err error) {
|
|
2163
|
+
bufPtr := m.c.BufPtr()
|
|
2164
|
+
|
|
2165
|
+
var fn string
|
|
2166
|
+
var fnIndex int
|
|
2167
|
+
var labelPosToLabel map[*labelPosition]label
|
|
2168
|
+
if wazevoapi.PerfMapEnabled {
|
|
2169
|
+
fn = wazevoapi.GetCurrentFunctionName(ctx)
|
|
2170
|
+
labelPosToLabel = make(map[*labelPosition]label)
|
|
2171
|
+
for i := 0; i <= m.labelPositionPool.MaxIDEncountered(); i++ {
|
|
2172
|
+
pos := m.labelPositionPool.Get(i)
|
|
2173
|
+
labelPosToLabel[pos] = label(i)
|
|
2174
|
+
}
|
|
2175
|
+
fnIndex = wazevoapi.GetCurrentFunctionIndex(ctx)
|
|
2176
|
+
}
|
|
2177
|
+
|
|
2178
|
+
m.labelResolutionPends = m.labelResolutionPends[:0]
|
|
2179
|
+
for _, pos := range m.orderedSSABlockLabelPos {
|
|
2180
|
+
offset := int64(len(*bufPtr))
|
|
2181
|
+
pos.binaryOffset = offset
|
|
2182
|
+
for cur := pos.begin; cur != pos.end.next; cur = cur.next {
|
|
2183
|
+
offset := int64(len(*bufPtr))
|
|
2184
|
+
|
|
2185
|
+
switch cur.kind {
|
|
2186
|
+
case nop0:
|
|
2187
|
+
l := cur.nop0Label()
|
|
2188
|
+
if pos := m.labelPositionPool.Get(int(l)); pos != nil {
|
|
2189
|
+
pos.binaryOffset = offset
|
|
2190
|
+
}
|
|
2191
|
+
case sourceOffsetInfo:
|
|
2192
|
+
m.c.AddSourceOffsetInfo(offset, cur.sourceOffsetInfo())
|
|
2193
|
+
}
|
|
2194
|
+
|
|
2195
|
+
needLabelResolution := cur.encode(m.c)
|
|
2196
|
+
if needLabelResolution {
|
|
2197
|
+
m.labelResolutionPends = append(m.labelResolutionPends,
|
|
2198
|
+
labelResolutionPend{instr: cur, instrOffset: offset, imm32Offset: int64(len(*bufPtr)) - 4},
|
|
2199
|
+
)
|
|
2200
|
+
}
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
if wazevoapi.PerfMapEnabled {
|
|
2204
|
+
l := labelPosToLabel[pos]
|
|
2205
|
+
size := int64(len(*bufPtr)) - offset
|
|
2206
|
+
wazevoapi.PerfMap.AddModuleEntry(fnIndex, offset, uint64(size), fmt.Sprintf("%s:::::%s", fn, l))
|
|
2207
|
+
}
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
for i := range m.consts {
|
|
2211
|
+
offset := int64(len(*bufPtr))
|
|
2212
|
+
vc := &m.consts[i]
|
|
2213
|
+
vc.labelPos.binaryOffset = offset
|
|
2214
|
+
if vc._var == nil {
|
|
2215
|
+
lo, hi := vc.lo, vc.hi
|
|
2216
|
+
m.c.Emit8Bytes(lo)
|
|
2217
|
+
m.c.Emit8Bytes(hi)
|
|
2218
|
+
} else {
|
|
2219
|
+
for _, b := range vc._var {
|
|
2220
|
+
m.c.EmitByte(b)
|
|
2221
|
+
}
|
|
2222
|
+
}
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
buf := *bufPtr
|
|
2226
|
+
for i := range m.labelResolutionPends {
|
|
2227
|
+
p := &m.labelResolutionPends[i]
|
|
2228
|
+
switch p.instr.kind {
|
|
2229
|
+
case jmp, jmpIf, lea, xmmUnaryRmR:
|
|
2230
|
+
target := p.instr.jmpLabel()
|
|
2231
|
+
targetOffset := m.labelPositionPool.Get(int(target)).binaryOffset
|
|
2232
|
+
imm32Offset := p.imm32Offset
|
|
2233
|
+
jmpOffset := int32(targetOffset - (p.imm32Offset + 4)) // +4 because RIP points to the next instruction.
|
|
2234
|
+
binary.LittleEndian.PutUint32(buf[imm32Offset:], uint32(jmpOffset))
|
|
2235
|
+
case jmpTableIsland:
|
|
2236
|
+
tableBegin := p.instrOffset
|
|
2237
|
+
// Each entry is the offset from the beginning of the jmpTableIsland instruction in 8 bytes.
|
|
2238
|
+
targets := m.jmpTableTargets[p.instr.u1]
|
|
2239
|
+
for i, l := range targets {
|
|
2240
|
+
targetOffset := m.labelPositionPool.Get(int(l)).binaryOffset
|
|
2241
|
+
jmpOffset := targetOffset - tableBegin
|
|
2242
|
+
binary.LittleEndian.PutUint64(buf[tableBegin+int64(i)*8:], uint64(jmpOffset))
|
|
2243
|
+
}
|
|
2244
|
+
default:
|
|
2245
|
+
panic("BUG")
|
|
2246
|
+
}
|
|
2247
|
+
}
|
|
2248
|
+
return
|
|
2249
|
+
}
|
|
2250
|
+
|
|
2251
|
+
// ResolveRelocations implements backend.Machine.
|
|
2252
|
+
func (m *machine) ResolveRelocations(refToBinaryOffset []int, _ int, binary []byte, relocations []backend.RelocationInfo, _ []int) {
|
|
2253
|
+
for _, r := range relocations {
|
|
2254
|
+
offset := r.Offset
|
|
2255
|
+
calleeFnOffset := refToBinaryOffset[r.FuncRef]
|
|
2256
|
+
// offset is the offset of the last 4 bytes of the call instruction.
|
|
2257
|
+
callInstrOffsetBytes := binary[offset : offset+4]
|
|
2258
|
+
diff := int64(calleeFnOffset) - (offset + 4) // +4 because we want the offset of the next instruction (In x64, RIP always points to the next instruction).
|
|
2259
|
+
callInstrOffsetBytes[0] = byte(diff)
|
|
2260
|
+
callInstrOffsetBytes[1] = byte(diff >> 8)
|
|
2261
|
+
callInstrOffsetBytes[2] = byte(diff >> 16)
|
|
2262
|
+
callInstrOffsetBytes[3] = byte(diff >> 24)
|
|
2263
|
+
}
|
|
2264
|
+
}
|
|
2265
|
+
|
|
2266
|
+
// CallTrampolineIslandInfo implements backend.Machine CallTrampolineIslandInfo.
|
|
2267
|
+
func (m *machine) CallTrampolineIslandInfo(_ int) (_, _ int, _ error) { return }
|
|
2268
|
+
|
|
2269
|
+
func (m *machine) lowerIcmpToFlag(xd, yd backend.SSAValueDefinition, _64 bool) {
|
|
2270
|
+
x := m.getOperand_Reg(xd)
|
|
2271
|
+
y := m.getOperand_Mem_Imm32_Reg(yd)
|
|
2272
|
+
cmp := m.allocateInstr().asCmpRmiR(true, y, x.reg(), _64)
|
|
2273
|
+
m.insert(cmp)
|
|
2274
|
+
}
|
|
2275
|
+
|
|
2276
|
+
func (m *machine) lowerFcmpToFlags(instr *ssa.Instruction) (f1, f2 cond, and bool) {
|
|
2277
|
+
x, y, c := instr.FcmpData()
|
|
2278
|
+
switch c {
|
|
2279
|
+
case ssa.FloatCmpCondEqual:
|
|
2280
|
+
f1, f2 = condNP, condZ
|
|
2281
|
+
and = true
|
|
2282
|
+
case ssa.FloatCmpCondNotEqual:
|
|
2283
|
+
f1, f2 = condP, condNZ
|
|
2284
|
+
case ssa.FloatCmpCondLessThan:
|
|
2285
|
+
f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThan)
|
|
2286
|
+
f2 = condInvalid
|
|
2287
|
+
x, y = y, x
|
|
2288
|
+
case ssa.FloatCmpCondLessThanOrEqual:
|
|
2289
|
+
f1 = condFromSSAFloatCmpCond(ssa.FloatCmpCondGreaterThanOrEqual)
|
|
2290
|
+
f2 = condInvalid
|
|
2291
|
+
x, y = y, x
|
|
2292
|
+
default:
|
|
2293
|
+
f1 = condFromSSAFloatCmpCond(c)
|
|
2294
|
+
f2 = condInvalid
|
|
2295
|
+
}
|
|
2296
|
+
|
|
2297
|
+
var opc sseOpcode
|
|
2298
|
+
if x.Type() == ssa.TypeF32 {
|
|
2299
|
+
opc = sseOpcodeUcomiss
|
|
2300
|
+
} else {
|
|
2301
|
+
opc = sseOpcodeUcomisd
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
xr := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
2305
|
+
yr := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
2306
|
+
m.insert(m.allocateInstr().asXmmCmpRmR(opc, yr, xr.reg()))
|
|
2307
|
+
return
|
|
2308
|
+
}
|
|
2309
|
+
|
|
2310
|
+
// allocateInstr allocates an instruction.
|
|
2311
|
+
func (m *machine) allocateInstr() *instruction {
|
|
2312
|
+
instr := m.instrPool.Allocate()
|
|
2313
|
+
if !m.regAllocStarted {
|
|
2314
|
+
instr.addedBeforeRegAlloc = true
|
|
2315
|
+
}
|
|
2316
|
+
return instr
|
|
2317
|
+
}
|
|
2318
|
+
|
|
2319
|
+
func (m *machine) allocateNop() *instruction {
|
|
2320
|
+
instr := m.allocateInstr()
|
|
2321
|
+
instr.kind = nop0
|
|
2322
|
+
return instr
|
|
2323
|
+
}
|
|
2324
|
+
|
|
2325
|
+
func (m *machine) insert(i *instruction) {
|
|
2326
|
+
m.pendingInstructions = append(m.pendingInstructions, i)
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
func (m *machine) allocateBrTarget() (nop *instruction, l label) { //nolint
|
|
2330
|
+
l, pos := m.allocateLabel()
|
|
2331
|
+
nop = m.allocateInstr()
|
|
2332
|
+
nop.asNop0WithLabel(l)
|
|
2333
|
+
pos.begin, pos.end = nop, nop
|
|
2334
|
+
return
|
|
2335
|
+
}
|
|
2336
|
+
|
|
2337
|
+
func (m *machine) allocateLabel() (label, *labelPosition) {
|
|
2338
|
+
l := m.nextLabel
|
|
2339
|
+
pos := m.labelPositionPool.GetOrAllocate(int(l))
|
|
2340
|
+
m.nextLabel++
|
|
2341
|
+
return l, pos
|
|
2342
|
+
}
|
|
2343
|
+
|
|
2344
|
+
func (m *machine) getVRegSpillSlotOffsetFromSP(id regalloc.VRegID, size byte) int64 {
|
|
2345
|
+
offset, ok := m.spillSlots[id]
|
|
2346
|
+
if !ok {
|
|
2347
|
+
offset = m.spillSlotSize
|
|
2348
|
+
m.spillSlots[id] = offset
|
|
2349
|
+
m.spillSlotSize += int64(size)
|
|
2350
|
+
}
|
|
2351
|
+
return offset
|
|
2352
|
+
}
|
|
2353
|
+
|
|
2354
|
+
func (m *machine) copyTo(src regalloc.VReg, dst regalloc.VReg) {
|
|
2355
|
+
mov := m.allocateInstr()
|
|
2356
|
+
if src.RegType() == regalloc.RegTypeInt {
|
|
2357
|
+
mov.asMovRR(src, dst, true)
|
|
2358
|
+
} else {
|
|
2359
|
+
mov.asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), dst)
|
|
2360
|
+
}
|
|
2361
|
+
m.insert(mov)
|
|
2362
|
+
}
|
|
2363
|
+
|
|
2364
|
+
func (m *machine) copyToTmp(v regalloc.VReg) regalloc.VReg {
|
|
2365
|
+
typ := m.c.TypeOf(v)
|
|
2366
|
+
tmp := m.c.AllocateVReg(typ)
|
|
2367
|
+
m.copyTo(v, tmp)
|
|
2368
|
+
return tmp
|
|
2369
|
+
}
|
|
2370
|
+
|
|
2371
|
+
func (m *machine) requiredStackSize() int64 {
|
|
2372
|
+
return m.maxRequiredStackSizeForCalls +
|
|
2373
|
+
m.frameSize() +
|
|
2374
|
+
16 + // Need for stack checking.
|
|
2375
|
+
16 // return address and the caller RBP.
|
|
2376
|
+
}
|
|
2377
|
+
|
|
2378
|
+
func (m *machine) frameSize() int64 {
|
|
2379
|
+
s := m.clobberedRegSlotSize() + m.spillSlotSize
|
|
2380
|
+
if s&0xf != 0 {
|
|
2381
|
+
panic(fmt.Errorf("BUG: frame size %d is not 16-byte aligned", s))
|
|
2382
|
+
}
|
|
2383
|
+
return s
|
|
2384
|
+
}
|
|
2385
|
+
|
|
2386
|
+
func (m *machine) clobberedRegSlotSize() int64 {
|
|
2387
|
+
return int64(len(m.clobberedRegs) * 16)
|
|
2388
|
+
}
|
|
2389
|
+
|
|
2390
|
+
func (m *machine) lowerIDivRem(si *ssa.Instruction, isDiv bool, signed bool) {
|
|
2391
|
+
x, y, execCtx := si.Arg3()
|
|
2392
|
+
|
|
2393
|
+
dividend := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
2394
|
+
divisor := m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
2395
|
+
ctxVReg := m.c.VRegOf(execCtx)
|
|
2396
|
+
tmpGp := m.c.AllocateVReg(si.Return().Type())
|
|
2397
|
+
|
|
2398
|
+
m.copyTo(dividend.reg(), raxVReg)
|
|
2399
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(rdxVReg))
|
|
2400
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
|
|
2401
|
+
seq := m.allocateInstr().asIdivRemSequence(ctxVReg, divisor.reg(), tmpGp, isDiv, signed, x.Type().Bits() == 64)
|
|
2402
|
+
m.insert(seq)
|
|
2403
|
+
rd := m.c.VRegOf(si.Return())
|
|
2404
|
+
if isDiv {
|
|
2405
|
+
m.copyTo(raxVReg, rd)
|
|
2406
|
+
} else {
|
|
2407
|
+
m.copyTo(rdxVReg, rd)
|
|
2408
|
+
}
|
|
2409
|
+
}
|
|
2410
|
+
|
|
2411
|
+
func (m *machine) lowerIDivRemSequenceAfterRegAlloc(i *instruction) {
|
|
2412
|
+
execCtx, divisor, tmpGp, isDiv, signed, _64 := i.idivRemSequenceData()
|
|
2413
|
+
|
|
2414
|
+
dividend := raxVReg
|
|
2415
|
+
|
|
2416
|
+
// Ensure yr is not zero.
|
|
2417
|
+
test := m.allocateInstr()
|
|
2418
|
+
test.asCmpRmiR(false, newOperandReg(divisor), divisor, _64)
|
|
2419
|
+
m.insert(test)
|
|
2420
|
+
|
|
2421
|
+
jnz := m.allocateInstr()
|
|
2422
|
+
m.insert(jnz)
|
|
2423
|
+
|
|
2424
|
+
nz := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerDivisionByZero)
|
|
2425
|
+
|
|
2426
|
+
// If not zero, we can proceed with the division.
|
|
2427
|
+
jnz.asJmpIf(condNZ, newOperandLabel(nz))
|
|
2428
|
+
|
|
2429
|
+
var ifRemNeg1 *instruction
|
|
2430
|
+
if signed {
|
|
2431
|
+
var neg1 uint64
|
|
2432
|
+
if _64 {
|
|
2433
|
+
neg1 = 0xffffffffffffffff
|
|
2434
|
+
} else {
|
|
2435
|
+
neg1 = 0xffffffff
|
|
2436
|
+
}
|
|
2437
|
+
m.lowerIconst(tmpGp, neg1, _64)
|
|
2438
|
+
|
|
2439
|
+
if isDiv {
|
|
2440
|
+
// For signed division, we have to have branches for "math.MinInt{32,64} / -1"
|
|
2441
|
+
// case which results in the floating point exception via division error as
|
|
2442
|
+
// the resulting value exceeds the maximum of signed int.
|
|
2443
|
+
|
|
2444
|
+
// First, we check if the divisor is -1.
|
|
2445
|
+
cmp := m.allocateInstr()
|
|
2446
|
+
cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
|
|
2447
|
+
m.insert(cmp)
|
|
2448
|
+
|
|
2449
|
+
ifNotNeg1 := m.allocateInstr()
|
|
2450
|
+
m.insert(ifNotNeg1)
|
|
2451
|
+
|
|
2452
|
+
var minInt uint64
|
|
2453
|
+
if _64 {
|
|
2454
|
+
minInt = 0x8000000000000000
|
|
2455
|
+
} else {
|
|
2456
|
+
minInt = 0x80000000
|
|
2457
|
+
}
|
|
2458
|
+
m.lowerIconst(tmpGp, minInt, _64)
|
|
2459
|
+
|
|
2460
|
+
// Next we check if the quotient is the most negative value for the signed integer, i.e.
|
|
2461
|
+
// if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1) respectively.
|
|
2462
|
+
cmp2 := m.allocateInstr()
|
|
2463
|
+
cmp2.asCmpRmiR(true, newOperandReg(tmpGp), dividend, _64)
|
|
2464
|
+
m.insert(cmp2)
|
|
2465
|
+
|
|
2466
|
+
ifNotMinInt := m.allocateInstr()
|
|
2467
|
+
m.insert(ifNotMinInt)
|
|
2468
|
+
|
|
2469
|
+
// Trap if we are trying to do (math.MinInt32 / -1) or (math.MinInt64 / -1),
|
|
2470
|
+
// as that is the overflow in division as the result becomes 2^31 which is larger than
|
|
2471
|
+
// the maximum of signed 32-bit int (2^31-1).
|
|
2472
|
+
end := m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
|
|
2473
|
+
ifNotNeg1.asJmpIf(condNZ, newOperandLabel(end))
|
|
2474
|
+
ifNotMinInt.asJmpIf(condNZ, newOperandLabel(end))
|
|
2475
|
+
} else {
|
|
2476
|
+
// If it is remainder, zeros DX register and compare the divisor to -1.
|
|
2477
|
+
xor := m.allocateInstr().asZeros(rdxVReg)
|
|
2478
|
+
m.insert(xor)
|
|
2479
|
+
|
|
2480
|
+
// We check if the divisor is -1.
|
|
2481
|
+
cmp := m.allocateInstr()
|
|
2482
|
+
cmp.asCmpRmiR(true, newOperandReg(tmpGp), divisor, _64)
|
|
2483
|
+
m.insert(cmp)
|
|
2484
|
+
|
|
2485
|
+
ifRemNeg1 = m.allocateInstr()
|
|
2486
|
+
m.insert(ifRemNeg1)
|
|
2487
|
+
}
|
|
2488
|
+
|
|
2489
|
+
// Sign-extend DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
|
|
2490
|
+
sed := m.allocateInstr()
|
|
2491
|
+
sed.asSignExtendData(_64)
|
|
2492
|
+
m.insert(sed)
|
|
2493
|
+
} else {
|
|
2494
|
+
// Zeros DX register to have 2*x.Type().Bits() dividend over DX and AX registers.
|
|
2495
|
+
zeros := m.allocateInstr().asZeros(rdxVReg)
|
|
2496
|
+
m.insert(zeros)
|
|
2497
|
+
}
|
|
2498
|
+
|
|
2499
|
+
div := m.allocateInstr()
|
|
2500
|
+
div.asDiv(newOperandReg(divisor), signed, _64)
|
|
2501
|
+
m.insert(div)
|
|
2502
|
+
|
|
2503
|
+
nop, end := m.allocateBrTarget()
|
|
2504
|
+
m.insert(nop)
|
|
2505
|
+
// If we are compiling a Rem instruction, when the divisor is -1 we land at the end of the function.
|
|
2506
|
+
if ifRemNeg1 != nil {
|
|
2507
|
+
ifRemNeg1.asJmpIf(condZ, newOperandLabel(end))
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
|
|
2511
|
+
func (m *machine) lowerRound(instr *ssa.Instruction, imm roundingMode) {
|
|
2512
|
+
x := instr.Arg()
|
|
2513
|
+
if !x.Type().IsFloat() {
|
|
2514
|
+
panic("BUG?")
|
|
2515
|
+
}
|
|
2516
|
+
var op sseOpcode
|
|
2517
|
+
if x.Type().Bits() == 64 {
|
|
2518
|
+
op = sseOpcodeRoundsd
|
|
2519
|
+
} else {
|
|
2520
|
+
op = sseOpcodeRoundss
|
|
2521
|
+
}
|
|
2522
|
+
|
|
2523
|
+
xDef := m.c.ValueDefinition(x)
|
|
2524
|
+
rm := m.getOperand_Mem_Reg(xDef)
|
|
2525
|
+
rd := m.c.VRegOf(instr.Return())
|
|
2526
|
+
|
|
2527
|
+
xmm := m.allocateInstr().asXmmUnaryRmRImm(op, uint8(imm), rm, rd)
|
|
2528
|
+
m.insert(xmm)
|
|
2529
|
+
}
|
|
2530
|
+
|
|
2531
|
+
func (m *machine) lowerFminFmax(instr *ssa.Instruction) {
|
|
2532
|
+
x, y := instr.Arg2()
|
|
2533
|
+
if !x.Type().IsFloat() {
|
|
2534
|
+
panic("BUG?")
|
|
2535
|
+
}
|
|
2536
|
+
|
|
2537
|
+
_64 := x.Type().Bits() == 64
|
|
2538
|
+
isMin := instr.Opcode() == ssa.OpcodeFmin
|
|
2539
|
+
var minMaxOp sseOpcode
|
|
2540
|
+
|
|
2541
|
+
switch {
|
|
2542
|
+
case _64 && isMin:
|
|
2543
|
+
minMaxOp = sseOpcodeMinpd
|
|
2544
|
+
case _64 && !isMin:
|
|
2545
|
+
minMaxOp = sseOpcodeMaxpd
|
|
2546
|
+
case !_64 && isMin:
|
|
2547
|
+
minMaxOp = sseOpcodeMinps
|
|
2548
|
+
case !_64 && !isMin:
|
|
2549
|
+
minMaxOp = sseOpcodeMaxps
|
|
2550
|
+
}
|
|
2551
|
+
|
|
2552
|
+
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
2553
|
+
rm := m.getOperand_Reg(xDef)
|
|
2554
|
+
// We cannot ensure that y is aligned to 16 bytes, so we have to use it on reg.
|
|
2555
|
+
rn := m.getOperand_Reg(yDef)
|
|
2556
|
+
rd := m.c.VRegOf(instr.Return())
|
|
2557
|
+
|
|
2558
|
+
tmp := m.copyToTmp(rm.reg())
|
|
2559
|
+
|
|
2560
|
+
// Check if this is (either x1 or x2 is NaN) or (x1 equals x2) case.
|
|
2561
|
+
cmp := m.allocateInstr()
|
|
2562
|
+
if _64 {
|
|
2563
|
+
cmp.asXmmCmpRmR(sseOpcodeUcomisd, rn, tmp)
|
|
2564
|
+
} else {
|
|
2565
|
+
cmp.asXmmCmpRmR(sseOpcodeUcomiss, rn, tmp)
|
|
2566
|
+
}
|
|
2567
|
+
m.insert(cmp)
|
|
2568
|
+
|
|
2569
|
+
// At this point, we have the three cases of conditional flags below
|
|
2570
|
+
// (See https://www.felixcloutier.com/x86/ucomiss#operation for detail.)
|
|
2571
|
+
//
|
|
2572
|
+
// 1) Two values are NaN-free and different: All flags are cleared.
|
|
2573
|
+
// 2) Two values are NaN-free and equal: Only ZF flags is set.
|
|
2574
|
+
// 3) One of Two values is NaN: ZF, PF and CF flags are set.
|
|
2575
|
+
|
|
2576
|
+
// Jump instruction to handle 1) case by checking the ZF flag
|
|
2577
|
+
// as ZF is only set for 2) and 3) cases.
|
|
2578
|
+
nanFreeOrDiffJump := m.allocateInstr()
|
|
2579
|
+
m.insert(nanFreeOrDiffJump)
|
|
2580
|
+
|
|
2581
|
+
// Start handling 2) and 3).
|
|
2582
|
+
|
|
2583
|
+
// Jump if one of two values is NaN by checking the parity flag (PF).
|
|
2584
|
+
ifIsNan := m.allocateInstr()
|
|
2585
|
+
m.insert(ifIsNan)
|
|
2586
|
+
|
|
2587
|
+
// Start handling 2) NaN-free and equal.
|
|
2588
|
+
|
|
2589
|
+
// Before we exit this case, we have to ensure that positive zero (or negative zero for min instruction) is
|
|
2590
|
+
// returned if two values are positive and negative zeros.
|
|
2591
|
+
var op sseOpcode
|
|
2592
|
+
switch {
|
|
2593
|
+
case !_64 && isMin:
|
|
2594
|
+
op = sseOpcodeOrps
|
|
2595
|
+
case _64 && isMin:
|
|
2596
|
+
op = sseOpcodeOrpd
|
|
2597
|
+
case !_64 && !isMin:
|
|
2598
|
+
op = sseOpcodeAndps
|
|
2599
|
+
case _64 && !isMin:
|
|
2600
|
+
op = sseOpcodeAndpd
|
|
2601
|
+
}
|
|
2602
|
+
orAnd := m.allocateInstr()
|
|
2603
|
+
orAnd.asXmmRmR(op, rn, tmp)
|
|
2604
|
+
m.insert(orAnd)
|
|
2605
|
+
|
|
2606
|
+
// Done, jump to end.
|
|
2607
|
+
sameExitJump := m.allocateInstr()
|
|
2608
|
+
m.insert(sameExitJump)
|
|
2609
|
+
|
|
2610
|
+
// Start handling 3) either is NaN.
|
|
2611
|
+
isNanTarget, isNan := m.allocateBrTarget()
|
|
2612
|
+
m.insert(isNanTarget)
|
|
2613
|
+
ifIsNan.asJmpIf(condP, newOperandLabel(isNan))
|
|
2614
|
+
|
|
2615
|
+
// We emit the ADD instruction to produce the NaN in tmp.
|
|
2616
|
+
add := m.allocateInstr()
|
|
2617
|
+
if _64 {
|
|
2618
|
+
add.asXmmRmR(sseOpcodeAddsd, rn, tmp)
|
|
2619
|
+
} else {
|
|
2620
|
+
add.asXmmRmR(sseOpcodeAddss, rn, tmp)
|
|
2621
|
+
}
|
|
2622
|
+
m.insert(add)
|
|
2623
|
+
|
|
2624
|
+
// Exit from the NaN case branch.
|
|
2625
|
+
nanExitJmp := m.allocateInstr()
|
|
2626
|
+
m.insert(nanExitJmp)
|
|
2627
|
+
|
|
2628
|
+
// Start handling 1).
|
|
2629
|
+
doMinMaxTarget, doMinMax := m.allocateBrTarget()
|
|
2630
|
+
m.insert(doMinMaxTarget)
|
|
2631
|
+
nanFreeOrDiffJump.asJmpIf(condNZ, newOperandLabel(doMinMax))
|
|
2632
|
+
|
|
2633
|
+
// Now handle the NaN-free and different values case.
|
|
2634
|
+
minMax := m.allocateInstr()
|
|
2635
|
+
minMax.asXmmRmR(minMaxOp, rn, tmp)
|
|
2636
|
+
m.insert(minMax)
|
|
2637
|
+
|
|
2638
|
+
endNop, end := m.allocateBrTarget()
|
|
2639
|
+
m.insert(endNop)
|
|
2640
|
+
nanExitJmp.asJmp(newOperandLabel(end))
|
|
2641
|
+
sameExitJump.asJmp(newOperandLabel(end))
|
|
2642
|
+
|
|
2643
|
+
m.copyTo(tmp, rd)
|
|
2644
|
+
}
|
|
2645
|
+
|
|
2646
|
+
func (m *machine) lowerFcopysign(instr *ssa.Instruction) {
|
|
2647
|
+
x, y := instr.Arg2()
|
|
2648
|
+
if !x.Type().IsFloat() {
|
|
2649
|
+
panic("BUG")
|
|
2650
|
+
}
|
|
2651
|
+
|
|
2652
|
+
_64 := x.Type().Bits() == 64
|
|
2653
|
+
|
|
2654
|
+
xDef, yDef := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
2655
|
+
rm := m.getOperand_Reg(xDef)
|
|
2656
|
+
rn := m.getOperand_Reg(yDef)
|
|
2657
|
+
rd := m.c.VRegOf(instr.Return())
|
|
2658
|
+
|
|
2659
|
+
// Clear the non-sign bits of src via AND with the mask.
|
|
2660
|
+
var opAnd, opOr sseOpcode
|
|
2661
|
+
var signMask uint64
|
|
2662
|
+
if _64 {
|
|
2663
|
+
signMask, opAnd, opOr = 0x8000000000000000, sseOpcodeAndpd, sseOpcodeOrpd
|
|
2664
|
+
} else {
|
|
2665
|
+
signMask, opAnd, opOr = 0x80000000, sseOpcodeAndps, sseOpcodeOrps
|
|
2666
|
+
}
|
|
2667
|
+
|
|
2668
|
+
signBitReg := m.c.AllocateVReg(x.Type())
|
|
2669
|
+
m.lowerFconst(signBitReg, signMask, _64)
|
|
2670
|
+
nonSignBitReg := m.c.AllocateVReg(x.Type())
|
|
2671
|
+
m.lowerFconst(nonSignBitReg, ^signMask, _64)
|
|
2672
|
+
|
|
2673
|
+
// Extract the sign bits of rn.
|
|
2674
|
+
and := m.allocateInstr().asXmmRmR(opAnd, rn, signBitReg)
|
|
2675
|
+
m.insert(and)
|
|
2676
|
+
|
|
2677
|
+
// Clear the sign bit of dst via AND with the non-sign bit mask.
|
|
2678
|
+
xor := m.allocateInstr().asXmmRmR(opAnd, rm, nonSignBitReg)
|
|
2679
|
+
m.insert(xor)
|
|
2680
|
+
|
|
2681
|
+
// Copy the sign bits of src to dst via OR.
|
|
2682
|
+
or := m.allocateInstr().asXmmRmR(opOr, newOperandReg(signBitReg), nonSignBitReg)
|
|
2683
|
+
m.insert(or)
|
|
2684
|
+
|
|
2685
|
+
m.copyTo(nonSignBitReg, rd)
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
func (m *machine) lowerBitcast(instr *ssa.Instruction) {
|
|
2689
|
+
x, dstTyp := instr.BitcastData()
|
|
2690
|
+
srcTyp := x.Type()
|
|
2691
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
2692
|
+
rd := m.c.VRegOf(instr.Return())
|
|
2693
|
+
switch {
|
|
2694
|
+
case srcTyp == ssa.TypeF32 && dstTyp == ssa.TypeI32:
|
|
2695
|
+
cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovd, rn.reg(), rd, false)
|
|
2696
|
+
m.insert(cvt)
|
|
2697
|
+
case srcTyp == ssa.TypeI32 && dstTyp == ssa.TypeF32:
|
|
2698
|
+
cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovd, rn, rd, false)
|
|
2699
|
+
m.insert(cvt)
|
|
2700
|
+
case srcTyp == ssa.TypeF64 && dstTyp == ssa.TypeI64:
|
|
2701
|
+
cvt := m.allocateInstr().asXmmToGpr(sseOpcodeMovq, rn.reg(), rd, true)
|
|
2702
|
+
m.insert(cvt)
|
|
2703
|
+
case srcTyp == ssa.TypeI64 && dstTyp == ssa.TypeF64:
|
|
2704
|
+
cvt := m.allocateInstr().asGprToXmm(sseOpcodeMovq, rn, rd, true)
|
|
2705
|
+
m.insert(cvt)
|
|
2706
|
+
default:
|
|
2707
|
+
panic(fmt.Sprintf("invalid bitcast from %s to %s", srcTyp, dstTyp))
|
|
2708
|
+
}
|
|
2709
|
+
}
|
|
2710
|
+
|
|
2711
|
+
func (m *machine) lowerFcvtToSint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
|
|
2712
|
+
var tmpXmm regalloc.VReg
|
|
2713
|
+
if dst64 {
|
|
2714
|
+
tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
|
|
2715
|
+
} else {
|
|
2716
|
+
tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
|
|
2717
|
+
}
|
|
2718
|
+
|
|
2719
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
|
|
2720
|
+
tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
|
|
2721
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
|
|
2722
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
|
|
2723
|
+
|
|
2724
|
+
m.insert(m.allocateFcvtToSintSequence(ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat))
|
|
2725
|
+
m.copyTo(tmpGp, rd)
|
|
2726
|
+
}
|
|
2727
|
+
|
|
2728
|
+
func (m *machine) lowerFcvtToSintSequenceAfterRegalloc(i *instruction) {
|
|
2729
|
+
execCtx, src, tmpGp, tmpGp2, tmpXmm, src64, dst64, sat := i.fcvtToSintSequenceData()
|
|
2730
|
+
var cmpOp, truncOp sseOpcode
|
|
2731
|
+
if src64 {
|
|
2732
|
+
cmpOp, truncOp = sseOpcodeUcomisd, sseOpcodeCvttsd2si
|
|
2733
|
+
} else {
|
|
2734
|
+
cmpOp, truncOp = sseOpcodeUcomiss, sseOpcodeCvttss2si
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
trunc := m.allocateInstr()
|
|
2738
|
+
trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
|
|
2739
|
+
m.insert(trunc)
|
|
2740
|
+
|
|
2741
|
+
// Check if the dst operand was INT_MIN, by checking it against 1.
|
|
2742
|
+
cmp1 := m.allocateInstr()
|
|
2743
|
+
cmp1.asCmpRmiR(true, newOperandImm32(1), tmpGp, dst64)
|
|
2744
|
+
m.insert(cmp1)
|
|
2745
|
+
|
|
2746
|
+
// If no overflow, then we are done.
|
|
2747
|
+
doneTarget, done := m.allocateBrTarget()
|
|
2748
|
+
ifNoOverflow := m.allocateInstr()
|
|
2749
|
+
ifNoOverflow.asJmpIf(condNO, newOperandLabel(done))
|
|
2750
|
+
m.insert(ifNoOverflow)
|
|
2751
|
+
|
|
2752
|
+
// Now, check for NaN.
|
|
2753
|
+
cmpNan := m.allocateInstr()
|
|
2754
|
+
cmpNan.asXmmCmpRmR(cmpOp, newOperandReg(src), src)
|
|
2755
|
+
m.insert(cmpNan)
|
|
2756
|
+
|
|
2757
|
+
// We allocate the "non-nan target" here, but we will insert it later.
|
|
2758
|
+
notNanTarget, notNaN := m.allocateBrTarget()
|
|
2759
|
+
ifNotNan := m.allocateInstr()
|
|
2760
|
+
ifNotNan.asJmpIf(condNP, newOperandLabel(notNaN))
|
|
2761
|
+
m.insert(ifNotNan)
|
|
2762
|
+
|
|
2763
|
+
if sat {
|
|
2764
|
+
// If NaN and saturating, return 0.
|
|
2765
|
+
zeroDst := m.allocateInstr().asZeros(tmpGp)
|
|
2766
|
+
m.insert(zeroDst)
|
|
2767
|
+
|
|
2768
|
+
jmpEnd := m.allocateInstr()
|
|
2769
|
+
jmpEnd.asJmp(newOperandLabel(done))
|
|
2770
|
+
m.insert(jmpEnd)
|
|
2771
|
+
|
|
2772
|
+
// Otherwise:
|
|
2773
|
+
m.insert(notNanTarget)
|
|
2774
|
+
|
|
2775
|
+
// Zero-out the tmp register.
|
|
2776
|
+
zero := m.allocateInstr().asZeros(tmpXmm)
|
|
2777
|
+
m.insert(zero)
|
|
2778
|
+
|
|
2779
|
+
cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
|
|
2780
|
+
m.insert(cmpXmm)
|
|
2781
|
+
|
|
2782
|
+
// if >= jump to end.
|
|
2783
|
+
jmpEnd2 := m.allocateInstr()
|
|
2784
|
+
jmpEnd2.asJmpIf(condB, newOperandLabel(done))
|
|
2785
|
+
m.insert(jmpEnd2)
|
|
2786
|
+
|
|
2787
|
+
// Otherwise, saturate to INT_MAX.
|
|
2788
|
+
if dst64 {
|
|
2789
|
+
m.lowerIconst(tmpGp, math.MaxInt64, dst64)
|
|
2790
|
+
} else {
|
|
2791
|
+
m.lowerIconst(tmpGp, math.MaxInt32, dst64)
|
|
2792
|
+
}
|
|
2793
|
+
|
|
2794
|
+
} else {
|
|
2795
|
+
|
|
2796
|
+
// If non-sat, NaN, trap.
|
|
2797
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
|
|
2798
|
+
|
|
2799
|
+
// Otherwise, we will jump here.
|
|
2800
|
+
m.insert(notNanTarget)
|
|
2801
|
+
|
|
2802
|
+
// jump over trap if src larger than threshold
|
|
2803
|
+
condAboveThreshold := condNB
|
|
2804
|
+
|
|
2805
|
+
// The magic constants are various combination of minInt for int[32|64] represented as float[32|64].
|
|
2806
|
+
var minInt uint64
|
|
2807
|
+
switch {
|
|
2808
|
+
case src64 && dst64:
|
|
2809
|
+
minInt = 0xc3e0000000000000
|
|
2810
|
+
case src64 && !dst64:
|
|
2811
|
+
condAboveThreshold = condNBE
|
|
2812
|
+
minInt = 0xC1E0_0000_0020_0000
|
|
2813
|
+
case !src64 && dst64:
|
|
2814
|
+
minInt = 0xDF00_0000
|
|
2815
|
+
case !src64 && !dst64:
|
|
2816
|
+
minInt = 0xCF00_0000
|
|
2817
|
+
}
|
|
2818
|
+
|
|
2819
|
+
loadToGP := m.allocateInstr().asImm(tmpGp2, minInt, src64)
|
|
2820
|
+
m.insert(loadToGP)
|
|
2821
|
+
|
|
2822
|
+
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp2), tmpXmm, src64)
|
|
2823
|
+
m.insert(movToXmm)
|
|
2824
|
+
|
|
2825
|
+
cmpXmm := m.allocateInstr().asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
|
|
2826
|
+
m.insert(cmpXmm)
|
|
2827
|
+
|
|
2828
|
+
jmpIfLarger := m.allocateInstr()
|
|
2829
|
+
checkPositiveTarget, checkPositive := m.allocateBrTarget()
|
|
2830
|
+
jmpIfLarger.asJmpIf(condAboveThreshold, newOperandLabel(checkPositive))
|
|
2831
|
+
m.insert(jmpIfLarger)
|
|
2832
|
+
|
|
2833
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
|
|
2834
|
+
|
|
2835
|
+
// If positive, it was a real overflow.
|
|
2836
|
+
m.insert(checkPositiveTarget)
|
|
2837
|
+
|
|
2838
|
+
// Zero out the temp register.
|
|
2839
|
+
xorpd := m.allocateInstr()
|
|
2840
|
+
xorpd.asXmmRmR(sseOpcodeXorpd, newOperandReg(tmpXmm), tmpXmm)
|
|
2841
|
+
m.insert(xorpd)
|
|
2842
|
+
|
|
2843
|
+
pos := m.allocateInstr()
|
|
2844
|
+
pos.asXmmCmpRmR(cmpOp, newOperandReg(src), tmpXmm)
|
|
2845
|
+
m.insert(pos)
|
|
2846
|
+
|
|
2847
|
+
// If >= jump to end.
|
|
2848
|
+
jmp := m.allocateInstr().asJmpIf(condNB, newOperandLabel(done))
|
|
2849
|
+
m.insert(jmp)
|
|
2850
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
|
|
2851
|
+
}
|
|
2852
|
+
|
|
2853
|
+
m.insert(doneTarget)
|
|
2854
|
+
}
|
|
2855
|
+
|
|
2856
|
+
func (m *machine) lowerFcvtToUint(ctxVReg, rn, rd regalloc.VReg, src64, dst64, sat bool) {
|
|
2857
|
+
tmpXmm, tmpXmm2 := m.c.AllocateVReg(ssa.TypeF64), m.c.AllocateVReg(ssa.TypeF64)
|
|
2858
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm))
|
|
2859
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpXmm2))
|
|
2860
|
+
tmpGp, tmpGp2 := m.c.AllocateVReg(ssa.TypeI64), m.c.AllocateVReg(ssa.TypeI64)
|
|
2861
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp))
|
|
2862
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpGp2))
|
|
2863
|
+
|
|
2864
|
+
m.insert(m.allocateFcvtToUintSequence(
|
|
2865
|
+
ctxVReg, rn, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat,
|
|
2866
|
+
))
|
|
2867
|
+
m.copyTo(tmpGp, rd)
|
|
2868
|
+
}
|
|
2869
|
+
|
|
2870
|
+
func (m *machine) lowerFcvtToUintSequenceAfterRegalloc(i *instruction) {
|
|
2871
|
+
execCtx, src, tmpGp, tmpGp2, tmpXmm, tmpXmm2, src64, dst64, sat := i.fcvtToUintSequenceData()
|
|
2872
|
+
|
|
2873
|
+
var subOp, cmpOp, truncOp sseOpcode
|
|
2874
|
+
if src64 {
|
|
2875
|
+
subOp, cmpOp, truncOp = sseOpcodeSubsd, sseOpcodeUcomisd, sseOpcodeCvttsd2si
|
|
2876
|
+
} else {
|
|
2877
|
+
subOp, cmpOp, truncOp = sseOpcodeSubss, sseOpcodeUcomiss, sseOpcodeCvttss2si
|
|
2878
|
+
}
|
|
2879
|
+
|
|
2880
|
+
doneTarget, done := m.allocateBrTarget()
|
|
2881
|
+
|
|
2882
|
+
switch {
|
|
2883
|
+
case src64 && dst64:
|
|
2884
|
+
loadToGP := m.allocateInstr().asImm(tmpGp, 0x43e0000000000000, true)
|
|
2885
|
+
m.insert(loadToGP)
|
|
2886
|
+
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
|
|
2887
|
+
m.insert(movToXmm)
|
|
2888
|
+
case src64 && !dst64:
|
|
2889
|
+
loadToGP := m.allocateInstr().asImm(tmpGp, 0x41e0000000000000, true)
|
|
2890
|
+
m.insert(loadToGP)
|
|
2891
|
+
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, true)
|
|
2892
|
+
m.insert(movToXmm)
|
|
2893
|
+
case !src64 && dst64:
|
|
2894
|
+
loadToGP := m.allocateInstr().asImm(tmpGp, 0x5f000000, false)
|
|
2895
|
+
m.insert(loadToGP)
|
|
2896
|
+
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
|
|
2897
|
+
m.insert(movToXmm)
|
|
2898
|
+
case !src64 && !dst64:
|
|
2899
|
+
loadToGP := m.allocateInstr().asImm(tmpGp, 0x4f000000, false)
|
|
2900
|
+
m.insert(loadToGP)
|
|
2901
|
+
movToXmm := m.allocateInstr().asGprToXmm(sseOpcodeMovq, newOperandReg(tmpGp), tmpXmm, false)
|
|
2902
|
+
m.insert(movToXmm)
|
|
2903
|
+
}
|
|
2904
|
+
|
|
2905
|
+
cmp := m.allocateInstr()
|
|
2906
|
+
cmp.asXmmCmpRmR(cmpOp, newOperandReg(tmpXmm), src)
|
|
2907
|
+
m.insert(cmp)
|
|
2908
|
+
|
|
2909
|
+
// If above `tmp` ("large threshold"), jump to `ifAboveThreshold`
|
|
2910
|
+
ifAboveThresholdTarget, ifAboveThreshold := m.allocateBrTarget()
|
|
2911
|
+
jmpIfAboveThreshold := m.allocateInstr()
|
|
2912
|
+
jmpIfAboveThreshold.asJmpIf(condNB, newOperandLabel(ifAboveThreshold))
|
|
2913
|
+
m.insert(jmpIfAboveThreshold)
|
|
2914
|
+
|
|
2915
|
+
ifNotNaNTarget, ifNotNaN := m.allocateBrTarget()
|
|
2916
|
+
jmpIfNotNaN := m.allocateInstr()
|
|
2917
|
+
jmpIfNotNaN.asJmpIf(condNP, newOperandLabel(ifNotNaN))
|
|
2918
|
+
m.insert(jmpIfNotNaN)
|
|
2919
|
+
|
|
2920
|
+
// If NaN, handle the error condition.
|
|
2921
|
+
if sat {
|
|
2922
|
+
// On NaN, saturating, we just return 0.
|
|
2923
|
+
zeros := m.allocateInstr().asZeros(tmpGp)
|
|
2924
|
+
m.insert(zeros)
|
|
2925
|
+
|
|
2926
|
+
jmpEnd := m.allocateInstr()
|
|
2927
|
+
jmpEnd.asJmp(newOperandLabel(done))
|
|
2928
|
+
m.insert(jmpEnd)
|
|
2929
|
+
} else {
|
|
2930
|
+
// On NaN, non-saturating, we trap.
|
|
2931
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeInvalidConversionToInteger)
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
// If not NaN, land here.
|
|
2935
|
+
m.insert(ifNotNaNTarget)
|
|
2936
|
+
|
|
2937
|
+
// Truncation happens here.
|
|
2938
|
+
|
|
2939
|
+
trunc := m.allocateInstr()
|
|
2940
|
+
trunc.asXmmToGpr(truncOp, src, tmpGp, dst64)
|
|
2941
|
+
m.insert(trunc)
|
|
2942
|
+
|
|
2943
|
+
// Check if the result is negative.
|
|
2944
|
+
cmpNeg := m.allocateInstr()
|
|
2945
|
+
cmpNeg.asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
|
|
2946
|
+
m.insert(cmpNeg)
|
|
2947
|
+
|
|
2948
|
+
// If non-neg, jump to end.
|
|
2949
|
+
jmpIfNonNeg := m.allocateInstr()
|
|
2950
|
+
jmpIfNonNeg.asJmpIf(condNL, newOperandLabel(done))
|
|
2951
|
+
m.insert(jmpIfNonNeg)
|
|
2952
|
+
|
|
2953
|
+
if sat {
|
|
2954
|
+
// If the input was "small" (< 2**(width -1)), the only way to get an integer
|
|
2955
|
+
// overflow is because the input was too small: saturate to the min value, i.e. 0.
|
|
2956
|
+
zeros := m.allocateInstr().asZeros(tmpGp)
|
|
2957
|
+
m.insert(zeros)
|
|
2958
|
+
|
|
2959
|
+
jmpEnd := m.allocateInstr()
|
|
2960
|
+
jmpEnd.asJmp(newOperandLabel(done))
|
|
2961
|
+
m.insert(jmpEnd)
|
|
2962
|
+
} else {
|
|
2963
|
+
// If not saturating, trap.
|
|
2964
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
|
|
2965
|
+
}
|
|
2966
|
+
|
|
2967
|
+
// If above the threshold, land here.
|
|
2968
|
+
m.insert(ifAboveThresholdTarget)
|
|
2969
|
+
|
|
2970
|
+
// tmpDiff := threshold - rn.
|
|
2971
|
+
copySrc := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandReg(src), tmpXmm2)
|
|
2972
|
+
m.insert(copySrc)
|
|
2973
|
+
|
|
2974
|
+
sub := m.allocateInstr()
|
|
2975
|
+
sub.asXmmRmR(subOp, newOperandReg(tmpXmm), tmpXmm2) // must be -0x8000000000000000
|
|
2976
|
+
m.insert(sub)
|
|
2977
|
+
|
|
2978
|
+
trunc2 := m.allocateInstr()
|
|
2979
|
+
trunc2.asXmmToGpr(truncOp, tmpXmm2, tmpGp, dst64)
|
|
2980
|
+
m.insert(trunc2)
|
|
2981
|
+
|
|
2982
|
+
// Check if the result is negative.
|
|
2983
|
+
cmpNeg2 := m.allocateInstr().asCmpRmiR(true, newOperandImm32(0), tmpGp, dst64)
|
|
2984
|
+
m.insert(cmpNeg2)
|
|
2985
|
+
|
|
2986
|
+
ifNextLargeTarget, ifNextLarge := m.allocateBrTarget()
|
|
2987
|
+
jmpIfNextLarge := m.allocateInstr()
|
|
2988
|
+
jmpIfNextLarge.asJmpIf(condNL, newOperandLabel(ifNextLarge))
|
|
2989
|
+
m.insert(jmpIfNextLarge)
|
|
2990
|
+
|
|
2991
|
+
if sat {
|
|
2992
|
+
// The input was "large" (>= maxInt), so the only way to get an integer
|
|
2993
|
+
// overflow is because the input was too large: saturate to the max value.
|
|
2994
|
+
var maxInt uint64
|
|
2995
|
+
if dst64 {
|
|
2996
|
+
maxInt = math.MaxUint64
|
|
2997
|
+
} else {
|
|
2998
|
+
maxInt = math.MaxUint32
|
|
2999
|
+
}
|
|
3000
|
+
m.lowerIconst(tmpGp, maxInt, dst64)
|
|
3001
|
+
|
|
3002
|
+
jmpToEnd := m.allocateInstr()
|
|
3003
|
+
jmpToEnd.asJmp(newOperandLabel(done))
|
|
3004
|
+
m.insert(jmpToEnd)
|
|
3005
|
+
} else {
|
|
3006
|
+
// If not saturating, trap.
|
|
3007
|
+
m.lowerExitWithCode(execCtx, wazevoapi.ExitCodeIntegerOverflow)
|
|
3008
|
+
}
|
|
3009
|
+
|
|
3010
|
+
m.insert(ifNextLargeTarget)
|
|
3011
|
+
|
|
3012
|
+
var op operand
|
|
3013
|
+
if dst64 {
|
|
3014
|
+
m.lowerIconst(tmpGp2, 0x8000000000000000, true)
|
|
3015
|
+
op = newOperandReg(tmpGp2)
|
|
3016
|
+
} else {
|
|
3017
|
+
op = newOperandImm32(0x80000000)
|
|
3018
|
+
}
|
|
3019
|
+
|
|
3020
|
+
add := m.allocateInstr()
|
|
3021
|
+
add.asAluRmiR(aluRmiROpcodeAdd, op, tmpGp, dst64)
|
|
3022
|
+
m.insert(add)
|
|
3023
|
+
|
|
3024
|
+
m.insert(doneTarget)
|
|
3025
|
+
}
|
|
3026
|
+
|
|
3027
|
+
func (m *machine) lowerFcvtFromSint(rn, rd operand, src64, dst64 bool) {
|
|
3028
|
+
var op sseOpcode
|
|
3029
|
+
if dst64 {
|
|
3030
|
+
op = sseOpcodeCvtsi2sd
|
|
3031
|
+
} else {
|
|
3032
|
+
op = sseOpcodeCvtsi2ss
|
|
3033
|
+
}
|
|
3034
|
+
|
|
3035
|
+
trunc := m.allocateInstr()
|
|
3036
|
+
trunc.asGprToXmm(op, rn, rd.reg(), src64)
|
|
3037
|
+
m.insert(trunc)
|
|
3038
|
+
}
|
|
3039
|
+
|
|
3040
|
+
func (m *machine) lowerFcvtFromUint(rn, rd operand, src64, dst64 bool) {
|
|
3041
|
+
var op sseOpcode
|
|
3042
|
+
if dst64 {
|
|
3043
|
+
op = sseOpcodeCvtsi2sd
|
|
3044
|
+
} else {
|
|
3045
|
+
op = sseOpcodeCvtsi2ss
|
|
3046
|
+
}
|
|
3047
|
+
|
|
3048
|
+
// Src is 32 bit, then we just perform the conversion with 64 bit width.
|
|
3049
|
+
//
|
|
3050
|
+
// See the following link for why we use 64bit conversion for unsigned 32bit integer sources:
|
|
3051
|
+
// https://stackoverflow.com/questions/41495498/fpu-operations-generated-by-gcc-during-casting-integer-to-float.
|
|
3052
|
+
//
|
|
3053
|
+
// Here's the summary:
|
|
3054
|
+
// >> CVTSI2SS is indeed designed for converting a signed integer to a scalar single-precision float,
|
|
3055
|
+
// >> not an unsigned integer like you have here. So what gives? Well, a 64-bit processor has 64-bit wide
|
|
3056
|
+
// >> registers available, so the unsigned 32-bit input values can be stored as signed 64-bit intermediate values,
|
|
3057
|
+
// >> which allows CVTSI2SS to be used after all.
|
|
3058
|
+
//
|
|
3059
|
+
if !src64 {
|
|
3060
|
+
// Before we convert, we have to clear the higher 32-bits of the 64-bit register
|
|
3061
|
+
// to get the correct result.
|
|
3062
|
+
tmp := m.c.AllocateVReg(ssa.TypeI32)
|
|
3063
|
+
m.insert(m.allocateInstr().asMovzxRmR(extModeLQ, rn, tmp))
|
|
3064
|
+
m.insert(m.allocateInstr().asGprToXmm(op, newOperandReg(tmp), rd.reg(), true))
|
|
3065
|
+
return
|
|
3066
|
+
}
|
|
3067
|
+
|
|
3068
|
+
// If uint64, we have to do a bit more work.
|
|
3069
|
+
endTarget, end := m.allocateBrTarget()
|
|
3070
|
+
|
|
3071
|
+
var tmpXmm regalloc.VReg
|
|
3072
|
+
if dst64 {
|
|
3073
|
+
tmpXmm = m.c.AllocateVReg(ssa.TypeF64)
|
|
3074
|
+
} else {
|
|
3075
|
+
tmpXmm = m.c.AllocateVReg(ssa.TypeF32)
|
|
3076
|
+
}
|
|
3077
|
+
|
|
3078
|
+
// Check if the most significant bit (sign bit) is set.
|
|
3079
|
+
test := m.allocateInstr()
|
|
3080
|
+
test.asCmpRmiR(false, rn, rn.reg(), src64)
|
|
3081
|
+
m.insert(test)
|
|
3082
|
+
|
|
3083
|
+
// Jump if the sign bit is set.
|
|
3084
|
+
ifSignTarget, ifSign := m.allocateBrTarget()
|
|
3085
|
+
jmpIfNeg := m.allocateInstr()
|
|
3086
|
+
jmpIfNeg.asJmpIf(condS, newOperandLabel(ifSign))
|
|
3087
|
+
m.insert(jmpIfNeg)
|
|
3088
|
+
|
|
3089
|
+
// If the sign bit is not set, we could fit the unsigned int into float32/float64.
|
|
3090
|
+
// So, we convert it to float and emit jump instruction to exit from this branch.
|
|
3091
|
+
cvt := m.allocateInstr()
|
|
3092
|
+
cvt.asGprToXmm(op, rn, tmpXmm, src64)
|
|
3093
|
+
m.insert(cvt)
|
|
3094
|
+
|
|
3095
|
+
// We are done, jump to end.
|
|
3096
|
+
jmpEnd := m.allocateInstr()
|
|
3097
|
+
jmpEnd.asJmp(newOperandLabel(end))
|
|
3098
|
+
m.insert(jmpEnd)
|
|
3099
|
+
|
|
3100
|
+
// Now handling the case where sign-bit is set.
|
|
3101
|
+
// We emit the following sequences:
|
|
3102
|
+
// mov %rn, %tmp
|
|
3103
|
+
// shr 1, %tmp
|
|
3104
|
+
// mov %rn, %tmp2
|
|
3105
|
+
// and 1, %tmp2
|
|
3106
|
+
// or %tmp2, %tmp
|
|
3107
|
+
// cvtsi2ss %tmp, %xmm0
|
|
3108
|
+
// addsd %xmm0, %xmm0
|
|
3109
|
+
m.insert(ifSignTarget)
|
|
3110
|
+
|
|
3111
|
+
tmp := m.copyToTmp(rn.reg())
|
|
3112
|
+
shr := m.allocateInstr()
|
|
3113
|
+
shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(1), tmp, src64)
|
|
3114
|
+
m.insert(shr)
|
|
3115
|
+
|
|
3116
|
+
tmp2 := m.copyToTmp(rn.reg())
|
|
3117
|
+
and := m.allocateInstr()
|
|
3118
|
+
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, src64)
|
|
3119
|
+
m.insert(and)
|
|
3120
|
+
|
|
3121
|
+
or := m.allocateInstr()
|
|
3122
|
+
or.asAluRmiR(aluRmiROpcodeOr, newOperandReg(tmp2), tmp, src64)
|
|
3123
|
+
m.insert(or)
|
|
3124
|
+
|
|
3125
|
+
cvt2 := m.allocateInstr()
|
|
3126
|
+
cvt2.asGprToXmm(op, newOperandReg(tmp), tmpXmm, src64)
|
|
3127
|
+
m.insert(cvt2)
|
|
3128
|
+
|
|
3129
|
+
addsd := m.allocateInstr()
|
|
3130
|
+
if dst64 {
|
|
3131
|
+
addsd.asXmmRmR(sseOpcodeAddsd, newOperandReg(tmpXmm), tmpXmm)
|
|
3132
|
+
} else {
|
|
3133
|
+
addsd.asXmmRmR(sseOpcodeAddss, newOperandReg(tmpXmm), tmpXmm)
|
|
3134
|
+
}
|
|
3135
|
+
m.insert(addsd)
|
|
3136
|
+
|
|
3137
|
+
m.insert(endTarget)
|
|
3138
|
+
m.copyTo(tmpXmm, rd.reg())
|
|
3139
|
+
}
|
|
3140
|
+
|
|
3141
|
+
func (m *machine) lowerVanyTrue(instr *ssa.Instruction) {
|
|
3142
|
+
x := instr.Arg()
|
|
3143
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3144
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3145
|
+
|
|
3146
|
+
tmp := m.c.AllocateVReg(ssa.TypeI32)
|
|
3147
|
+
|
|
3148
|
+
cmp := m.allocateInstr()
|
|
3149
|
+
cmp.asXmmCmpRmR(sseOpcodePtest, rm, rm.reg())
|
|
3150
|
+
m.insert(cmp)
|
|
3151
|
+
|
|
3152
|
+
setcc := m.allocateInstr()
|
|
3153
|
+
setcc.asSetcc(condNZ, tmp)
|
|
3154
|
+
m.insert(setcc)
|
|
3155
|
+
|
|
3156
|
+
// Clear the irrelevant bits.
|
|
3157
|
+
and := m.allocateInstr()
|
|
3158
|
+
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp, false)
|
|
3159
|
+
m.insert(and)
|
|
3160
|
+
|
|
3161
|
+
m.copyTo(tmp, rd)
|
|
3162
|
+
}
|
|
3163
|
+
|
|
3164
|
+
func (m *machine) lowerVallTrue(instr *ssa.Instruction) {
|
|
3165
|
+
x, lane := instr.ArgWithLane()
|
|
3166
|
+
var op sseOpcode
|
|
3167
|
+
switch lane {
|
|
3168
|
+
case ssa.VecLaneI8x16:
|
|
3169
|
+
op = sseOpcodePcmpeqb
|
|
3170
|
+
case ssa.VecLaneI16x8:
|
|
3171
|
+
op = sseOpcodePcmpeqw
|
|
3172
|
+
case ssa.VecLaneI32x4:
|
|
3173
|
+
op = sseOpcodePcmpeqd
|
|
3174
|
+
case ssa.VecLaneI64x2:
|
|
3175
|
+
op = sseOpcodePcmpeqq
|
|
3176
|
+
}
|
|
3177
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3178
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3179
|
+
|
|
3180
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3181
|
+
|
|
3182
|
+
zeros := m.allocateInstr()
|
|
3183
|
+
zeros.asZeros(tmp)
|
|
3184
|
+
m.insert(zeros)
|
|
3185
|
+
|
|
3186
|
+
pcmp := m.allocateInstr()
|
|
3187
|
+
pcmp.asXmmRmR(op, rm, tmp)
|
|
3188
|
+
m.insert(pcmp)
|
|
3189
|
+
|
|
3190
|
+
test := m.allocateInstr()
|
|
3191
|
+
test.asXmmCmpRmR(sseOpcodePtest, newOperandReg(tmp), tmp)
|
|
3192
|
+
m.insert(test)
|
|
3193
|
+
|
|
3194
|
+
tmp2 := m.c.AllocateVReg(ssa.TypeI32)
|
|
3195
|
+
|
|
3196
|
+
setcc := m.allocateInstr()
|
|
3197
|
+
setcc.asSetcc(condZ, tmp2)
|
|
3198
|
+
m.insert(setcc)
|
|
3199
|
+
|
|
3200
|
+
// Clear the irrelevant bits.
|
|
3201
|
+
and := m.allocateInstr()
|
|
3202
|
+
and.asAluRmiR(aluRmiROpcodeAnd, newOperandImm32(1), tmp2, false)
|
|
3203
|
+
m.insert(and)
|
|
3204
|
+
|
|
3205
|
+
m.copyTo(tmp2, rd)
|
|
3206
|
+
}
|
|
3207
|
+
|
|
3208
|
+
func (m *machine) lowerVhighBits(instr *ssa.Instruction) {
|
|
3209
|
+
x, lane := instr.ArgWithLane()
|
|
3210
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3211
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3212
|
+
switch lane {
|
|
3213
|
+
case ssa.VecLaneI8x16:
|
|
3214
|
+
mov := m.allocateInstr()
|
|
3215
|
+
mov.asXmmToGpr(sseOpcodePmovmskb, rm.reg(), rd, false)
|
|
3216
|
+
m.insert(mov)
|
|
3217
|
+
|
|
3218
|
+
case ssa.VecLaneI16x8:
|
|
3219
|
+
// When we have:
|
|
3220
|
+
// R1 = [R1(w1), R1(w2), R1(w3), R1(w4), R1(w5), R1(w6), R1(w7), R1(v8)]
|
|
3221
|
+
// R2 = [R2(w1), R2(w2), R2(w3), R2(v4), R2(w5), R2(w6), R2(w7), R2(v8)]
|
|
3222
|
+
// where RX(wn) is n-th signed word (16-bit) of RX register,
|
|
3223
|
+
//
|
|
3224
|
+
// "PACKSSWB R1, R2" produces
|
|
3225
|
+
// R1 = [
|
|
3226
|
+
// byte_sat(R1(w1)), byte_sat(R1(w2)), byte_sat(R1(w3)), byte_sat(R1(w4)),
|
|
3227
|
+
// byte_sat(R1(w5)), byte_sat(R1(w6)), byte_sat(R1(w7)), byte_sat(R1(w8)),
|
|
3228
|
+
// byte_sat(R2(w1)), byte_sat(R2(w2)), byte_sat(R2(w3)), byte_sat(R2(w4)),
|
|
3229
|
+
// byte_sat(R2(w5)), byte_sat(R2(w6)), byte_sat(R2(w7)), byte_sat(R2(w8)),
|
|
3230
|
+
// ]
|
|
3231
|
+
// where R1 is the destination register, and
|
|
3232
|
+
// byte_sat(w) = int8(w) if w fits as signed 8-bit,
|
|
3233
|
+
// 0x80 if w is less than 0x80
|
|
3234
|
+
// 0x7F if w is greater than 0x7f
|
|
3235
|
+
//
|
|
3236
|
+
// See https://www.felixcloutier.com/x86/packsswb:packssdw for detail.
|
|
3237
|
+
//
|
|
3238
|
+
// Therefore, v.register ends up having i-th and (i+8)-th bit set if i-th lane is negative (for i in 0..8).
|
|
3239
|
+
tmp := m.copyToTmp(rm.reg())
|
|
3240
|
+
res := m.c.AllocateVReg(ssa.TypeI32)
|
|
3241
|
+
|
|
3242
|
+
pak := m.allocateInstr()
|
|
3243
|
+
pak.asXmmRmR(sseOpcodePacksswb, rm, tmp)
|
|
3244
|
+
m.insert(pak)
|
|
3245
|
+
|
|
3246
|
+
mov := m.allocateInstr()
|
|
3247
|
+
mov.asXmmToGpr(sseOpcodePmovmskb, tmp, res, false)
|
|
3248
|
+
m.insert(mov)
|
|
3249
|
+
|
|
3250
|
+
// Clear the higher bits than 8.
|
|
3251
|
+
shr := m.allocateInstr()
|
|
3252
|
+
shr.asShiftR(shiftROpShiftRightLogical, newOperandImm32(8), res, false)
|
|
3253
|
+
m.insert(shr)
|
|
3254
|
+
|
|
3255
|
+
m.copyTo(res, rd)
|
|
3256
|
+
|
|
3257
|
+
case ssa.VecLaneI32x4:
|
|
3258
|
+
mov := m.allocateInstr()
|
|
3259
|
+
mov.asXmmToGpr(sseOpcodeMovmskps, rm.reg(), rd, true)
|
|
3260
|
+
m.insert(mov)
|
|
3261
|
+
|
|
3262
|
+
case ssa.VecLaneI64x2:
|
|
3263
|
+
mov := m.allocateInstr()
|
|
3264
|
+
mov.asXmmToGpr(sseOpcodeMovmskpd, rm.reg(), rd, true)
|
|
3265
|
+
m.insert(mov)
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
|
|
3269
|
+
func (m *machine) lowerVbnot(instr *ssa.Instruction) {
|
|
3270
|
+
x := instr.Arg()
|
|
3271
|
+
xDef := m.c.ValueDefinition(x)
|
|
3272
|
+
rm := m.getOperand_Reg(xDef)
|
|
3273
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3274
|
+
|
|
3275
|
+
tmp := m.copyToTmp(rm.reg())
|
|
3276
|
+
tmp2 := m.c.AllocateVReg(ssa.TypeV128)
|
|
3277
|
+
|
|
3278
|
+
// Ensure tmp2 is considered defined by regalloc.
|
|
3279
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
|
|
3280
|
+
|
|
3281
|
+
// Set all bits on tmp register.
|
|
3282
|
+
pak := m.allocateInstr()
|
|
3283
|
+
pak.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp2), tmp2)
|
|
3284
|
+
m.insert(pak)
|
|
3285
|
+
|
|
3286
|
+
// Then XOR with tmp to reverse all bits on v.register.
|
|
3287
|
+
xor := m.allocateInstr()
|
|
3288
|
+
xor.asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp)
|
|
3289
|
+
m.insert(xor)
|
|
3290
|
+
|
|
3291
|
+
m.copyTo(tmp, rd)
|
|
3292
|
+
}
|
|
3293
|
+
|
|
3294
|
+
func (m *machine) lowerSplat(x, ret ssa.Value, lane ssa.VecLane) {
|
|
3295
|
+
tmpDst := m.c.AllocateVReg(ssa.TypeV128)
|
|
3296
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmpDst))
|
|
3297
|
+
|
|
3298
|
+
switch lane {
|
|
3299
|
+
case ssa.VecLaneI8x16:
|
|
3300
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3301
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp))
|
|
3302
|
+
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3303
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrb, 0, xx, tmpDst))
|
|
3304
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp), tmp))
|
|
3305
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpDst))
|
|
3306
|
+
case ssa.VecLaneI16x8:
|
|
3307
|
+
xx := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3308
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 0, xx, tmpDst))
|
|
3309
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrw, 1, xx, tmpDst))
|
|
3310
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
|
|
3311
|
+
case ssa.VecLaneI32x4:
|
|
3312
|
+
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3313
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrd, 0, xx, tmpDst))
|
|
3314
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
|
|
3315
|
+
case ssa.VecLaneI64x2:
|
|
3316
|
+
xx := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3317
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 0, xx, tmpDst))
|
|
3318
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePinsrq, 1, xx, tmpDst))
|
|
3319
|
+
case ssa.VecLaneF32x4:
|
|
3320
|
+
xx := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3321
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodeInsertps, 0, xx, tmpDst))
|
|
3322
|
+
m.insert(m.allocateInstr().asXmmRmRImm(sseOpcodePshufd, 0, newOperandReg(tmpDst), tmpDst))
|
|
3323
|
+
case ssa.VecLaneF64x2:
|
|
3324
|
+
xx := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3325
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovsd, xx, tmpDst))
|
|
3326
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeMovlhps, xx, tmpDst))
|
|
3327
|
+
default:
|
|
3328
|
+
panic(fmt.Sprintf("invalid lane type: %s", lane))
|
|
3329
|
+
}
|
|
3330
|
+
|
|
3331
|
+
m.copyTo(tmpDst, m.c.VRegOf(ret))
|
|
3332
|
+
}
|
|
3333
|
+
|
|
3334
|
+
func (m *machine) lowerShuffle(x, y ssa.Value, lo, hi uint64, ret ssa.Value) {
|
|
3335
|
+
var xMask, yMask [2]uint64
|
|
3336
|
+
for i := 0; i < 8; i++ {
|
|
3337
|
+
loLane := byte(lo >> (i * 8))
|
|
3338
|
+
if loLane < 16 {
|
|
3339
|
+
xMask[0] |= uint64(loLane) << (i * 8)
|
|
3340
|
+
yMask[0] |= uint64(0x80) << (i * 8)
|
|
3341
|
+
} else {
|
|
3342
|
+
xMask[0] |= uint64(0x80) << (i * 8)
|
|
3343
|
+
yMask[0] |= uint64(loLane-16) << (i * 8)
|
|
3344
|
+
}
|
|
3345
|
+
hiLane := byte(hi >> (i * 8))
|
|
3346
|
+
if hiLane < 16 {
|
|
3347
|
+
xMask[1] |= uint64(hiLane) << (i * 8)
|
|
3348
|
+
yMask[1] |= uint64(0x80) << (i * 8)
|
|
3349
|
+
} else {
|
|
3350
|
+
xMask[1] |= uint64(0x80) << (i * 8)
|
|
3351
|
+
yMask[1] |= uint64(hiLane-16) << (i * 8)
|
|
3352
|
+
}
|
|
3353
|
+
}
|
|
3354
|
+
|
|
3355
|
+
xl, xmaskPos := m.allocateLabel()
|
|
3356
|
+
m.consts = append(m.consts, _const{lo: xMask[0], hi: xMask[1], label: xl, labelPos: xmaskPos})
|
|
3357
|
+
yl, ymaskPos := m.allocateLabel()
|
|
3358
|
+
m.consts = append(m.consts, _const{lo: yMask[0], hi: yMask[1], label: yl, labelPos: ymaskPos})
|
|
3359
|
+
|
|
3360
|
+
xx, yy := m.getOperand_Reg(m.c.ValueDefinition(x)), m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
3361
|
+
tmpX, tmpY := m.copyToTmp(xx.reg()), m.copyToTmp(yy.reg())
|
|
3362
|
+
|
|
3363
|
+
// Apply mask to X.
|
|
3364
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3365
|
+
loadMaskLo := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(xl)), tmp)
|
|
3366
|
+
m.insert(loadMaskLo)
|
|
3367
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpX))
|
|
3368
|
+
|
|
3369
|
+
// Apply mask to Y.
|
|
3370
|
+
loadMaskHi := m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, newOperandMem(m.newAmodeRipRel(yl)), tmp)
|
|
3371
|
+
m.insert(loadMaskHi)
|
|
3372
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePshufb, newOperandReg(tmp), tmpY))
|
|
3373
|
+
|
|
3374
|
+
// Combine the results.
|
|
3375
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodeOrps, newOperandReg(tmpX), tmpY))
|
|
3376
|
+
|
|
3377
|
+
m.copyTo(tmpY, m.c.VRegOf(ret))
|
|
3378
|
+
}
|
|
3379
|
+
|
|
3380
|
+
func (m *machine) lowerVbBinOpUnaligned(op sseOpcode, x, y, ret ssa.Value) {
|
|
3381
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3382
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
3383
|
+
rd := m.c.VRegOf(ret)
|
|
3384
|
+
|
|
3385
|
+
tmp := m.copyToTmp(rn.reg())
|
|
3386
|
+
|
|
3387
|
+
binOp := m.allocateInstr()
|
|
3388
|
+
binOp.asXmmRmR(op, rm, tmp)
|
|
3389
|
+
m.insert(binOp)
|
|
3390
|
+
|
|
3391
|
+
m.copyTo(tmp, rd)
|
|
3392
|
+
}
|
|
3393
|
+
|
|
3394
|
+
func (m *machine) lowerVbBinOp(op sseOpcode, x, y, ret ssa.Value) {
|
|
3395
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3396
|
+
rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3397
|
+
rd := m.c.VRegOf(ret)
|
|
3398
|
+
|
|
3399
|
+
tmp := m.copyToTmp(rn.reg())
|
|
3400
|
+
|
|
3401
|
+
binOp := m.allocateInstr()
|
|
3402
|
+
binOp.asXmmRmR(op, rm, tmp)
|
|
3403
|
+
m.insert(binOp)
|
|
3404
|
+
|
|
3405
|
+
m.copyTo(tmp, rd)
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
func (m *machine) lowerVFcmp(x, y ssa.Value, c ssa.FloatCmpCond, ret ssa.Value, lane ssa.VecLane) {
|
|
3409
|
+
var cmpOp sseOpcode
|
|
3410
|
+
switch lane {
|
|
3411
|
+
case ssa.VecLaneF32x4:
|
|
3412
|
+
cmpOp = sseOpcodeCmpps
|
|
3413
|
+
case ssa.VecLaneF64x2:
|
|
3414
|
+
cmpOp = sseOpcodeCmppd
|
|
3415
|
+
default:
|
|
3416
|
+
panic(fmt.Sprintf("invalid lane type: %s", lane))
|
|
3417
|
+
}
|
|
3418
|
+
|
|
3419
|
+
xx, yy := m.c.ValueDefinition(x), m.c.ValueDefinition(y)
|
|
3420
|
+
var cmpImm cmpPred
|
|
3421
|
+
switch c {
|
|
3422
|
+
case ssa.FloatCmpCondGreaterThan:
|
|
3423
|
+
yy, xx = xx, yy
|
|
3424
|
+
cmpImm = cmpPredLT_OS
|
|
3425
|
+
case ssa.FloatCmpCondGreaterThanOrEqual:
|
|
3426
|
+
yy, xx = xx, yy
|
|
3427
|
+
cmpImm = cmpPredLE_OS
|
|
3428
|
+
case ssa.FloatCmpCondEqual:
|
|
3429
|
+
cmpImm = cmpPredEQ_OQ
|
|
3430
|
+
case ssa.FloatCmpCondNotEqual:
|
|
3431
|
+
cmpImm = cmpPredNEQ_UQ
|
|
3432
|
+
case ssa.FloatCmpCondLessThan:
|
|
3433
|
+
cmpImm = cmpPredLT_OS
|
|
3434
|
+
case ssa.FloatCmpCondLessThanOrEqual:
|
|
3435
|
+
cmpImm = cmpPredLE_OS
|
|
3436
|
+
default:
|
|
3437
|
+
panic(fmt.Sprintf("invalid float comparison condition: %s", c))
|
|
3438
|
+
}
|
|
3439
|
+
|
|
3440
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3441
|
+
xxx := m.getOperand_Mem_Reg(xx)
|
|
3442
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, xxx, tmp))
|
|
3443
|
+
|
|
3444
|
+
rm := m.getOperand_Mem_Reg(yy)
|
|
3445
|
+
m.insert(m.allocateInstr().asXmmRmRImm(cmpOp, byte(cmpImm), rm, tmp))
|
|
3446
|
+
|
|
3447
|
+
m.copyTo(tmp, m.c.VRegOf(ret))
|
|
3448
|
+
}
|
|
3449
|
+
|
|
3450
|
+
func (m *machine) lowerVIcmp(x, y ssa.Value, c ssa.IntegerCmpCond, ret ssa.Value, lane ssa.VecLane) {
|
|
3451
|
+
var eq, gt, maxu, minu, mins sseOpcode
|
|
3452
|
+
switch lane {
|
|
3453
|
+
case ssa.VecLaneI8x16:
|
|
3454
|
+
eq, gt, maxu, minu, mins = sseOpcodePcmpeqb, sseOpcodePcmpgtb, sseOpcodePmaxub, sseOpcodePminub, sseOpcodePminsb
|
|
3455
|
+
case ssa.VecLaneI16x8:
|
|
3456
|
+
eq, gt, maxu, minu, mins = sseOpcodePcmpeqw, sseOpcodePcmpgtw, sseOpcodePmaxuw, sseOpcodePminuw, sseOpcodePminsw
|
|
3457
|
+
case ssa.VecLaneI32x4:
|
|
3458
|
+
eq, gt, maxu, minu, mins = sseOpcodePcmpeqd, sseOpcodePcmpgtd, sseOpcodePmaxud, sseOpcodePminud, sseOpcodePminsd
|
|
3459
|
+
case ssa.VecLaneI64x2:
|
|
3460
|
+
eq, gt = sseOpcodePcmpeqq, sseOpcodePcmpgtq
|
|
3461
|
+
default:
|
|
3462
|
+
panic(fmt.Sprintf("invalid lane type: %s", lane))
|
|
3463
|
+
}
|
|
3464
|
+
|
|
3465
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3466
|
+
var op operand
|
|
3467
|
+
switch c {
|
|
3468
|
+
case ssa.IntegerCmpCondSignedLessThanOrEqual:
|
|
3469
|
+
if lane == ssa.VecLaneI64x2 {
|
|
3470
|
+
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3471
|
+
// Copy x to tmp.
|
|
3472
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
|
|
3473
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3474
|
+
} else {
|
|
3475
|
+
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3476
|
+
// Copy y to tmp.
|
|
3477
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
|
|
3478
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3479
|
+
}
|
|
3480
|
+
case ssa.IntegerCmpCondSignedGreaterThanOrEqual:
|
|
3481
|
+
if lane == ssa.VecLaneI64x2 {
|
|
3482
|
+
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3483
|
+
// Copy y to tmp.
|
|
3484
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
|
|
3485
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3486
|
+
} else {
|
|
3487
|
+
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3488
|
+
// Copy x to tmp.
|
|
3489
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
|
|
3490
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3491
|
+
}
|
|
3492
|
+
case ssa.IntegerCmpCondSignedLessThan, ssa.IntegerCmpCondUnsignedLessThan, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
|
|
3493
|
+
y := m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3494
|
+
// Copy y to tmp.
|
|
3495
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, y, tmp))
|
|
3496
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3497
|
+
default:
|
|
3498
|
+
x := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3499
|
+
// Copy x to tmp.
|
|
3500
|
+
m.insert(m.allocateInstr().asXmmUnaryRmR(sseOpcodeMovdqu, x, tmp))
|
|
3501
|
+
op = m.getOperand_Mem_Reg(m.c.ValueDefinition(y))
|
|
3502
|
+
}
|
|
3503
|
+
|
|
3504
|
+
switch c {
|
|
3505
|
+
case ssa.IntegerCmpCondEqual:
|
|
3506
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
|
|
3507
|
+
case ssa.IntegerCmpCondNotEqual:
|
|
3508
|
+
// First we compare for equality.
|
|
3509
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
|
|
3510
|
+
// Then flip the bits. To do so, we set all bits on tmp2.
|
|
3511
|
+
tmp2 := m.c.AllocateVReg(ssa.TypeV128)
|
|
3512
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
|
|
3513
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
|
|
3514
|
+
// And then xor with tmp.
|
|
3515
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
|
|
3516
|
+
case ssa.IntegerCmpCondSignedGreaterThan, ssa.IntegerCmpCondSignedLessThan:
|
|
3517
|
+
m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
|
|
3518
|
+
case ssa.IntegerCmpCondSignedGreaterThanOrEqual, ssa.IntegerCmpCondSignedLessThanOrEqual:
|
|
3519
|
+
if lane == ssa.VecLaneI64x2 {
|
|
3520
|
+
m.insert(m.allocateInstr().asXmmRmR(gt, op, tmp))
|
|
3521
|
+
// Then flip the bits. To do so, we set all bits on tmp2.
|
|
3522
|
+
tmp2 := m.c.AllocateVReg(ssa.TypeV128)
|
|
3523
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
|
|
3524
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
|
|
3525
|
+
// And then xor with tmp.
|
|
3526
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
|
|
3527
|
+
} else {
|
|
3528
|
+
// First take min of x and y.
|
|
3529
|
+
m.insert(m.allocateInstr().asXmmRmR(mins, op, tmp))
|
|
3530
|
+
// Then compare for equality.
|
|
3531
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
|
|
3532
|
+
}
|
|
3533
|
+
case ssa.IntegerCmpCondUnsignedGreaterThan, ssa.IntegerCmpCondUnsignedLessThan:
|
|
3534
|
+
// First maxu of x and y.
|
|
3535
|
+
m.insert(m.allocateInstr().asXmmRmR(maxu, op, tmp))
|
|
3536
|
+
// Then compare for equality.
|
|
3537
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
|
|
3538
|
+
// Then flip the bits. To do so, we set all bits on tmp2.
|
|
3539
|
+
tmp2 := m.c.AllocateVReg(ssa.TypeV128)
|
|
3540
|
+
m.insert(m.allocateInstr().asDefineUninitializedReg(tmp2))
|
|
3541
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, newOperandReg(tmp2), tmp2))
|
|
3542
|
+
// And then xor with tmp.
|
|
3543
|
+
m.insert(m.allocateInstr().asXmmRmR(sseOpcodePxor, newOperandReg(tmp2), tmp))
|
|
3544
|
+
case ssa.IntegerCmpCondUnsignedGreaterThanOrEqual, ssa.IntegerCmpCondUnsignedLessThanOrEqual:
|
|
3545
|
+
m.insert(m.allocateInstr().asXmmRmR(minu, op, tmp))
|
|
3546
|
+
m.insert(m.allocateInstr().asXmmRmR(eq, op, tmp))
|
|
3547
|
+
default:
|
|
3548
|
+
panic("BUG")
|
|
3549
|
+
}
|
|
3550
|
+
|
|
3551
|
+
m.copyTo(tmp, m.c.VRegOf(ret))
|
|
3552
|
+
}
|
|
3553
|
+
|
|
3554
|
+
func (m *machine) lowerVbandnot(instr *ssa.Instruction, op sseOpcode) {
|
|
3555
|
+
x, y := instr.Arg2()
|
|
3556
|
+
xDef := m.c.ValueDefinition(x)
|
|
3557
|
+
yDef := m.c.ValueDefinition(y)
|
|
3558
|
+
rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
|
|
3559
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3560
|
+
|
|
3561
|
+
tmp := m.copyToTmp(rn.reg())
|
|
3562
|
+
|
|
3563
|
+
// pandn between rn, rm.
|
|
3564
|
+
pand := m.allocateInstr()
|
|
3565
|
+
pand.asXmmRmR(sseOpcodePandn, rm, tmp)
|
|
3566
|
+
m.insert(pand)
|
|
3567
|
+
|
|
3568
|
+
m.copyTo(tmp, rd)
|
|
3569
|
+
}
|
|
3570
|
+
|
|
3571
|
+
func (m *machine) lowerVbitselect(instr *ssa.Instruction) {
|
|
3572
|
+
c, x, y := instr.SelectData()
|
|
3573
|
+
xDef := m.c.ValueDefinition(x)
|
|
3574
|
+
yDef := m.c.ValueDefinition(y)
|
|
3575
|
+
rm, rn := m.getOperand_Reg(xDef), m.getOperand_Reg(yDef)
|
|
3576
|
+
creg := m.getOperand_Reg(m.c.ValueDefinition(c))
|
|
3577
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3578
|
+
|
|
3579
|
+
tmpC := m.copyToTmp(creg.reg())
|
|
3580
|
+
tmpX := m.copyToTmp(rm.reg())
|
|
3581
|
+
|
|
3582
|
+
// And between c, x (overwrites x).
|
|
3583
|
+
pand := m.allocateInstr()
|
|
3584
|
+
pand.asXmmRmR(sseOpcodePand, creg, tmpX)
|
|
3585
|
+
m.insert(pand)
|
|
3586
|
+
|
|
3587
|
+
// Andn between y, c (overwrites c).
|
|
3588
|
+
pandn := m.allocateInstr()
|
|
3589
|
+
pandn.asXmmRmR(sseOpcodePandn, rn, tmpC)
|
|
3590
|
+
m.insert(pandn)
|
|
3591
|
+
|
|
3592
|
+
por := m.allocateInstr()
|
|
3593
|
+
por.asXmmRmR(sseOpcodePor, newOperandReg(tmpC), tmpX)
|
|
3594
|
+
m.insert(por)
|
|
3595
|
+
|
|
3596
|
+
m.copyTo(tmpX, rd)
|
|
3597
|
+
}
|
|
3598
|
+
|
|
3599
|
+
func (m *machine) lowerVFmin(instr *ssa.Instruction) {
|
|
3600
|
+
x, y, lane := instr.Arg2WithLane()
|
|
3601
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3602
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
3603
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3604
|
+
|
|
3605
|
+
var min, cmp, andn, or, srl /* shift right logical */ sseOpcode
|
|
3606
|
+
var shiftNumToInverseNaN uint32
|
|
3607
|
+
if lane == ssa.VecLaneF32x4 {
|
|
3608
|
+
min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodePsrld, 0xa
|
|
3609
|
+
} else {
|
|
3610
|
+
min, cmp, andn, or, srl, shiftNumToInverseNaN = sseOpcodeMinpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodePsrlq, 0xd
|
|
3611
|
+
}
|
|
3612
|
+
|
|
3613
|
+
tmp1 := m.copyToTmp(rn.reg())
|
|
3614
|
+
tmp2 := m.copyToTmp(rm.reg())
|
|
3615
|
+
|
|
3616
|
+
// tmp1=min(rn, rm)
|
|
3617
|
+
minIns1 := m.allocateInstr()
|
|
3618
|
+
minIns1.asXmmRmR(min, rn, tmp2)
|
|
3619
|
+
m.insert(minIns1)
|
|
3620
|
+
|
|
3621
|
+
// tmp2=min(rm, rn)
|
|
3622
|
+
minIns2 := m.allocateInstr()
|
|
3623
|
+
minIns2.asXmmRmR(min, rm, tmp1)
|
|
3624
|
+
m.insert(minIns2)
|
|
3625
|
+
|
|
3626
|
+
// tmp3:=tmp1=min(rn, rm)
|
|
3627
|
+
tmp3 := m.copyToTmp(tmp1)
|
|
3628
|
+
|
|
3629
|
+
// tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
|
|
3630
|
+
// NaN if rn == NaN || rm == NaN
|
|
3631
|
+
// min(rm, rm) otherwise
|
|
3632
|
+
orIns := m.allocateInstr()
|
|
3633
|
+
orIns.asXmmRmR(or, newOperandReg(tmp2), tmp1)
|
|
3634
|
+
m.insert(orIns)
|
|
3635
|
+
|
|
3636
|
+
// tmp3 is originally min(rn,rm).
|
|
3637
|
+
// tmp3 = 0^ (set all bits) if rn == NaN || rm == NaN
|
|
3638
|
+
// 0 otherwise
|
|
3639
|
+
cmpIns := m.allocateInstr()
|
|
3640
|
+
cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp2), tmp3)
|
|
3641
|
+
m.insert(cmpIns)
|
|
3642
|
+
|
|
3643
|
+
// tmp1 = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
|
|
3644
|
+
// ^0 if rn == NaN || rm == NaN
|
|
3645
|
+
// min(v1, v2) otherwise
|
|
3646
|
+
orIns2 := m.allocateInstr()
|
|
3647
|
+
orIns2.asXmmRmR(or, newOperandReg(tmp3), tmp1)
|
|
3648
|
+
m.insert(orIns2)
|
|
3649
|
+
|
|
3650
|
+
// tmp3 = set all bits on the mantissa bits
|
|
3651
|
+
// 0 otherwise
|
|
3652
|
+
shift := m.allocateInstr()
|
|
3653
|
+
shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp3)
|
|
3654
|
+
m.insert(shift)
|
|
3655
|
+
|
|
3656
|
+
// tmp3 = tmp1 and !tmp3
|
|
3657
|
+
// = -0 if (rn == -0 || rm == -0) && rn != NaN && rm !=NaN
|
|
3658
|
+
// set all bits on exponential and sign bit (== NaN) if rn == NaN || rm == NaN
|
|
3659
|
+
// min(rn, rm) otherwise
|
|
3660
|
+
andnIns := m.allocateInstr()
|
|
3661
|
+
andnIns.asXmmRmR(andn, newOperandReg(tmp1), tmp3)
|
|
3662
|
+
m.insert(andnIns)
|
|
3663
|
+
|
|
3664
|
+
m.copyTo(tmp3, rd)
|
|
3665
|
+
}
|
|
3666
|
+
|
|
3667
|
+
func (m *machine) lowerVFmax(instr *ssa.Instruction) {
|
|
3668
|
+
x, y, lane := instr.Arg2WithLane()
|
|
3669
|
+
rn := m.getOperand_Reg(m.c.ValueDefinition(x))
|
|
3670
|
+
rm := m.getOperand_Reg(m.c.ValueDefinition(y))
|
|
3671
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3672
|
+
|
|
3673
|
+
var max, cmp, andn, or, xor, sub, srl /* shift right logical */ sseOpcode
|
|
3674
|
+
var shiftNumToInverseNaN uint32
|
|
3675
|
+
if lane == ssa.VecLaneF32x4 {
|
|
3676
|
+
max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxps, sseOpcodeCmpps, sseOpcodeAndnps, sseOpcodeOrps, sseOpcodeXorps, sseOpcodeSubps, sseOpcodePsrld, 0xa
|
|
3677
|
+
} else {
|
|
3678
|
+
max, cmp, andn, or, xor, sub, srl, shiftNumToInverseNaN = sseOpcodeMaxpd, sseOpcodeCmppd, sseOpcodeAndnpd, sseOpcodeOrpd, sseOpcodeXorpd, sseOpcodeSubpd, sseOpcodePsrlq, 0xd
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3681
|
+
tmp0 := m.copyToTmp(rm.reg())
|
|
3682
|
+
tmp1 := m.copyToTmp(rn.reg())
|
|
3683
|
+
|
|
3684
|
+
// tmp0=max(rn, rm)
|
|
3685
|
+
maxIns1 := m.allocateInstr()
|
|
3686
|
+
maxIns1.asXmmRmR(max, rn, tmp0)
|
|
3687
|
+
m.insert(maxIns1)
|
|
3688
|
+
|
|
3689
|
+
// tmp1=max(rm, rn)
|
|
3690
|
+
maxIns2 := m.allocateInstr()
|
|
3691
|
+
maxIns2.asXmmRmR(max, rm, tmp1)
|
|
3692
|
+
m.insert(maxIns2)
|
|
3693
|
+
|
|
3694
|
+
// tmp2=max(rm, rn)
|
|
3695
|
+
tmp2 := m.copyToTmp(tmp1)
|
|
3696
|
+
|
|
3697
|
+
// tmp2 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
|
|
3698
|
+
// 0 if (rn == 0 && rm == 0)
|
|
3699
|
+
// -0 if (rn == -0 && rm == -0)
|
|
3700
|
+
// v1^v2 if rn == NaN || rm == NaN
|
|
3701
|
+
// 0 otherwise
|
|
3702
|
+
xorInstr := m.allocateInstr()
|
|
3703
|
+
xorInstr.asXmmRmR(xor, newOperandReg(tmp0), tmp2)
|
|
3704
|
+
m.insert(xorInstr)
|
|
3705
|
+
// tmp1 = -0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0)
|
|
3706
|
+
// 0 if (rn == 0 && rm == 0)
|
|
3707
|
+
// -0 if (rn == -0 && rm == -0)
|
|
3708
|
+
// NaN if rn == NaN || rm == NaN
|
|
3709
|
+
// max(v1, v2) otherwise
|
|
3710
|
+
orInstr := m.allocateInstr()
|
|
3711
|
+
orInstr.asXmmRmR(or, newOperandReg(tmp2), tmp1)
|
|
3712
|
+
m.insert(orInstr)
|
|
3713
|
+
|
|
3714
|
+
tmp3 := m.copyToTmp(tmp1)
|
|
3715
|
+
|
|
3716
|
+
// tmp3 = 0 if (rn == -0 && rm == 0) || (rn == 0 && rm == -0) || (rn == 0 && rm == 0)
|
|
3717
|
+
// -0 if (rn == -0 && rm == -0)
|
|
3718
|
+
// NaN if rn == NaN || rm == NaN
|
|
3719
|
+
// max(v1, v2) otherwise
|
|
3720
|
+
//
|
|
3721
|
+
// Note: -0 - (-0) = 0 (!= -0) in floating point operation.
|
|
3722
|
+
subIns := m.allocateInstr()
|
|
3723
|
+
subIns.asXmmRmR(sub, newOperandReg(tmp2), tmp3)
|
|
3724
|
+
m.insert(subIns)
|
|
3725
|
+
|
|
3726
|
+
// tmp1 = 0^ if rn == NaN || rm == NaN
|
|
3727
|
+
cmpIns := m.allocateInstr()
|
|
3728
|
+
cmpIns.asXmmRmRImm(cmp, uint8(cmpPredUNORD_Q), newOperandReg(tmp1), tmp1)
|
|
3729
|
+
m.insert(cmpIns)
|
|
3730
|
+
|
|
3731
|
+
// tmp1 = set all bits on the mantissa bits
|
|
3732
|
+
// 0 otherwise
|
|
3733
|
+
shift := m.allocateInstr()
|
|
3734
|
+
shift.asXmmRmiReg(srl, newOperandImm32(shiftNumToInverseNaN), tmp1)
|
|
3735
|
+
m.insert(shift)
|
|
3736
|
+
|
|
3737
|
+
andnIns := m.allocateInstr()
|
|
3738
|
+
andnIns.asXmmRmR(andn, newOperandReg(tmp3), tmp1)
|
|
3739
|
+
m.insert(andnIns)
|
|
3740
|
+
|
|
3741
|
+
m.copyTo(tmp1, rd)
|
|
3742
|
+
}
|
|
3743
|
+
|
|
3744
|
+
func (m *machine) lowerVFabs(instr *ssa.Instruction) {
|
|
3745
|
+
x, lane := instr.ArgWithLane()
|
|
3746
|
+
rm := m.getOperand_Mem_Reg(m.c.ValueDefinition(x))
|
|
3747
|
+
rd := m.c.VRegOf(instr.Return())
|
|
3748
|
+
|
|
3749
|
+
tmp := m.c.AllocateVReg(ssa.TypeV128)
|
|
3750
|
+
|
|
3751
|
+
def := m.allocateInstr()
|
|
3752
|
+
def.asDefineUninitializedReg(tmp)
|
|
3753
|
+
m.insert(def)
|
|
3754
|
+
|
|
3755
|
+
// Set all bits on tmp.
|
|
3756
|
+
pcmp := m.allocateInstr()
|
|
3757
|
+
pcmp.asXmmRmR(sseOpcodePcmpeqd, newOperandReg(tmp), tmp)
|
|
3758
|
+
m.insert(pcmp)
|
|
3759
|
+
|
|
3760
|
+
switch lane {
|
|
3761
|
+
case ssa.VecLaneF32x4:
|
|
3762
|
+
// Shift right packed single floats by 1 to clear the sign bits.
|
|
3763
|
+
shift := m.allocateInstr()
|
|
3764
|
+
shift.asXmmRmiReg(sseOpcodePsrld, newOperandImm32(1), tmp)
|
|
3765
|
+
m.insert(shift)
|
|
3766
|
+
// Clear the sign bit of rm.
|
|
3767
|
+
andp := m.allocateInstr()
|
|
3768
|
+
andp.asXmmRmR(sseOpcodeAndpd, rm, tmp)
|
|
3769
|
+
m.insert(andp)
|
|
3770
|
+
case ssa.VecLaneF64x2:
|
|
3771
|
+
// Shift right packed single floats by 1 to clear the sign bits.
|
|
3772
|
+
shift := m.allocateInstr()
|
|
3773
|
+
shift.asXmmRmiReg(sseOpcodePsrlq, newOperandImm32(1), tmp)
|
|
3774
|
+
m.insert(shift)
|
|
3775
|
+
// Clear the sign bit of rm.
|
|
3776
|
+
andp := m.allocateInstr()
|
|
3777
|
+
andp.asXmmRmR(sseOpcodeAndps, rm, tmp)
|
|
3778
|
+
m.insert(andp)
|
|
3779
|
+
}
|
|
3780
|
+
|
|
3781
|
+
m.copyTo(tmp, rd)
|
|
3782
|
+
}
|