space-data-module-sdk 0.8.23 → 0.8.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/isomorphic-pthreads.html +29 -5
- package/docs/isomorphic-pthreads.md +98 -8
- package/package.json +1 -1
- package/src/compiler/compileModule.js +378 -385
- package/src/compiler/emceptionNode.js +339 -70
- package/src/host/browserModuleHarness.js +13 -0
- package/src/host/hostWorkerBundleSources.js +3 -3
- package/src/host/nodeSyncFsIo.js +3 -1
- package/src/host/wasiThreadBrowserWorker.mjs +41 -21
- package/src/host/wasiThreadHost.js +279 -150
- package/src/host/wasiThreadPool.js +389 -0
- package/src/host/wasiThreadWorker.mjs +62 -20
- package/src/host/wasiThreadWorkerRuntime.js +6 -1
- package/src/index.d.ts +31 -0
- package/src/testing/index.d.ts +2 -0
|
@@ -714,167 +714,159 @@ async function compileWithEmception(options = {}) {
|
|
|
714
714
|
invokeSource,
|
|
715
715
|
exportedSymbols,
|
|
716
716
|
outputPath,
|
|
717
|
+
tempDir,
|
|
717
718
|
compileOptions,
|
|
718
719
|
} = options;
|
|
719
|
-
const tempDir = await mkdtemp(
|
|
720
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
721
|
-
);
|
|
722
720
|
const resolvedOutputPath = path.resolve(
|
|
723
721
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
724
722
|
);
|
|
725
723
|
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
const wasmOutputPath = path.posix.join(workDir, "module.wasm");
|
|
739
|
-
|
|
740
|
-
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
741
|
-
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
724
|
+
return runWithEmceptionLock(async (emception) => {
|
|
725
|
+
const workDir = "/working/space-data-module-sdk-compile";
|
|
726
|
+
const runtimeIncludeDir = path.posix.join(workDir, "flatbuffers-runtime");
|
|
727
|
+
const sourcePath = path.posix.join(workDir, `module.${sourceExtension}`);
|
|
728
|
+
const manifestSourcePath = path.posix.join(workDir, "plugin-manifest-exports.cpp");
|
|
729
|
+
const invokeHeaderPath = path.posix.join(workDir, "space_data_module_invoke.h");
|
|
730
|
+
const invokeSourcePath = path.posix.join(workDir, "plugin-invoke-bridge.cpp");
|
|
731
|
+
const sourceObjectPath = path.posix.join(workDir, "module.o");
|
|
732
|
+
const linkObjectPath = path.posix.join(workDir, "module-link.o");
|
|
733
|
+
const manifestObjectPath = path.posix.join(workDir, "plugin-manifest-exports.o");
|
|
734
|
+
const invokeObjectPath = path.posix.join(workDir, "plugin-invoke-bridge.o");
|
|
735
|
+
const wasmOutputPath = path.posix.join(workDir, "module.wasm");
|
|
742
736
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
|
|
746
|
-
await writeFilesToEmception(emception, workDir, schemaHeaders);
|
|
747
|
-
emception.writeFile(sourcePath, sourceCode);
|
|
748
|
-
emception.writeFile(manifestSourcePath, manifestSource);
|
|
749
|
-
emception.writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
750
|
-
emception.writeFile(invokeSourcePath, invokeSource);
|
|
751
|
-
|
|
752
|
-
const commands = [
|
|
753
|
-
[
|
|
754
|
-
sourceCompilerCommand,
|
|
755
|
-
"-c",
|
|
756
|
-
sourcePath,
|
|
757
|
-
`-I${workDir}`,
|
|
758
|
-
`-I${runtimeIncludeDir}`,
|
|
759
|
-
...buildSourceCompilerArgs(compileOptions),
|
|
760
|
-
"-o",
|
|
761
|
-
sourceObjectPath,
|
|
762
|
-
],
|
|
763
|
-
];
|
|
764
|
-
|
|
765
|
-
for (const command of commands) {
|
|
766
|
-
const result = emception.run(command.join(" "));
|
|
767
|
-
if (result.returncode !== 0) {
|
|
768
|
-
throw new Error(
|
|
769
|
-
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
770
|
-
);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
const sourceObjectBytes = new Uint8Array(emception.readFile(sourceObjectPath));
|
|
775
|
-
const guestLink = deriveGuestLinkRenameArgs({
|
|
776
|
-
objectBytes: sourceObjectBytes,
|
|
777
|
-
pluginId: manifest?.pluginId,
|
|
778
|
-
methodIds: Array.isArray(manifest?.methods)
|
|
779
|
-
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
780
|
-
: [],
|
|
781
|
-
});
|
|
737
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
738
|
+
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
782
739
|
|
|
783
|
-
|
|
740
|
+
try {
|
|
741
|
+
emception.FS.mkdirTree(workDir);
|
|
742
|
+
await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
|
|
743
|
+
await writeFilesToEmception(emception, workDir, schemaHeaders);
|
|
744
|
+
emception.writeFile(sourcePath, sourceCode);
|
|
745
|
+
emception.writeFile(manifestSourcePath, manifestSource);
|
|
746
|
+
emception.writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
747
|
+
emception.writeFile(invokeSourcePath, invokeSource);
|
|
748
|
+
|
|
749
|
+
const commands = [
|
|
750
|
+
[
|
|
784
751
|
sourceCompilerCommand,
|
|
785
752
|
"-c",
|
|
786
753
|
sourcePath,
|
|
787
754
|
`-I${workDir}`,
|
|
788
755
|
`-I${runtimeIncludeDir}`,
|
|
789
|
-
...guestLink.renameArgs,
|
|
790
756
|
...buildSourceCompilerArgs(compileOptions),
|
|
791
757
|
"-o",
|
|
792
|
-
|
|
793
|
-
]
|
|
794
|
-
|
|
795
|
-
|
|
758
|
+
sourceObjectPath,
|
|
759
|
+
],
|
|
760
|
+
];
|
|
761
|
+
|
|
762
|
+
for (const command of commands) {
|
|
763
|
+
const result = emception.run(command.join(" "));
|
|
764
|
+
if (result.returncode !== 0) {
|
|
796
765
|
throw new Error(
|
|
797
|
-
`Compilation failed with ${
|
|
766
|
+
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
798
767
|
);
|
|
799
768
|
}
|
|
769
|
+
}
|
|
800
770
|
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
[
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
invokeObjectPath,
|
|
828
|
-
],
|
|
829
|
-
[
|
|
830
|
-
"em++",
|
|
831
|
-
sourceObjectPath,
|
|
832
|
-
manifestObjectPath,
|
|
833
|
-
invokeObjectPath,
|
|
834
|
-
...args,
|
|
835
|
-
"-o",
|
|
836
|
-
wasmOutputPath,
|
|
837
|
-
],
|
|
838
|
-
];
|
|
839
|
-
for (const command of remainingCommands) {
|
|
840
|
-
const result = emception.run(command.join(" "));
|
|
841
|
-
if (result.returncode !== 0) {
|
|
842
|
-
throw new Error(
|
|
843
|
-
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
844
|
-
);
|
|
845
|
-
}
|
|
846
|
-
}
|
|
771
|
+
const sourceObjectBytes = new Uint8Array(emception.readFile(sourceObjectPath));
|
|
772
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
773
|
+
objectBytes: sourceObjectBytes,
|
|
774
|
+
pluginId: manifest?.pluginId,
|
|
775
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
776
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
777
|
+
: [],
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
const linkCompileCommand = [
|
|
781
|
+
sourceCompilerCommand,
|
|
782
|
+
"-c",
|
|
783
|
+
sourcePath,
|
|
784
|
+
`-I${workDir}`,
|
|
785
|
+
`-I${runtimeIncludeDir}`,
|
|
786
|
+
...guestLink.renameArgs,
|
|
787
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
788
|
+
"-o",
|
|
789
|
+
linkObjectPath,
|
|
790
|
+
];
|
|
791
|
+
const linkCompileResult = emception.run(linkCompileCommand.join(" "));
|
|
792
|
+
if (linkCompileResult.returncode !== 0) {
|
|
793
|
+
throw new Error(
|
|
794
|
+
`Compilation failed with ${linkCompileCommand[0]} (emception): ${linkCompileResult.stderr || linkCompileResult.stdout}`,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
847
797
|
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
798
|
+
// The embedded manifest and the invoke bridge get the module source's
|
|
799
|
+
// own optimization flags. Without them both objects built at -O0 and
|
|
800
|
+
// every payload byte crossing the bridge (the std::vector copies of
|
|
801
|
+
// each request and response) cost hundreds of interpreted wasm
|
|
802
|
+
// instructions. The wasi path already compiles them this way.
|
|
803
|
+
const remainingCommands = [
|
|
804
|
+
[
|
|
805
|
+
"em++",
|
|
806
|
+
"-c",
|
|
807
|
+
manifestSourcePath,
|
|
808
|
+
"-std=c++17",
|
|
809
|
+
`-I${workDir}`,
|
|
810
|
+
`-I${runtimeIncludeDir}`,
|
|
811
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
812
|
+
"-o",
|
|
813
|
+
manifestObjectPath,
|
|
814
|
+
],
|
|
815
|
+
[
|
|
816
|
+
"em++",
|
|
817
|
+
"-c",
|
|
818
|
+
invokeSourcePath,
|
|
819
|
+
"-std=c++17",
|
|
820
|
+
`-I${workDir}`,
|
|
821
|
+
`-I${runtimeIncludeDir}`,
|
|
822
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
823
|
+
"-o",
|
|
824
|
+
invokeObjectPath,
|
|
825
|
+
],
|
|
826
|
+
[
|
|
827
|
+
"em++",
|
|
828
|
+
sourceObjectPath,
|
|
829
|
+
manifestObjectPath,
|
|
830
|
+
invokeObjectPath,
|
|
831
|
+
...args,
|
|
832
|
+
"-o",
|
|
833
|
+
wasmOutputPath,
|
|
834
|
+
],
|
|
835
|
+
];
|
|
836
|
+
for (const command of remainingCommands) {
|
|
837
|
+
const result = emception.run(command.join(" "));
|
|
838
|
+
if (result.returncode !== 0) {
|
|
839
|
+
throw new Error(
|
|
840
|
+
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
841
|
+
);
|
|
871
842
|
}
|
|
872
843
|
}
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
844
|
+
|
|
845
|
+
const wasmBytes = new Uint8Array(emception.readFile(wasmOutputPath));
|
|
846
|
+
const linkObjectBytes = new Uint8Array(emception.readFile(linkObjectPath));
|
|
847
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
848
|
+
return {
|
|
849
|
+
wasmBytes,
|
|
850
|
+
outputPath: resolvedOutputPath,
|
|
851
|
+
guestLink: {
|
|
852
|
+
format: "wasm-object",
|
|
853
|
+
language,
|
|
854
|
+
symbolPrefix: guestLink.prefix,
|
|
855
|
+
methodSymbols: guestLink.methodSymbols,
|
|
856
|
+
threadModel:
|
|
857
|
+
compileOptions.threadModel ?? ModuleThreadModel.SINGLE_THREAD,
|
|
858
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
859
|
+
objectBytes: linkObjectBytes,
|
|
860
|
+
},
|
|
861
|
+
};
|
|
862
|
+
} finally {
|
|
863
|
+
try {
|
|
864
|
+
removeEmceptionDirectory(emception, workDir);
|
|
865
|
+
} catch {
|
|
866
|
+
// Best-effort cleanup only; the shared emception instance remains usable.
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
});
|
|
878
870
|
}
|
|
879
871
|
|
|
880
872
|
async function ensureSystemCompilerAvailable(command) {
|
|
@@ -918,11 +910,9 @@ async function compileWithSystemEmscripten(options = {}) {
|
|
|
918
910
|
invokeSource,
|
|
919
911
|
exportedSymbols,
|
|
920
912
|
outputPath,
|
|
913
|
+
tempDir,
|
|
921
914
|
compileOptions,
|
|
922
915
|
} = options;
|
|
923
|
-
const tempDir = await mkdtemp(
|
|
924
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
925
|
-
);
|
|
926
916
|
const resolvedOutputPath = path.resolve(
|
|
927
917
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
928
918
|
);
|
|
@@ -937,101 +927,95 @@ async function compileWithSystemEmscripten(options = {}) {
|
|
|
937
927
|
const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
|
|
938
928
|
const wasmOutputPath = path.join(tempDir, "module.wasm");
|
|
939
929
|
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
});
|
|
930
|
+
await ensureSystemCompilerAvailable(sourceCompilerCommand);
|
|
931
|
+
await ensureSystemCompilerAvailable("em++");
|
|
932
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
933
|
+
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
934
|
+
await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
|
|
935
|
+
await writeFilesToDirectory(tempDir, schemaHeaders);
|
|
936
|
+
await writeFile(sourcePath, sourceCode);
|
|
937
|
+
await writeFile(manifestSourcePath, manifestSource);
|
|
938
|
+
await writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
939
|
+
await writeFile(invokeSourcePath, invokeSource);
|
|
940
|
+
|
|
941
|
+
const sourceCompileArgs = [
|
|
942
|
+
"-c",
|
|
943
|
+
sourcePath,
|
|
944
|
+
`-I${tempDir}`,
|
|
945
|
+
`-I${runtimeIncludeDir}`,
|
|
946
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
947
|
+
"-o",
|
|
948
|
+
sourceObjectPath,
|
|
949
|
+
];
|
|
950
|
+
await runSystemCompiler(sourceCompilerCommand, sourceCompileArgs);
|
|
951
|
+
|
|
952
|
+
const sourceObjectBytes = new Uint8Array(await readFile(sourceObjectPath));
|
|
953
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
954
|
+
objectBytes: sourceObjectBytes,
|
|
955
|
+
pluginId: manifest?.pluginId,
|
|
956
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
957
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
958
|
+
: [],
|
|
959
|
+
});
|
|
971
960
|
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
961
|
+
const linkCompileArgs = [
|
|
962
|
+
"-c",
|
|
963
|
+
sourcePath,
|
|
964
|
+
`-I${tempDir}`,
|
|
965
|
+
`-I${runtimeIncludeDir}`,
|
|
966
|
+
...guestLink.renameArgs,
|
|
967
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
968
|
+
"-o",
|
|
969
|
+
linkObjectPath,
|
|
970
|
+
];
|
|
971
|
+
await runSystemCompiler(sourceCompilerCommand, linkCompileArgs);
|
|
972
|
+
// Same optimization flags as the module source (see compileWithEmception).
|
|
973
|
+
// buildSourceCompilerArgs carries -pthread whenever usesPthreadCompileFlags.
|
|
974
|
+
await runSystemCompiler("em++", [
|
|
975
|
+
"-c",
|
|
976
|
+
manifestSourcePath,
|
|
977
|
+
"-std=c++17",
|
|
978
|
+
`-I${tempDir}`,
|
|
979
|
+
`-I${runtimeIncludeDir}`,
|
|
980
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
981
|
+
"-o",
|
|
982
|
+
manifestObjectPath,
|
|
983
|
+
]);
|
|
984
|
+
await runSystemCompiler("em++", [
|
|
985
|
+
"-c",
|
|
986
|
+
invokeSourcePath,
|
|
987
|
+
"-std=c++17",
|
|
988
|
+
`-I${tempDir}`,
|
|
989
|
+
`-I${runtimeIncludeDir}`,
|
|
990
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
991
|
+
"-o",
|
|
992
|
+
invokeObjectPath,
|
|
993
|
+
]);
|
|
994
|
+
await runSystemCompiler("em++", [
|
|
995
|
+
sourceObjectPath,
|
|
996
|
+
manifestObjectPath,
|
|
997
|
+
invokeObjectPath,
|
|
998
|
+
...args,
|
|
999
|
+
"-o",
|
|
1000
|
+
wasmOutputPath,
|
|
1001
|
+
]);
|
|
1013
1002
|
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
};
|
|
1031
|
-
} catch (error) {
|
|
1032
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
1033
|
-
throw error;
|
|
1034
|
-
}
|
|
1003
|
+
const wasmBytes = new Uint8Array(await readFile(wasmOutputPath));
|
|
1004
|
+
const linkObjectBytes = new Uint8Array(await readFile(linkObjectPath));
|
|
1005
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1006
|
+
return {
|
|
1007
|
+
wasmBytes,
|
|
1008
|
+
outputPath: resolvedOutputPath,
|
|
1009
|
+
guestLink: {
|
|
1010
|
+
format: "wasm-object",
|
|
1011
|
+
language,
|
|
1012
|
+
symbolPrefix: guestLink.prefix,
|
|
1013
|
+
methodSymbols: guestLink.methodSymbols,
|
|
1014
|
+
threadModel: compileOptions.threadModel,
|
|
1015
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
1016
|
+
objectBytes: linkObjectBytes,
|
|
1017
|
+
},
|
|
1018
|
+
};
|
|
1035
1019
|
}
|
|
1036
1020
|
|
|
1037
1021
|
async function runWasiCompiler(command, args, options = {}) {
|
|
@@ -1070,6 +1054,7 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1070
1054
|
invokeSource,
|
|
1071
1055
|
exportedSymbols,
|
|
1072
1056
|
outputPath,
|
|
1057
|
+
tempDir,
|
|
1073
1058
|
compileOptions,
|
|
1074
1059
|
} = options;
|
|
1075
1060
|
const toolchain = resolveWasiThreadsToolchain();
|
|
@@ -1077,9 +1062,6 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1077
1062
|
const sourceDriver = isCpp ? toolchain.clangxx : toolchain.clang;
|
|
1078
1063
|
const cppDriver = toolchain.clangxx;
|
|
1079
1064
|
|
|
1080
|
-
const tempDir = await mkdtemp(
|
|
1081
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
1082
|
-
);
|
|
1083
1065
|
const resolvedOutputPath = path.resolve(
|
|
1084
1066
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
1085
1067
|
);
|
|
@@ -1094,98 +1076,92 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1094
1076
|
const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
|
|
1095
1077
|
const wasmOutputPath = path.join(tempDir, "module.wasm");
|
|
1096
1078
|
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
]);
|
|
1079
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
1080
|
+
const linkArgs = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
1081
|
+
await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
|
|
1082
|
+
await writeFilesToDirectory(tempDir, schemaHeaders);
|
|
1083
|
+
await writeFile(sourcePath, sourceCode);
|
|
1084
|
+
await writeFile(manifestSourcePath, manifestSource);
|
|
1085
|
+
await writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
1086
|
+
await writeFile(invokeSourcePath, invokeSource);
|
|
1087
|
+
|
|
1088
|
+
const includeArgs = [`-I${tempDir}`, `-I${runtimeIncludeDir}`];
|
|
1089
|
+
const sourceCompilerArgs = buildSourceCompilerArgs(compileOptions);
|
|
1090
|
+
|
|
1091
|
+
await runWasiCompiler(sourceDriver, [
|
|
1092
|
+
...toolchain.toolchainArgs,
|
|
1093
|
+
"-c",
|
|
1094
|
+
sourcePath,
|
|
1095
|
+
...includeArgs,
|
|
1096
|
+
...(isCpp ? ["-std=c++17"] : []),
|
|
1097
|
+
...sourceCompilerArgs,
|
|
1098
|
+
"-o",
|
|
1099
|
+
sourceObjectPath,
|
|
1100
|
+
]);
|
|
1120
1101
|
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1102
|
+
const sourceObjectBytes = new Uint8Array(await readFile(sourceObjectPath));
|
|
1103
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
1104
|
+
objectBytes: sourceObjectBytes,
|
|
1105
|
+
pluginId: manifest?.pluginId,
|
|
1106
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
1107
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
1108
|
+
: [],
|
|
1109
|
+
});
|
|
1129
1110
|
|
|
1130
|
-
|
|
1111
|
+
await runWasiCompiler(sourceDriver, [
|
|
1112
|
+
...toolchain.toolchainArgs,
|
|
1113
|
+
"-c",
|
|
1114
|
+
sourcePath,
|
|
1115
|
+
...includeArgs,
|
|
1116
|
+
...(isCpp ? ["-std=c++17"] : []),
|
|
1117
|
+
...guestLink.renameArgs,
|
|
1118
|
+
...sourceCompilerArgs,
|
|
1119
|
+
"-o",
|
|
1120
|
+
linkObjectPath,
|
|
1121
|
+
]);
|
|
1122
|
+
|
|
1123
|
+
for (const [srcPath, objPath] of [
|
|
1124
|
+
[manifestSourcePath, manifestObjectPath],
|
|
1125
|
+
[invokeSourcePath, invokeObjectPath],
|
|
1126
|
+
]) {
|
|
1127
|
+
await runWasiCompiler(cppDriver, [
|
|
1131
1128
|
...toolchain.toolchainArgs,
|
|
1132
1129
|
"-c",
|
|
1133
|
-
|
|
1130
|
+
srcPath,
|
|
1131
|
+
"-std=c++17",
|
|
1134
1132
|
...includeArgs,
|
|
1135
|
-
...(isCpp ? ["-std=c++17"] : []),
|
|
1136
|
-
...guestLink.renameArgs,
|
|
1137
1133
|
...sourceCompilerArgs,
|
|
1138
1134
|
"-o",
|
|
1139
|
-
|
|
1135
|
+
objPath,
|
|
1140
1136
|
]);
|
|
1137
|
+
}
|
|
1141
1138
|
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
...includeArgs,
|
|
1152
|
-
...sourceCompilerArgs,
|
|
1153
|
-
"-o",
|
|
1154
|
-
objPath,
|
|
1155
|
-
]);
|
|
1156
|
-
}
|
|
1157
|
-
|
|
1158
|
-
await runWasiCompiler(cppDriver, [
|
|
1159
|
-
...toolchain.toolchainArgs,
|
|
1160
|
-
sourceObjectPath,
|
|
1161
|
-
manifestObjectPath,
|
|
1162
|
-
invokeObjectPath,
|
|
1163
|
-
...linkArgs,
|
|
1164
|
-
"-o",
|
|
1165
|
-
wasmOutputPath,
|
|
1166
|
-
]);
|
|
1139
|
+
await runWasiCompiler(cppDriver, [
|
|
1140
|
+
...toolchain.toolchainArgs,
|
|
1141
|
+
sourceObjectPath,
|
|
1142
|
+
manifestObjectPath,
|
|
1143
|
+
invokeObjectPath,
|
|
1144
|
+
...linkArgs,
|
|
1145
|
+
"-o",
|
|
1146
|
+
wasmOutputPath,
|
|
1147
|
+
]);
|
|
1167
1148
|
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
};
|
|
1185
|
-
} catch (error) {
|
|
1186
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
1187
|
-
throw error;
|
|
1188
|
-
}
|
|
1149
|
+
const wasmBytes = new Uint8Array(await readFile(wasmOutputPath));
|
|
1150
|
+
const linkObjectBytes = new Uint8Array(await readFile(linkObjectPath));
|
|
1151
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1152
|
+
return {
|
|
1153
|
+
wasmBytes,
|
|
1154
|
+
outputPath: resolvedOutputPath,
|
|
1155
|
+
guestLink: {
|
|
1156
|
+
format: "wasm-object",
|
|
1157
|
+
language,
|
|
1158
|
+
symbolPrefix: guestLink.prefix,
|
|
1159
|
+
methodSymbols: guestLink.methodSymbols,
|
|
1160
|
+
threadModel: compileOptions.threadModel,
|
|
1161
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
1162
|
+
objectBytes: linkObjectBytes,
|
|
1163
|
+
},
|
|
1164
|
+
};
|
|
1189
1165
|
}
|
|
1190
1166
|
|
|
1191
1167
|
export async function compileModuleFromSource(options = {}) {
|
|
@@ -1243,9 +1219,6 @@ export async function compileModuleFromSource(options = {}) {
|
|
|
1243
1219
|
),
|
|
1244
1220
|
];
|
|
1245
1221
|
|
|
1246
|
-
let wasmBytes;
|
|
1247
|
-
let resolvedOutputPath = null;
|
|
1248
|
-
let tempDir = null;
|
|
1249
1222
|
const threadModel = resolveThreadModel({
|
|
1250
1223
|
manifest,
|
|
1251
1224
|
threadModel: options.threadModel,
|
|
@@ -1275,72 +1248,92 @@ export async function compileModuleFromSource(options = {}) {
|
|
|
1275
1248
|
: useSystemEmscripten
|
|
1276
1249
|
? compileWithSystemEmscripten
|
|
1277
1250
|
: compileWithEmception;
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
invokeSource,
|
|
1287
|
-
exportedSymbols,
|
|
1288
|
-
outputPath: options.outputPath,
|
|
1289
|
-
compileOptions,
|
|
1290
|
-
});
|
|
1291
|
-
wasmBytes = appendWasmCustomSection(
|
|
1292
|
-
result.wasmBytes,
|
|
1293
|
-
SDS_MANIFEST_SECTION_NAME,
|
|
1294
|
-
encodePluginManifest(manifest),
|
|
1251
|
+
// Scratch for the toolchain and, without an explicit outputPath, the home of
|
|
1252
|
+
// the emitted artifact. Removed in finally on success, failure and throw.
|
|
1253
|
+
// `keepTempDir: true` hands it to the caller on success instead
|
|
1254
|
+
// (result.tempDir, freed by cleanupCompilation); a failed compile never
|
|
1255
|
+
// hands anything off, so its dir is always removed.
|
|
1256
|
+
const keepTempDir = options.keepTempDir === true;
|
|
1257
|
+
const tempDir = await mkdtemp(
|
|
1258
|
+
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
1295
1259
|
);
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
// to quietly build a plain wasm32-wasip1 object under the sequential model:
|
|
1312
|
-
// it is a concurrency exemption, not a toolchain exemption.
|
|
1313
|
-
threadFeatures = assertSequentialArtifact(wasmBytes, {
|
|
1314
|
-
source: resolvedOutputPath,
|
|
1315
|
-
target: resolveWasiThreadsToolchain().target,
|
|
1260
|
+
let handedOff = false;
|
|
1261
|
+
try {
|
|
1262
|
+
const result = await compileFunction({
|
|
1263
|
+
manifest,
|
|
1264
|
+
language: compiler.language,
|
|
1265
|
+
sourceCompilerCommand: compiler.command,
|
|
1266
|
+
sourceExtension: compiler.extension,
|
|
1267
|
+
sourceCode,
|
|
1268
|
+
manifestSource,
|
|
1269
|
+
invokeHeaderSource,
|
|
1270
|
+
invokeSource,
|
|
1271
|
+
exportedSymbols,
|
|
1272
|
+
outputPath: options.outputPath,
|
|
1273
|
+
tempDir,
|
|
1274
|
+
compileOptions,
|
|
1316
1275
|
});
|
|
1317
|
-
|
|
1276
|
+
const wasmBytes = appendWasmCustomSection(
|
|
1277
|
+
result.wasmBytes,
|
|
1278
|
+
SDS_MANIFEST_SECTION_NAME,
|
|
1279
|
+
encodePluginManifest(manifest),
|
|
1280
|
+
);
|
|
1281
|
+
const resolvedOutputPath = result.outputPath;
|
|
1282
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1318
1283
|
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1284
|
+
// Enforce the isomorphic-pthreads guardrail on the EMITTED artifact: a build
|
|
1285
|
+
// that claims the pthreads thread model but did not emit a shared-memory /
|
|
1286
|
+
// atomics wasm must FAIL the compile here rather than ship. See
|
|
1287
|
+
// docs/isomorphic-pthreads.md.
|
|
1288
|
+
let threadFeatures = null;
|
|
1289
|
+
if (threadModel === ModuleThreadModel.EMSCRIPTEN_PTHREADS) {
|
|
1290
|
+
threadFeatures = assertPthreadArtifact(wasmBytes, {
|
|
1291
|
+
source: resolvedOutputPath,
|
|
1292
|
+
});
|
|
1293
|
+
} else if (threadModel === ModuleThreadModel.WASI_SEQUENTIAL) {
|
|
1294
|
+
// Mirror guard. The target is passed so that SDN_WASI_TARGET cannot be used
|
|
1295
|
+
// to quietly build a plain wasm32-wasip1 object under the sequential model:
|
|
1296
|
+
// it is a concurrency exemption, not a toolchain exemption.
|
|
1297
|
+
threadFeatures = assertSequentialArtifact(wasmBytes, {
|
|
1298
|
+
source: resolvedOutputPath,
|
|
1299
|
+
target: resolveWasiThreadsToolchain().target,
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1325
1302
|
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1303
|
+
// Validate the compiled artifact
|
|
1304
|
+
const report = await validateArtifactWithStandards({
|
|
1305
|
+
manifest,
|
|
1306
|
+
wasmPath: resolvedOutputPath,
|
|
1307
|
+
...standardsOptions,
|
|
1308
|
+
});
|
|
1309
|
+
|
|
1310
|
+
handedOff = keepTempDir;
|
|
1311
|
+
return {
|
|
1312
|
+
compiler: useWasiThreads
|
|
1313
|
+
? threadModel === ModuleThreadModel.WASI_SEQUENTIAL
|
|
1314
|
+
? "wasm32-wasi-clang++ (wasi-sequential)"
|
|
1315
|
+
: "wasm32-wasi-clang++ (wasi-threads)"
|
|
1316
|
+
: useSystemEmscripten
|
|
1317
|
+
? "em++ (system emscripten pthreads)"
|
|
1318
|
+
: "em++ (emception)",
|
|
1319
|
+
language: compiler.language,
|
|
1320
|
+
threadModel,
|
|
1321
|
+
// Without an explicit outputPath the artifact lived in tempDir, which is
|
|
1322
|
+
// gone unless the caller kept it.
|
|
1323
|
+
outputPath:
|
|
1324
|
+
options.outputPath != null || keepTempDir ? resolvedOutputPath : null,
|
|
1325
|
+
tempDir: keepTempDir ? tempDir : null,
|
|
1326
|
+
wasmBytes,
|
|
1327
|
+
guestLink: result.guestLink,
|
|
1328
|
+
manifestWarnings: warnings,
|
|
1329
|
+
threadFeatures,
|
|
1330
|
+
report,
|
|
1331
|
+
};
|
|
1332
|
+
} finally {
|
|
1333
|
+
if (!handedOff) {
|
|
1334
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1344
1337
|
}
|
|
1345
1338
|
|
|
1346
1339
|
export async function cleanupCompilation(result) {
|