space-data-module-sdk 0.8.22 → 0.8.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/compiler/compileModule.js +378 -376
- package/src/compiler/emceptionNode.js +339 -70
- package/src/index.d.ts +16 -0
package/package.json
CHANGED
|
@@ -714,160 +714,159 @@ async function compileWithEmception(options = {}) {
|
|
|
714
714
|
invokeSource,
|
|
715
715
|
exportedSymbols,
|
|
716
716
|
outputPath,
|
|
717
|
+
tempDir,
|
|
717
718
|
compileOptions,
|
|
718
719
|
} = options;
|
|
719
|
-
const tempDir = await mkdtemp(
|
|
720
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
721
|
-
);
|
|
722
720
|
const resolvedOutputPath = path.resolve(
|
|
723
721
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
724
722
|
);
|
|
725
723
|
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
const wasmOutputPath = path.posix.join(workDir, "module.wasm");
|
|
739
|
-
|
|
740
|
-
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
741
|
-
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
724
|
+
return runWithEmceptionLock(async (emception) => {
|
|
725
|
+
const workDir = "/working/space-data-module-sdk-compile";
|
|
726
|
+
const runtimeIncludeDir = path.posix.join(workDir, "flatbuffers-runtime");
|
|
727
|
+
const sourcePath = path.posix.join(workDir, `module.${sourceExtension}`);
|
|
728
|
+
const manifestSourcePath = path.posix.join(workDir, "plugin-manifest-exports.cpp");
|
|
729
|
+
const invokeHeaderPath = path.posix.join(workDir, "space_data_module_invoke.h");
|
|
730
|
+
const invokeSourcePath = path.posix.join(workDir, "plugin-invoke-bridge.cpp");
|
|
731
|
+
const sourceObjectPath = path.posix.join(workDir, "module.o");
|
|
732
|
+
const linkObjectPath = path.posix.join(workDir, "module-link.o");
|
|
733
|
+
const manifestObjectPath = path.posix.join(workDir, "plugin-manifest-exports.o");
|
|
734
|
+
const invokeObjectPath = path.posix.join(workDir, "plugin-invoke-bridge.o");
|
|
735
|
+
const wasmOutputPath = path.posix.join(workDir, "module.wasm");
|
|
742
736
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
|
|
746
|
-
await writeFilesToEmception(emception, workDir, schemaHeaders);
|
|
747
|
-
emception.writeFile(sourcePath, sourceCode);
|
|
748
|
-
emception.writeFile(manifestSourcePath, manifestSource);
|
|
749
|
-
emception.writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
750
|
-
emception.writeFile(invokeSourcePath, invokeSource);
|
|
751
|
-
|
|
752
|
-
const commands = [
|
|
753
|
-
[
|
|
754
|
-
sourceCompilerCommand,
|
|
755
|
-
"-c",
|
|
756
|
-
sourcePath,
|
|
757
|
-
`-I${workDir}`,
|
|
758
|
-
`-I${runtimeIncludeDir}`,
|
|
759
|
-
...buildSourceCompilerArgs(compileOptions),
|
|
760
|
-
"-o",
|
|
761
|
-
sourceObjectPath,
|
|
762
|
-
],
|
|
763
|
-
];
|
|
764
|
-
|
|
765
|
-
for (const command of commands) {
|
|
766
|
-
const result = emception.run(command.join(" "));
|
|
767
|
-
if (result.returncode !== 0) {
|
|
768
|
-
throw new Error(
|
|
769
|
-
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
770
|
-
);
|
|
771
|
-
}
|
|
772
|
-
}
|
|
773
|
-
|
|
774
|
-
const sourceObjectBytes = new Uint8Array(emception.readFile(sourceObjectPath));
|
|
775
|
-
const guestLink = deriveGuestLinkRenameArgs({
|
|
776
|
-
objectBytes: sourceObjectBytes,
|
|
777
|
-
pluginId: manifest?.pluginId,
|
|
778
|
-
methodIds: Array.isArray(manifest?.methods)
|
|
779
|
-
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
780
|
-
: [],
|
|
781
|
-
});
|
|
737
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
738
|
+
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
782
739
|
|
|
783
|
-
|
|
740
|
+
try {
|
|
741
|
+
emception.FS.mkdirTree(workDir);
|
|
742
|
+
await writeFilesToEmception(emception, runtimeIncludeDir, runtimeHeaders);
|
|
743
|
+
await writeFilesToEmception(emception, workDir, schemaHeaders);
|
|
744
|
+
emception.writeFile(sourcePath, sourceCode);
|
|
745
|
+
emception.writeFile(manifestSourcePath, manifestSource);
|
|
746
|
+
emception.writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
747
|
+
emception.writeFile(invokeSourcePath, invokeSource);
|
|
748
|
+
|
|
749
|
+
const commands = [
|
|
750
|
+
[
|
|
784
751
|
sourceCompilerCommand,
|
|
785
752
|
"-c",
|
|
786
753
|
sourcePath,
|
|
787
754
|
`-I${workDir}`,
|
|
788
755
|
`-I${runtimeIncludeDir}`,
|
|
789
|
-
...guestLink.renameArgs,
|
|
790
756
|
...buildSourceCompilerArgs(compileOptions),
|
|
791
757
|
"-o",
|
|
792
|
-
|
|
793
|
-
]
|
|
794
|
-
|
|
795
|
-
|
|
758
|
+
sourceObjectPath,
|
|
759
|
+
],
|
|
760
|
+
];
|
|
761
|
+
|
|
762
|
+
for (const command of commands) {
|
|
763
|
+
const result = emception.run(command.join(" "));
|
|
764
|
+
if (result.returncode !== 0) {
|
|
796
765
|
throw new Error(
|
|
797
|
-
`Compilation failed with ${
|
|
766
|
+
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
798
767
|
);
|
|
799
768
|
}
|
|
769
|
+
}
|
|
800
770
|
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
...args,
|
|
828
|
-
"-o",
|
|
829
|
-
wasmOutputPath,
|
|
830
|
-
],
|
|
831
|
-
];
|
|
832
|
-
for (const command of remainingCommands) {
|
|
833
|
-
const result = emception.run(command.join(" "));
|
|
834
|
-
if (result.returncode !== 0) {
|
|
835
|
-
throw new Error(
|
|
836
|
-
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
837
|
-
);
|
|
838
|
-
}
|
|
839
|
-
}
|
|
771
|
+
const sourceObjectBytes = new Uint8Array(emception.readFile(sourceObjectPath));
|
|
772
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
773
|
+
objectBytes: sourceObjectBytes,
|
|
774
|
+
pluginId: manifest?.pluginId,
|
|
775
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
776
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
777
|
+
: [],
|
|
778
|
+
});
|
|
779
|
+
|
|
780
|
+
const linkCompileCommand = [
|
|
781
|
+
sourceCompilerCommand,
|
|
782
|
+
"-c",
|
|
783
|
+
sourcePath,
|
|
784
|
+
`-I${workDir}`,
|
|
785
|
+
`-I${runtimeIncludeDir}`,
|
|
786
|
+
...guestLink.renameArgs,
|
|
787
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
788
|
+
"-o",
|
|
789
|
+
linkObjectPath,
|
|
790
|
+
];
|
|
791
|
+
const linkCompileResult = emception.run(linkCompileCommand.join(" "));
|
|
792
|
+
if (linkCompileResult.returncode !== 0) {
|
|
793
|
+
throw new Error(
|
|
794
|
+
`Compilation failed with ${linkCompileCommand[0]} (emception): ${linkCompileResult.stderr || linkCompileResult.stdout}`,
|
|
795
|
+
);
|
|
796
|
+
}
|
|
840
797
|
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
798
|
+
// The embedded manifest and the invoke bridge get the module source's
|
|
799
|
+
// own optimization flags. Without them both objects built at -O0 and
|
|
800
|
+
// every payload byte crossing the bridge (the std::vector copies of
|
|
801
|
+
// each request and response) cost hundreds of interpreted wasm
|
|
802
|
+
// instructions. The wasi path already compiles them this way.
|
|
803
|
+
const remainingCommands = [
|
|
804
|
+
[
|
|
805
|
+
"em++",
|
|
806
|
+
"-c",
|
|
807
|
+
manifestSourcePath,
|
|
808
|
+
"-std=c++17",
|
|
809
|
+
`-I${workDir}`,
|
|
810
|
+
`-I${runtimeIncludeDir}`,
|
|
811
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
812
|
+
"-o",
|
|
813
|
+
manifestObjectPath,
|
|
814
|
+
],
|
|
815
|
+
[
|
|
816
|
+
"em++",
|
|
817
|
+
"-c",
|
|
818
|
+
invokeSourcePath,
|
|
819
|
+
"-std=c++17",
|
|
820
|
+
`-I${workDir}`,
|
|
821
|
+
`-I${runtimeIncludeDir}`,
|
|
822
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
823
|
+
"-o",
|
|
824
|
+
invokeObjectPath,
|
|
825
|
+
],
|
|
826
|
+
[
|
|
827
|
+
"em++",
|
|
828
|
+
sourceObjectPath,
|
|
829
|
+
manifestObjectPath,
|
|
830
|
+
invokeObjectPath,
|
|
831
|
+
...args,
|
|
832
|
+
"-o",
|
|
833
|
+
wasmOutputPath,
|
|
834
|
+
],
|
|
835
|
+
];
|
|
836
|
+
for (const command of remainingCommands) {
|
|
837
|
+
const result = emception.run(command.join(" "));
|
|
838
|
+
if (result.returncode !== 0) {
|
|
839
|
+
throw new Error(
|
|
840
|
+
`Compilation failed with ${command[0]} (emception): ${result.stderr || result.stdout}`,
|
|
841
|
+
);
|
|
864
842
|
}
|
|
865
843
|
}
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
844
|
+
|
|
845
|
+
const wasmBytes = new Uint8Array(emception.readFile(wasmOutputPath));
|
|
846
|
+
const linkObjectBytes = new Uint8Array(emception.readFile(linkObjectPath));
|
|
847
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
848
|
+
return {
|
|
849
|
+
wasmBytes,
|
|
850
|
+
outputPath: resolvedOutputPath,
|
|
851
|
+
guestLink: {
|
|
852
|
+
format: "wasm-object",
|
|
853
|
+
language,
|
|
854
|
+
symbolPrefix: guestLink.prefix,
|
|
855
|
+
methodSymbols: guestLink.methodSymbols,
|
|
856
|
+
threadModel:
|
|
857
|
+
compileOptions.threadModel ?? ModuleThreadModel.SINGLE_THREAD,
|
|
858
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
859
|
+
objectBytes: linkObjectBytes,
|
|
860
|
+
},
|
|
861
|
+
};
|
|
862
|
+
} finally {
|
|
863
|
+
try {
|
|
864
|
+
removeEmceptionDirectory(emception, workDir);
|
|
865
|
+
} catch {
|
|
866
|
+
// Best-effort cleanup only; the shared emception instance remains usable.
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
});
|
|
871
870
|
}
|
|
872
871
|
|
|
873
872
|
async function ensureSystemCompilerAvailable(command) {
|
|
@@ -911,11 +910,9 @@ async function compileWithSystemEmscripten(options = {}) {
|
|
|
911
910
|
invokeSource,
|
|
912
911
|
exportedSymbols,
|
|
913
912
|
outputPath,
|
|
913
|
+
tempDir,
|
|
914
914
|
compileOptions,
|
|
915
915
|
} = options;
|
|
916
|
-
const tempDir = await mkdtemp(
|
|
917
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
918
|
-
);
|
|
919
916
|
const resolvedOutputPath = path.resolve(
|
|
920
917
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
921
918
|
);
|
|
@@ -930,99 +927,95 @@ async function compileWithSystemEmscripten(options = {}) {
|
|
|
930
927
|
const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
|
|
931
928
|
const wasmOutputPath = path.join(tempDir, "module.wasm");
|
|
932
929
|
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
});
|
|
930
|
+
await ensureSystemCompilerAvailable(sourceCompilerCommand);
|
|
931
|
+
await ensureSystemCompilerAvailable("em++");
|
|
932
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
933
|
+
const args = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
934
|
+
await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
|
|
935
|
+
await writeFilesToDirectory(tempDir, schemaHeaders);
|
|
936
|
+
await writeFile(sourcePath, sourceCode);
|
|
937
|
+
await writeFile(manifestSourcePath, manifestSource);
|
|
938
|
+
await writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
939
|
+
await writeFile(invokeSourcePath, invokeSource);
|
|
940
|
+
|
|
941
|
+
const sourceCompileArgs = [
|
|
942
|
+
"-c",
|
|
943
|
+
sourcePath,
|
|
944
|
+
`-I${tempDir}`,
|
|
945
|
+
`-I${runtimeIncludeDir}`,
|
|
946
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
947
|
+
"-o",
|
|
948
|
+
sourceObjectPath,
|
|
949
|
+
];
|
|
950
|
+
await runSystemCompiler(sourceCompilerCommand, sourceCompileArgs);
|
|
951
|
+
|
|
952
|
+
const sourceObjectBytes = new Uint8Array(await readFile(sourceObjectPath));
|
|
953
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
954
|
+
objectBytes: sourceObjectBytes,
|
|
955
|
+
pluginId: manifest?.pluginId,
|
|
956
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
957
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
958
|
+
: [],
|
|
959
|
+
});
|
|
964
960
|
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
961
|
+
const linkCompileArgs = [
|
|
962
|
+
"-c",
|
|
963
|
+
sourcePath,
|
|
964
|
+
`-I${tempDir}`,
|
|
965
|
+
`-I${runtimeIncludeDir}`,
|
|
966
|
+
...guestLink.renameArgs,
|
|
967
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
968
|
+
"-o",
|
|
969
|
+
linkObjectPath,
|
|
970
|
+
];
|
|
971
|
+
await runSystemCompiler(sourceCompilerCommand, linkCompileArgs);
|
|
972
|
+
// Same optimization flags as the module source (see compileWithEmception).
|
|
973
|
+
// buildSourceCompilerArgs carries -pthread whenever usesPthreadCompileFlags.
|
|
974
|
+
await runSystemCompiler("em++", [
|
|
975
|
+
"-c",
|
|
976
|
+
manifestSourcePath,
|
|
977
|
+
"-std=c++17",
|
|
978
|
+
`-I${tempDir}`,
|
|
979
|
+
`-I${runtimeIncludeDir}`,
|
|
980
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
981
|
+
"-o",
|
|
982
|
+
manifestObjectPath,
|
|
983
|
+
]);
|
|
984
|
+
await runSystemCompiler("em++", [
|
|
985
|
+
"-c",
|
|
986
|
+
invokeSourcePath,
|
|
987
|
+
"-std=c++17",
|
|
988
|
+
`-I${tempDir}`,
|
|
989
|
+
`-I${runtimeIncludeDir}`,
|
|
990
|
+
...buildSourceCompilerArgs(compileOptions),
|
|
991
|
+
"-o",
|
|
992
|
+
invokeObjectPath,
|
|
993
|
+
]);
|
|
994
|
+
await runSystemCompiler("em++", [
|
|
995
|
+
sourceObjectPath,
|
|
996
|
+
manifestObjectPath,
|
|
997
|
+
invokeObjectPath,
|
|
998
|
+
...args,
|
|
999
|
+
"-o",
|
|
1000
|
+
wasmOutputPath,
|
|
1001
|
+
]);
|
|
1004
1002
|
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
};
|
|
1022
|
-
} catch (error) {
|
|
1023
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
1024
|
-
throw error;
|
|
1025
|
-
}
|
|
1003
|
+
const wasmBytes = new Uint8Array(await readFile(wasmOutputPath));
|
|
1004
|
+
const linkObjectBytes = new Uint8Array(await readFile(linkObjectPath));
|
|
1005
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1006
|
+
return {
|
|
1007
|
+
wasmBytes,
|
|
1008
|
+
outputPath: resolvedOutputPath,
|
|
1009
|
+
guestLink: {
|
|
1010
|
+
format: "wasm-object",
|
|
1011
|
+
language,
|
|
1012
|
+
symbolPrefix: guestLink.prefix,
|
|
1013
|
+
methodSymbols: guestLink.methodSymbols,
|
|
1014
|
+
threadModel: compileOptions.threadModel,
|
|
1015
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
1016
|
+
objectBytes: linkObjectBytes,
|
|
1017
|
+
},
|
|
1018
|
+
};
|
|
1026
1019
|
}
|
|
1027
1020
|
|
|
1028
1021
|
async function runWasiCompiler(command, args, options = {}) {
|
|
@@ -1061,6 +1054,7 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1061
1054
|
invokeSource,
|
|
1062
1055
|
exportedSymbols,
|
|
1063
1056
|
outputPath,
|
|
1057
|
+
tempDir,
|
|
1064
1058
|
compileOptions,
|
|
1065
1059
|
} = options;
|
|
1066
1060
|
const toolchain = resolveWasiThreadsToolchain();
|
|
@@ -1068,9 +1062,6 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1068
1062
|
const sourceDriver = isCpp ? toolchain.clangxx : toolchain.clang;
|
|
1069
1063
|
const cppDriver = toolchain.clangxx;
|
|
1070
1064
|
|
|
1071
|
-
const tempDir = await mkdtemp(
|
|
1072
|
-
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
1073
|
-
);
|
|
1074
1065
|
const resolvedOutputPath = path.resolve(
|
|
1075
1066
|
outputPath ?? path.join(tempDir, "module.wasm"),
|
|
1076
1067
|
);
|
|
@@ -1085,98 +1076,92 @@ async function compileWithWasiThreads(options = {}) {
|
|
|
1085
1076
|
const invokeObjectPath = path.join(tempDir, "plugin-invoke-bridge.o");
|
|
1086
1077
|
const wasmOutputPath = path.join(tempDir, "module.wasm");
|
|
1087
1078
|
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
]);
|
|
1079
|
+
const { runtimeHeaders, schemaHeaders } = await getInvokeCppSupportFiles();
|
|
1080
|
+
const linkArgs = buildCompilerArgs(exportedSymbols, compileOptions);
|
|
1081
|
+
await writeFilesToDirectory(runtimeIncludeDir, runtimeHeaders);
|
|
1082
|
+
await writeFilesToDirectory(tempDir, schemaHeaders);
|
|
1083
|
+
await writeFile(sourcePath, sourceCode);
|
|
1084
|
+
await writeFile(manifestSourcePath, manifestSource);
|
|
1085
|
+
await writeFile(invokeHeaderPath, invokeHeaderSource);
|
|
1086
|
+
await writeFile(invokeSourcePath, invokeSource);
|
|
1087
|
+
|
|
1088
|
+
const includeArgs = [`-I${tempDir}`, `-I${runtimeIncludeDir}`];
|
|
1089
|
+
const sourceCompilerArgs = buildSourceCompilerArgs(compileOptions);
|
|
1090
|
+
|
|
1091
|
+
await runWasiCompiler(sourceDriver, [
|
|
1092
|
+
...toolchain.toolchainArgs,
|
|
1093
|
+
"-c",
|
|
1094
|
+
sourcePath,
|
|
1095
|
+
...includeArgs,
|
|
1096
|
+
...(isCpp ? ["-std=c++17"] : []),
|
|
1097
|
+
...sourceCompilerArgs,
|
|
1098
|
+
"-o",
|
|
1099
|
+
sourceObjectPath,
|
|
1100
|
+
]);
|
|
1111
1101
|
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1102
|
+
const sourceObjectBytes = new Uint8Array(await readFile(sourceObjectPath));
|
|
1103
|
+
const guestLink = deriveGuestLinkRenameArgs({
|
|
1104
|
+
objectBytes: sourceObjectBytes,
|
|
1105
|
+
pluginId: manifest?.pluginId,
|
|
1106
|
+
methodIds: Array.isArray(manifest?.methods)
|
|
1107
|
+
? manifest.methods.map((method) => String(method?.methodId ?? ""))
|
|
1108
|
+
: [],
|
|
1109
|
+
});
|
|
1120
1110
|
|
|
1121
|
-
|
|
1111
|
+
await runWasiCompiler(sourceDriver, [
|
|
1112
|
+
...toolchain.toolchainArgs,
|
|
1113
|
+
"-c",
|
|
1114
|
+
sourcePath,
|
|
1115
|
+
...includeArgs,
|
|
1116
|
+
...(isCpp ? ["-std=c++17"] : []),
|
|
1117
|
+
...guestLink.renameArgs,
|
|
1118
|
+
...sourceCompilerArgs,
|
|
1119
|
+
"-o",
|
|
1120
|
+
linkObjectPath,
|
|
1121
|
+
]);
|
|
1122
|
+
|
|
1123
|
+
for (const [srcPath, objPath] of [
|
|
1124
|
+
[manifestSourcePath, manifestObjectPath],
|
|
1125
|
+
[invokeSourcePath, invokeObjectPath],
|
|
1126
|
+
]) {
|
|
1127
|
+
await runWasiCompiler(cppDriver, [
|
|
1122
1128
|
...toolchain.toolchainArgs,
|
|
1123
1129
|
"-c",
|
|
1124
|
-
|
|
1130
|
+
srcPath,
|
|
1131
|
+
"-std=c++17",
|
|
1125
1132
|
...includeArgs,
|
|
1126
|
-
...(isCpp ? ["-std=c++17"] : []),
|
|
1127
|
-
...guestLink.renameArgs,
|
|
1128
1133
|
...sourceCompilerArgs,
|
|
1129
1134
|
"-o",
|
|
1130
|
-
|
|
1135
|
+
objPath,
|
|
1131
1136
|
]);
|
|
1137
|
+
}
|
|
1132
1138
|
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
...includeArgs,
|
|
1143
|
-
...sourceCompilerArgs,
|
|
1144
|
-
"-o",
|
|
1145
|
-
objPath,
|
|
1146
|
-
]);
|
|
1147
|
-
}
|
|
1148
|
-
|
|
1149
|
-
await runWasiCompiler(cppDriver, [
|
|
1150
|
-
...toolchain.toolchainArgs,
|
|
1151
|
-
sourceObjectPath,
|
|
1152
|
-
manifestObjectPath,
|
|
1153
|
-
invokeObjectPath,
|
|
1154
|
-
...linkArgs,
|
|
1155
|
-
"-o",
|
|
1156
|
-
wasmOutputPath,
|
|
1157
|
-
]);
|
|
1139
|
+
await runWasiCompiler(cppDriver, [
|
|
1140
|
+
...toolchain.toolchainArgs,
|
|
1141
|
+
sourceObjectPath,
|
|
1142
|
+
manifestObjectPath,
|
|
1143
|
+
invokeObjectPath,
|
|
1144
|
+
...linkArgs,
|
|
1145
|
+
"-o",
|
|
1146
|
+
wasmOutputPath,
|
|
1147
|
+
]);
|
|
1158
1148
|
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
};
|
|
1176
|
-
} catch (error) {
|
|
1177
|
-
await rm(tempDir, { recursive: true, force: true });
|
|
1178
|
-
throw error;
|
|
1179
|
-
}
|
|
1149
|
+
const wasmBytes = new Uint8Array(await readFile(wasmOutputPath));
|
|
1150
|
+
const linkObjectBytes = new Uint8Array(await readFile(linkObjectPath));
|
|
1151
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1152
|
+
return {
|
|
1153
|
+
wasmBytes,
|
|
1154
|
+
outputPath: resolvedOutputPath,
|
|
1155
|
+
guestLink: {
|
|
1156
|
+
format: "wasm-object",
|
|
1157
|
+
language,
|
|
1158
|
+
symbolPrefix: guestLink.prefix,
|
|
1159
|
+
methodSymbols: guestLink.methodSymbols,
|
|
1160
|
+
threadModel: compileOptions.threadModel,
|
|
1161
|
+
capabilities: compileOptions.guestLinkCapabilities,
|
|
1162
|
+
objectBytes: linkObjectBytes,
|
|
1163
|
+
},
|
|
1164
|
+
};
|
|
1180
1165
|
}
|
|
1181
1166
|
|
|
1182
1167
|
export async function compileModuleFromSource(options = {}) {
|
|
@@ -1234,9 +1219,6 @@ export async function compileModuleFromSource(options = {}) {
|
|
|
1234
1219
|
),
|
|
1235
1220
|
];
|
|
1236
1221
|
|
|
1237
|
-
let wasmBytes;
|
|
1238
|
-
let resolvedOutputPath = null;
|
|
1239
|
-
let tempDir = null;
|
|
1240
1222
|
const threadModel = resolveThreadModel({
|
|
1241
1223
|
manifest,
|
|
1242
1224
|
threadModel: options.threadModel,
|
|
@@ -1266,72 +1248,92 @@ export async function compileModuleFromSource(options = {}) {
|
|
|
1266
1248
|
: useSystemEmscripten
|
|
1267
1249
|
? compileWithSystemEmscripten
|
|
1268
1250
|
: compileWithEmception;
|
|
1269
|
-
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
invokeSource,
|
|
1278
|
-
exportedSymbols,
|
|
1279
|
-
outputPath: options.outputPath,
|
|
1280
|
-
compileOptions,
|
|
1281
|
-
});
|
|
1282
|
-
wasmBytes = appendWasmCustomSection(
|
|
1283
|
-
result.wasmBytes,
|
|
1284
|
-
SDS_MANIFEST_SECTION_NAME,
|
|
1285
|
-
encodePluginManifest(manifest),
|
|
1251
|
+
// Scratch for the toolchain and, without an explicit outputPath, the home of
|
|
1252
|
+
// the emitted artifact. Removed in finally on success, failure and throw.
|
|
1253
|
+
// `keepTempDir: true` hands it to the caller on success instead
|
|
1254
|
+
// (result.tempDir, freed by cleanupCompilation); a failed compile never
|
|
1255
|
+
// hands anything off, so its dir is always removed.
|
|
1256
|
+
const keepTempDir = options.keepTempDir === true;
|
|
1257
|
+
const tempDir = await mkdtemp(
|
|
1258
|
+
path.join(os.tmpdir(), "space-data-module-sdk-compile-"),
|
|
1286
1259
|
);
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
// to quietly build a plain wasm32-wasip1 object under the sequential model:
|
|
1303
|
-
// it is a concurrency exemption, not a toolchain exemption.
|
|
1304
|
-
threadFeatures = assertSequentialArtifact(wasmBytes, {
|
|
1305
|
-
source: resolvedOutputPath,
|
|
1306
|
-
target: resolveWasiThreadsToolchain().target,
|
|
1260
|
+
let handedOff = false;
|
|
1261
|
+
try {
|
|
1262
|
+
const result = await compileFunction({
|
|
1263
|
+
manifest,
|
|
1264
|
+
language: compiler.language,
|
|
1265
|
+
sourceCompilerCommand: compiler.command,
|
|
1266
|
+
sourceExtension: compiler.extension,
|
|
1267
|
+
sourceCode,
|
|
1268
|
+
manifestSource,
|
|
1269
|
+
invokeHeaderSource,
|
|
1270
|
+
invokeSource,
|
|
1271
|
+
exportedSymbols,
|
|
1272
|
+
outputPath: options.outputPath,
|
|
1273
|
+
tempDir,
|
|
1274
|
+
compileOptions,
|
|
1307
1275
|
});
|
|
1308
|
-
|
|
1276
|
+
const wasmBytes = appendWasmCustomSection(
|
|
1277
|
+
result.wasmBytes,
|
|
1278
|
+
SDS_MANIFEST_SECTION_NAME,
|
|
1279
|
+
encodePluginManifest(manifest),
|
|
1280
|
+
);
|
|
1281
|
+
const resolvedOutputPath = result.outputPath;
|
|
1282
|
+
await writeFile(resolvedOutputPath, wasmBytes);
|
|
1309
1283
|
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1284
|
+
// Enforce the isomorphic-pthreads guardrail on the EMITTED artifact: a build
|
|
1285
|
+
// that claims the pthreads thread model but did not emit a shared-memory /
|
|
1286
|
+
// atomics wasm must FAIL the compile here rather than ship. See
|
|
1287
|
+
// docs/isomorphic-pthreads.md.
|
|
1288
|
+
let threadFeatures = null;
|
|
1289
|
+
if (threadModel === ModuleThreadModel.EMSCRIPTEN_PTHREADS) {
|
|
1290
|
+
threadFeatures = assertPthreadArtifact(wasmBytes, {
|
|
1291
|
+
source: resolvedOutputPath,
|
|
1292
|
+
});
|
|
1293
|
+
} else if (threadModel === ModuleThreadModel.WASI_SEQUENTIAL) {
|
|
1294
|
+
// Mirror guard. The target is passed so that SDN_WASI_TARGET cannot be used
|
|
1295
|
+
// to quietly build a plain wasm32-wasip1 object under the sequential model:
|
|
1296
|
+
// it is a concurrency exemption, not a toolchain exemption.
|
|
1297
|
+
threadFeatures = assertSequentialArtifact(wasmBytes, {
|
|
1298
|
+
source: resolvedOutputPath,
|
|
1299
|
+
target: resolveWasiThreadsToolchain().target,
|
|
1300
|
+
});
|
|
1301
|
+
}
|
|
1316
1302
|
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1303
|
+
// Validate the compiled artifact
|
|
1304
|
+
const report = await validateArtifactWithStandards({
|
|
1305
|
+
manifest,
|
|
1306
|
+
wasmPath: resolvedOutputPath,
|
|
1307
|
+
...standardsOptions,
|
|
1308
|
+
});
|
|
1309
|
+
|
|
1310
|
+
handedOff = keepTempDir;
|
|
1311
|
+
return {
|
|
1312
|
+
compiler: useWasiThreads
|
|
1313
|
+
? threadModel === ModuleThreadModel.WASI_SEQUENTIAL
|
|
1314
|
+
? "wasm32-wasi-clang++ (wasi-sequential)"
|
|
1315
|
+
: "wasm32-wasi-clang++ (wasi-threads)"
|
|
1316
|
+
: useSystemEmscripten
|
|
1317
|
+
? "em++ (system emscripten pthreads)"
|
|
1318
|
+
: "em++ (emception)",
|
|
1319
|
+
language: compiler.language,
|
|
1320
|
+
threadModel,
|
|
1321
|
+
// Without an explicit outputPath the artifact lived in tempDir, which is
|
|
1322
|
+
// gone unless the caller kept it.
|
|
1323
|
+
outputPath:
|
|
1324
|
+
options.outputPath != null || keepTempDir ? resolvedOutputPath : null,
|
|
1325
|
+
tempDir: keepTempDir ? tempDir : null,
|
|
1326
|
+
wasmBytes,
|
|
1327
|
+
guestLink: result.guestLink,
|
|
1328
|
+
manifestWarnings: warnings,
|
|
1329
|
+
threadFeatures,
|
|
1330
|
+
report,
|
|
1331
|
+
};
|
|
1332
|
+
} finally {
|
|
1333
|
+
if (!handedOff) {
|
|
1334
|
+
await rm(tempDir, { recursive: true, force: true });
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1335
1337
|
}
|
|
1336
1338
|
|
|
1337
1339
|
export async function cleanupCompilation(result) {
|
|
@@ -1,17 +1,41 @@
|
|
|
1
1
|
import os from "node:os";
|
|
2
2
|
import path from "node:path";
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
3
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
4
|
+
import {
|
|
5
|
+
cp,
|
|
6
|
+
lstat,
|
|
7
|
+
mkdtemp,
|
|
8
|
+
readFile,
|
|
9
|
+
readdir,
|
|
10
|
+
rename,
|
|
11
|
+
rm,
|
|
12
|
+
stat,
|
|
13
|
+
writeFile,
|
|
14
|
+
} from "node:fs/promises";
|
|
15
|
+
import { constants as fsConstants, createReadStream, readFileSync } from "node:fs";
|
|
5
16
|
import { createRequire } from "node:module";
|
|
6
17
|
import { pathToFileURL } from "node:url";
|
|
7
18
|
|
|
8
19
|
const require = createRequire(import.meta.url);
|
|
20
|
+
// Bump when the patched tree's layout or meaning changes. The patch transform's
|
|
21
|
+
// own source text is part of the cache key as well (see
|
|
22
|
+
// computeEmceptionIdentity), so editing patchEmceptionModuleSource cannot reuse
|
|
23
|
+
// a tree patched by an older transform even if this string is left alone.
|
|
9
24
|
const PATCH_VERSION = "space-data-module-sdk-emception-node-v1";
|
|
25
|
+
const PATCH_KEY_TAG = "node-v1";
|
|
10
26
|
const PATCH_MARKER_FILENAME = ".space-data-module-sdk-emception-patch";
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
27
|
+
// One shared root per (patch, emception build) lives at
|
|
28
|
+
// <tmpdir>/space-data-module-sdk-emception-<tag>-<version>-<fingerprint>-u<uid>.
|
|
29
|
+
// Every process of every consumer run by that user reuses it. Builds happen in a sibling
|
|
30
|
+
// staging dir that is renamed into place only once it is complete.
|
|
31
|
+
const PATCHED_ROOT_PREFIX = "space-data-module-sdk-emception-";
|
|
32
|
+
const STAGING_MARKER = ".staging-";
|
|
33
|
+
const STALE_MARKER = ".stale-";
|
|
34
|
+
const ORPHAN_NAME_PATTERN = new RegExp(
|
|
35
|
+
`^${PATCHED_ROOT_PREFIX.replaceAll("-", "\\-")}.+\\.(?:staging|stale)-(\\d+)-[A-Za-z0-9]+$`,
|
|
14
36
|
);
|
|
37
|
+
const RENAME_COLLISION_CODES = new Set(["ENOTEMPTY", "EEXIST", "EPERM", "EBUSY"]);
|
|
38
|
+
const MAX_PUBLISH_ATTEMPTS = 4;
|
|
15
39
|
const FILE_URL_FETCH_PATCH_FLAG =
|
|
16
40
|
"__spaceDataModuleSdkFileUrlFetchPatched";
|
|
17
41
|
|
|
@@ -84,90 +108,335 @@ function installNodeRuntimeShims() {
|
|
|
84
108
|
}
|
|
85
109
|
}
|
|
86
110
|
|
|
87
|
-
|
|
88
|
-
|
|
111
|
+
// Pure transform of one emception .mjs file. Its source text is hashed into
|
|
112
|
+
// the cache key, so any edit here yields a fresh patched root.
|
|
113
|
+
function patchEmceptionModuleSource(fileName, source) {
|
|
114
|
+
let patched = source.replaceAll(
|
|
115
|
+
'scriptDirectory=__dirname+"/"',
|
|
116
|
+
'scriptDirectory=(new URL(".", import.meta.url)).pathname',
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
if (fileName === "emception.mjs") {
|
|
120
|
+
patched = patched.replace(
|
|
121
|
+
"this.#fs = await new FileSystem();",
|
|
122
|
+
[
|
|
123
|
+
"this.#fs = await new FileSystem({",
|
|
124
|
+
" locateFile: (file, scriptDirectory) => scriptDirectory + file,",
|
|
125
|
+
' cache: "/tmp/emception-cache",',
|
|
126
|
+
" });",
|
|
127
|
+
].join("\n"),
|
|
128
|
+
);
|
|
129
|
+
patched = patched.replace(
|
|
130
|
+
"const config = {",
|
|
131
|
+
[
|
|
132
|
+
"const config = {",
|
|
133
|
+
" locateFile: (file, scriptDirectory) => scriptDirectory + file,",
|
|
134
|
+
].join("\n"),
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
if (fileName === "FileSystem.mjs") {
|
|
139
|
+
patched = patched.replace(
|
|
140
|
+
[
|
|
141
|
+
" if (!this.exists(cache)) {",
|
|
142
|
+
" this.persist(cache);",
|
|
143
|
+
" }",
|
|
144
|
+
" await this.pull();",
|
|
145
|
+
].join("\n"),
|
|
146
|
+
[
|
|
147
|
+
' if (typeof indexedDB !== "undefined") {',
|
|
148
|
+
" if (!this.exists(cache)) {",
|
|
149
|
+
" this.persist(cache);",
|
|
150
|
+
" }",
|
|
151
|
+
" await this.pull();",
|
|
152
|
+
" }",
|
|
153
|
+
].join("\n"),
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return patched;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Relative POSIX paths of every regular file under rootDir, sorted.
|
|
161
|
+
async function listTreeFiles(rootDir, relativeDir = "") {
|
|
162
|
+
const entries = await readdir(path.join(rootDir, relativeDir), {
|
|
163
|
+
withFileTypes: true,
|
|
164
|
+
});
|
|
165
|
+
const files = [];
|
|
89
166
|
for (const entry of entries) {
|
|
90
|
-
const
|
|
167
|
+
const relativePath = relativeDir
|
|
168
|
+
? path.posix.join(relativeDir, entry.name)
|
|
169
|
+
: entry.name;
|
|
91
170
|
if (entry.isDirectory()) {
|
|
92
|
-
await
|
|
93
|
-
|
|
171
|
+
files.push(...(await listTreeFiles(rootDir, relativePath)));
|
|
172
|
+
} else if (entry.isFile()) {
|
|
173
|
+
files.push(relativePath);
|
|
94
174
|
}
|
|
95
|
-
|
|
175
|
+
}
|
|
176
|
+
return files.sort();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function patchEmceptionModuleTree(rootDir, files) {
|
|
180
|
+
for (const relativePath of files) {
|
|
181
|
+
if (!relativePath.endsWith(".mjs")) {
|
|
96
182
|
continue;
|
|
97
183
|
}
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
184
|
+
const fullPath = path.join(rootDir, relativePath);
|
|
185
|
+
const source = await readFile(fullPath, "utf8");
|
|
186
|
+
const patched = patchEmceptionModuleSource(
|
|
187
|
+
path.posix.basename(relativePath),
|
|
188
|
+
source,
|
|
103
189
|
);
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
source = source.replace(
|
|
107
|
-
"this.#fs = await new FileSystem();",
|
|
108
|
-
[
|
|
109
|
-
"this.#fs = await new FileSystem({",
|
|
110
|
-
" locateFile: (file, scriptDirectory) => scriptDirectory + file,",
|
|
111
|
-
' cache: "/tmp/emception-cache",',
|
|
112
|
-
" });",
|
|
113
|
-
].join("\n"),
|
|
114
|
-
);
|
|
115
|
-
source = source.replace(
|
|
116
|
-
"const config = {",
|
|
117
|
-
[
|
|
118
|
-
"const config = {",
|
|
119
|
-
" locateFile: (file, scriptDirectory) => scriptDirectory + file,",
|
|
120
|
-
].join("\n"),
|
|
121
|
-
);
|
|
190
|
+
if (patched !== source) {
|
|
191
|
+
await writeFile(fullPath, patched, "utf8");
|
|
122
192
|
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
123
195
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
" await this.pull();",
|
|
131
|
-
].join("\n"),
|
|
132
|
-
[
|
|
133
|
-
' if (typeof indexedDB !== "undefined") {',
|
|
134
|
-
" if (!this.exists(cache)) {",
|
|
135
|
-
" this.persist(cache);",
|
|
136
|
-
" }",
|
|
137
|
-
" await this.pull();",
|
|
138
|
-
" }",
|
|
139
|
-
].join("\n"),
|
|
196
|
+
async function readEmceptionPackageVersion(sourceRoot) {
|
|
197
|
+
let directory = sourceRoot;
|
|
198
|
+
for (;;) {
|
|
199
|
+
try {
|
|
200
|
+
const packageJson = JSON.parse(
|
|
201
|
+
await readFile(path.join(directory, "package.json"), "utf8"),
|
|
140
202
|
);
|
|
203
|
+
if (packageJson?.name === "sdn-emception") {
|
|
204
|
+
return String(packageJson.version ?? "unknown");
|
|
205
|
+
}
|
|
206
|
+
} catch {
|
|
207
|
+
// Keep walking up.
|
|
141
208
|
}
|
|
209
|
+
const parent = path.dirname(directory);
|
|
210
|
+
if (parent === directory) {
|
|
211
|
+
return "unknown";
|
|
212
|
+
}
|
|
213
|
+
directory = parent;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
142
216
|
|
|
143
|
-
|
|
217
|
+
// The cache key: the patch version, the patch transform, the emception
|
|
218
|
+
// package version, and a sha256 over every source file's path, size and
|
|
219
|
+
// bytes. Two installs with the same version but different bytes never share
|
|
220
|
+
// a root.
|
|
221
|
+
async function computeEmceptionIdentity(sourceRoot) {
|
|
222
|
+
const version = await readEmceptionPackageVersion(sourceRoot);
|
|
223
|
+
const files = await listTreeFiles(sourceRoot);
|
|
224
|
+
const hash = createHash("sha256");
|
|
225
|
+
hash.update(`${PATCH_VERSION}\0${patchEmceptionModuleSource.toString()}\0`);
|
|
226
|
+
hash.update(`${version}\0`);
|
|
227
|
+
for (const relativePath of files) {
|
|
228
|
+
const fullPath = path.join(sourceRoot, relativePath);
|
|
229
|
+
const { size } = await stat(fullPath);
|
|
230
|
+
hash.update(`${relativePath}\0${size}\0`);
|
|
231
|
+
for await (const chunk of createReadStream(fullPath)) {
|
|
232
|
+
hash.update(chunk);
|
|
233
|
+
}
|
|
234
|
+
hash.update("\0");
|
|
144
235
|
}
|
|
236
|
+
const fingerprint = hash.digest("hex");
|
|
237
|
+
const safeVersion = version.replace(/[^A-Za-z0-9._]/g, "_");
|
|
238
|
+
// Per user: where the tmp dir is shared (Linux /tmp), users never collide.
|
|
239
|
+
const uid = currentUid();
|
|
240
|
+
return {
|
|
241
|
+
version,
|
|
242
|
+
fingerprint,
|
|
243
|
+
files,
|
|
244
|
+
key: `${PATCH_KEY_TAG}-${safeVersion}-${fingerprint.slice(0, 16)}${uid === null ? "" : `-u${uid}`}`,
|
|
245
|
+
};
|
|
145
246
|
}
|
|
146
247
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
248
|
+
// The content check on reuse. The marker is written last inside the staging
|
|
249
|
+
// dir, so a published root always has one; this also catches a root whose
|
|
250
|
+
// files were later deleted or truncated (tmp cleaners purge by age).
|
|
251
|
+
async function isPatchedRootValid(rootDir, identity) {
|
|
252
|
+
// The root is imported as code, so it must be a real directory that this
|
|
253
|
+
// user owns and nobody else can write. Anything else is never trusted.
|
|
254
|
+
try {
|
|
255
|
+
const rootStat = await lstat(rootDir);
|
|
256
|
+
if (!rootStat.isDirectory()) {
|
|
257
|
+
return false;
|
|
258
|
+
}
|
|
259
|
+
const uid = currentUid();
|
|
260
|
+
if (uid !== null && (rootStat.uid !== uid || (rootStat.mode & 0o022) !== 0)) {
|
|
261
|
+
return false;
|
|
262
|
+
}
|
|
263
|
+
} catch {
|
|
264
|
+
return false;
|
|
265
|
+
}
|
|
266
|
+
let marker;
|
|
267
|
+
try {
|
|
268
|
+
marker = JSON.parse(
|
|
269
|
+
await readFile(path.join(rootDir, PATCH_MARKER_FILENAME), "utf8"),
|
|
270
|
+
);
|
|
271
|
+
} catch {
|
|
272
|
+
return false;
|
|
273
|
+
}
|
|
274
|
+
if (
|
|
275
|
+
marker?.patchVersion !== PATCH_VERSION ||
|
|
276
|
+
marker.emceptionVersion !== identity.version ||
|
|
277
|
+
marker.fingerprint !== identity.fingerprint ||
|
|
278
|
+
!Array.isArray(marker.files) ||
|
|
279
|
+
marker.files.length !== identity.files.length
|
|
280
|
+
) {
|
|
281
|
+
return false;
|
|
282
|
+
}
|
|
283
|
+
for (let index = 0; index < marker.files.length; index += 1) {
|
|
284
|
+
const [relativePath, size] = marker.files[index] ?? [];
|
|
285
|
+
if (relativePath !== identity.files[index]) {
|
|
286
|
+
return false;
|
|
287
|
+
}
|
|
288
|
+
try {
|
|
289
|
+
const entry = await stat(path.join(rootDir, relativePath));
|
|
290
|
+
if (!entry.isFile() || entry.size !== size) {
|
|
291
|
+
return false;
|
|
157
292
|
}
|
|
293
|
+
} catch {
|
|
294
|
+
return false;
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
return true;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
function currentUid() {
|
|
301
|
+
return typeof process.getuid === "function" ? process.getuid() : null;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function isProcessAlive(pid) {
|
|
305
|
+
try {
|
|
306
|
+
process.kill(pid, 0);
|
|
307
|
+
return true;
|
|
308
|
+
} catch (error) {
|
|
309
|
+
return error?.code === "EPERM";
|
|
310
|
+
}
|
|
311
|
+
}
|
|
158
312
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
313
|
+
// Staging and stale dirs carry their creator's pid. A process that died
|
|
314
|
+
// mid-build leaves one behind; nothing else would ever remove it.
|
|
315
|
+
async function removeOrphanedBuildDirs(cacheRoot) {
|
|
316
|
+
let names;
|
|
317
|
+
try {
|
|
318
|
+
names = await readdir(cacheRoot);
|
|
319
|
+
} catch {
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
for (const name of names) {
|
|
323
|
+
const match = ORPHAN_NAME_PATTERN.exec(name);
|
|
324
|
+
if (!match) {
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
const pid = Number(match[1]);
|
|
328
|
+
if (pid === process.pid || isProcessAlive(pid)) {
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
const uid = currentUid();
|
|
332
|
+
if (uid !== null) {
|
|
333
|
+
const entry = await lstat(path.join(cacheRoot, name)).catch(() => null);
|
|
334
|
+
if (!entry || entry.uid !== uid) {
|
|
335
|
+
continue;
|
|
164
336
|
}
|
|
337
|
+
}
|
|
338
|
+
await rm(path.join(cacheRoot, name), { recursive: true, force: true }).catch(
|
|
339
|
+
() => {},
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
165
343
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
344
|
+
// rename(2) of a directory is atomic: the root either does not exist or is
|
|
345
|
+
// complete. Losing the race to another process is success when its root
|
|
346
|
+
// passes the content check.
|
|
347
|
+
async function publishPatchedRoot(stagingDir, finalDir, identity) {
|
|
348
|
+
let lastError = null;
|
|
349
|
+
for (let attempt = 0; attempt < MAX_PUBLISH_ATTEMPTS; attempt += 1) {
|
|
350
|
+
try {
|
|
351
|
+
await rename(stagingDir, finalDir);
|
|
352
|
+
return;
|
|
353
|
+
} catch (error) {
|
|
354
|
+
if (!RENAME_COLLISION_CODES.has(error?.code)) {
|
|
355
|
+
throw error;
|
|
356
|
+
}
|
|
357
|
+
lastError = error;
|
|
358
|
+
}
|
|
359
|
+
if (await isPatchedRootValid(finalDir, identity)) {
|
|
360
|
+
return;
|
|
361
|
+
}
|
|
362
|
+
// What sits at finalDir is incomplete. Move it aside in one rename so no
|
|
363
|
+
// reader ever sees it half-deleted, then retry.
|
|
364
|
+
const staleDir = `${finalDir}${STALE_MARKER}${process.pid}-${randomBytes(4).toString("hex")}`;
|
|
365
|
+
try {
|
|
366
|
+
await rename(finalDir, staleDir);
|
|
367
|
+
} catch (error) {
|
|
368
|
+
if (error?.code === "ENOENT") {
|
|
369
|
+
continue;
|
|
370
|
+
}
|
|
169
371
|
throw error;
|
|
372
|
+
}
|
|
373
|
+
await rm(staleDir, { recursive: true, force: true });
|
|
374
|
+
}
|
|
375
|
+
throw new Error(
|
|
376
|
+
`Could not publish the patched emception root at ${finalDir}.`,
|
|
377
|
+
{ cause: lastError },
|
|
378
|
+
);
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/**
|
|
382
|
+
* Returns the shared patched emception root under `cacheRoot` (default
|
|
383
|
+
* `os.tmpdir()`), building it first if it is missing or fails the content
|
|
384
|
+
* check. Safe to call from any number of processes at once.
|
|
385
|
+
*/
|
|
386
|
+
export async function preparePatchedEmceptionRoot(options = {}) {
|
|
387
|
+
const cacheRoot = options.cacheRoot ?? os.tmpdir();
|
|
388
|
+
const sourceRoot =
|
|
389
|
+
options.sourceRoot ?? path.dirname(require.resolve("sdn-emception"));
|
|
390
|
+
const identity = await computeEmceptionIdentity(sourceRoot);
|
|
391
|
+
const finalDir = path.join(cacheRoot, `${PATCHED_ROOT_PREFIX}${identity.key}`);
|
|
392
|
+
if (await isPatchedRootValid(finalDir, identity)) {
|
|
393
|
+
return finalDir;
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
await removeOrphanedBuildDirs(cacheRoot);
|
|
397
|
+
const stagingDir = await mkdtemp(
|
|
398
|
+
`${finalDir}${STAGING_MARKER}${process.pid}-`,
|
|
399
|
+
);
|
|
400
|
+
try {
|
|
401
|
+
// Copy-on-write clone where the filesystem supports it (APFS, btrfs);
|
|
402
|
+
// a plain copy elsewhere.
|
|
403
|
+
await cp(sourceRoot, stagingDir, {
|
|
404
|
+
recursive: true,
|
|
405
|
+
mode: fsConstants.COPYFILE_FICLONE,
|
|
170
406
|
});
|
|
407
|
+
await patchEmceptionModuleTree(stagingDir, identity.files);
|
|
408
|
+
const files = [];
|
|
409
|
+
for (const relativePath of identity.files) {
|
|
410
|
+
const { size } = await stat(path.join(stagingDir, relativePath));
|
|
411
|
+
files.push([relativePath, size]);
|
|
412
|
+
}
|
|
413
|
+
await writeFile(
|
|
414
|
+
path.join(stagingDir, PATCH_MARKER_FILENAME),
|
|
415
|
+
`${JSON.stringify({
|
|
416
|
+
patchVersion: PATCH_VERSION,
|
|
417
|
+
emceptionVersion: identity.version,
|
|
418
|
+
fingerprint: identity.fingerprint,
|
|
419
|
+
files,
|
|
420
|
+
})}\n`,
|
|
421
|
+
"utf8",
|
|
422
|
+
);
|
|
423
|
+
await publishPatchedRoot(stagingDir, finalDir, identity);
|
|
424
|
+
} finally {
|
|
425
|
+
// A no-op after a successful rename; removes the copy after a lost race
|
|
426
|
+
// or a failure.
|
|
427
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
428
|
+
}
|
|
429
|
+
return finalDir;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function getPatchedEmceptionRoot() {
|
|
433
|
+
if (!patchedEmceptionRootPromise) {
|
|
434
|
+
patchedEmceptionRootPromise = preparePatchedEmceptionRoot().catch(
|
|
435
|
+
(error) => {
|
|
436
|
+
patchedEmceptionRootPromise = null;
|
|
437
|
+
throw error;
|
|
438
|
+
},
|
|
439
|
+
);
|
|
171
440
|
}
|
|
172
441
|
|
|
173
442
|
return patchedEmceptionRootPromise;
|
|
@@ -181,7 +450,7 @@ class EmceptionController {
|
|
|
181
450
|
if (!this.#instancePromise) {
|
|
182
451
|
this.#instancePromise = (async () => {
|
|
183
452
|
installNodeRuntimeShims();
|
|
184
|
-
const patchedRoot = await
|
|
453
|
+
const patchedRoot = await getPatchedEmceptionRoot();
|
|
185
454
|
const moduleUrl = pathToFileURL(path.join(patchedRoot, "emception.mjs")).href;
|
|
186
455
|
const { default: Emception } = await import(moduleUrl);
|
|
187
456
|
const emception = new Emception({
|
package/src/index.d.ts
CHANGED
|
@@ -978,7 +978,16 @@ export interface CompilationResult {
|
|
|
978
978
|
compiler: string;
|
|
979
979
|
language: string;
|
|
980
980
|
threadModel: ModuleThreadModelName;
|
|
981
|
+
/**
|
|
982
|
+
* The explicit `outputPath`, or the artifact inside `tempDir` when the
|
|
983
|
+
* caller passed `keepTempDir: true`. `null` otherwise: the compile's temp
|
|
984
|
+
* dir has already been removed, so use `wasmBytes`.
|
|
985
|
+
*/
|
|
981
986
|
outputPath: string | null;
|
|
987
|
+
/**
|
|
988
|
+
* Non-null only with `keepTempDir: true`. The caller owns it and frees it
|
|
989
|
+
* with `cleanupCompilation(result)`.
|
|
990
|
+
*/
|
|
982
991
|
tempDir: string | null;
|
|
983
992
|
wasmBytes: Uint8Array;
|
|
984
993
|
guestLink: GuestLinkArtifact | null;
|
|
@@ -1054,6 +1063,13 @@ export function compileModuleFromSource(options: {
|
|
|
1054
1063
|
language?: string;
|
|
1055
1064
|
threadModel?: ModuleThreadModelName;
|
|
1056
1065
|
outputPath?: string;
|
|
1066
|
+
/**
|
|
1067
|
+
* Keep the compile's temp dir (toolchain scratch, intermediate objects, and
|
|
1068
|
+
* the artifact when no `outputPath` is given) after a successful compile.
|
|
1069
|
+
* Default `false`: it is removed before the promise settles, on success and
|
|
1070
|
+
* on failure. A failed compile always removes it.
|
|
1071
|
+
*/
|
|
1072
|
+
keepTempDir?: boolean;
|
|
1057
1073
|
importedMemory?: boolean;
|
|
1058
1074
|
sharedMemory?: boolean;
|
|
1059
1075
|
initialMemoryBytes?: number;
|