totem-llm 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/collector/processLink/convert/generic.js +1 -1
- package/collector/utils/extensions/WebsiteDepth/index.js +2 -2
- package/collector/utils/runtimeSettings/index.js +22 -3
- package/package.json +1 -1
- package/server/public/assets/{index-e77e714c.js → index-b6046b8c.js} +1 -1
- package/server/public/assets/{index-d6e7eb97.js → index-c493e3e5.js} +1 -1
- package/server/public/assets/{index-5d573563.js → index-e7cfd6f7.js} +1 -1
- package/server/public/index.js +2 -2
|
@@ -162,7 +162,7 @@ async function getPageContent({ link, captureAs = "text", headers = {} }) {
|
|
|
162
162
|
launchOptions: {
|
|
163
163
|
headless: launchConfig.headless,
|
|
164
164
|
ignoreHTTPSErrors: true,
|
|
165
|
-
args: runtimeSettings.
|
|
165
|
+
args: runtimeSettings.getBrowserLaunchArgs(),
|
|
166
166
|
},
|
|
167
167
|
gotoOptions: {
|
|
168
168
|
waitUntil: "networkidle2",
|
|
@@ -72,7 +72,7 @@ async function getPageLinks(url, baseUrl) {
|
|
|
72
72
|
launchOptions: {
|
|
73
73
|
headless: launchConfig.headless,
|
|
74
74
|
ignoreHTTPSErrors: true,
|
|
75
|
-
args: runtimeSettings.
|
|
75
|
+
args: runtimeSettings.getBrowserLaunchArgs(),
|
|
76
76
|
},
|
|
77
77
|
gotoOptions: { waitUntil: "networkidle2" },
|
|
78
78
|
});
|
|
@@ -138,7 +138,7 @@ async function bulkScrapePages(links, outFolderPath) {
|
|
|
138
138
|
launchOptions: {
|
|
139
139
|
headless: launchConfig.headless,
|
|
140
140
|
ignoreHTTPSErrors: true,
|
|
141
|
-
args: runtimeSettings.
|
|
141
|
+
args: runtimeSettings.getBrowserLaunchArgs(),
|
|
142
142
|
},
|
|
143
143
|
gotoOptions: { waitUntil: "networkidle2" },
|
|
144
144
|
async evaluate(page, browser) {
|
|
@@ -35,9 +35,15 @@ class RuntimeSettings {
|
|
|
35
35
|
default: [],
|
|
36
36
|
validate: (value) => {
|
|
37
37
|
let args = [];
|
|
38
|
-
if (Array.isArray(value))
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
if (Array.isArray(value)) {
|
|
39
|
+
args = value.map((arg) => String(arg).trim()).filter(Boolean);
|
|
40
|
+
}
|
|
41
|
+
if (typeof value === "string") {
|
|
42
|
+
args = value
|
|
43
|
+
.split(",")
|
|
44
|
+
.map((arg) => arg.trim())
|
|
45
|
+
.filter(Boolean);
|
|
46
|
+
}
|
|
41
47
|
return args;
|
|
42
48
|
},
|
|
43
49
|
},
|
|
@@ -79,6 +85,19 @@ class RuntimeSettings {
|
|
|
79
85
|
: this.settingConfigs[key].default;
|
|
80
86
|
}
|
|
81
87
|
|
|
88
|
+
/**
|
|
89
|
+
* Returns browser launch args, with Linux sandbox fallback when none are provided.
|
|
90
|
+
* @returns {string[]}
|
|
91
|
+
*/
|
|
92
|
+
getBrowserLaunchArgs() {
|
|
93
|
+
const args = this.get("browserLaunchArgs");
|
|
94
|
+
if (Array.isArray(args) && args.length > 0) return args;
|
|
95
|
+
if (process.platform === "linux") {
|
|
96
|
+
return ["--no-sandbox", "--disable-setuid-sandbox"];
|
|
97
|
+
}
|
|
98
|
+
return [];
|
|
99
|
+
}
|
|
100
|
+
|
|
82
101
|
/**
|
|
83
102
|
* Set a runtime setting
|
|
84
103
|
* - Will throw an error if the setting requested is not a supported runtime setting key
|