seo-intel 1.1.9 → 1.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +13 -0
- package/crawler/index.js +4 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,18 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.1.10 (2026-03-27)
|
|
4
|
+
|
|
5
|
+
### Security
|
|
6
|
+
- Fix SSRF: llms.txt URLs now respect robots.txt before enqueue (crawler/index.js)
|
|
7
|
+
|
|
8
|
+
### Fixes
|
|
9
|
+
- SQL injection audit complete — all queries use parameterised statements (no changes needed)
|
|
10
|
+
|
|
11
|
+
### Testing
|
|
12
|
+
- Mock crawl test passes end-to-end: crawls http://localhost:19876, stores 7 pages in SQLite
|
|
13
|
+
- CI: Ubuntu job now runs mock crawl test after smoke checks
|
|
14
|
+
- Fixed mock-crawl-test.js: server binds to 127.0.0.1, CLI resolved from install root, DB assertions corrected
|
|
15
|
+
|
|
3
16
|
## 1.1.9 (2026-03-27)
|
|
4
17
|
|
|
5
18
|
### Security
|
package/crawler/index.js
CHANGED
|
@@ -233,6 +233,10 @@ export async function* crawlDomain(startUrl, opts = {}) {
|
|
|
233
233
|
} catch {
|
|
234
234
|
continue;
|
|
235
235
|
}
|
|
236
|
+
if (!opts.stealth) {
|
|
237
|
+
const robotsResult = await checkRobots(u).catch(() => ({ allowed: true }));
|
|
238
|
+
if (!robotsResult.allowed) continue;
|
|
239
|
+
}
|
|
236
240
|
if (!queue.some(q => q.url === u)) {
|
|
237
241
|
queue.push({ url: u, depth: 1 });
|
|
238
242
|
added++;
|